From 4f58676a595f43580b415bbc24a24bf60ecf27ad Mon Sep 17 00:00:00 2001 From: "Simms, Andrew" Date: Mon, 29 Jun 2026 07:46:29 -0600 Subject: [PATCH 01/26] Version: Bump to 1.1.0 --- mhkit/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mhkit/__init__.py b/mhkit/__init__.py index 6f9eefeec..0d248d505 100644 --- a/mhkit/__init__.py +++ b/mhkit/__init__.py @@ -11,7 +11,7 @@ configure_warnings() -__version__ = "v1.0.1" +__version__ = "v1.1.0" __copyright__ = """ Copyright 2019, Alliance for Energy Innovation, LLC under the terms of From 3cca334de21fd04b8870ebbabee7641bc8b84ce7 Mon Sep 17 00:00:00 2001 From: "Simms, Andrew" Date: Mon, 29 Jun 2026 07:48:18 -0600 Subject: [PATCH 02/26] Examples: Update NLR/NatLabRockies links --- examples/WPTO_hindcast_example.ipynb | 4 ++-- examples/metocean_example.ipynb | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/examples/WPTO_hindcast_example.ipynb b/examples/WPTO_hindcast_example.ipynb index 1f251b7a9..3a6fe3faf 100644 --- a/examples/WPTO_hindcast_example.ipynb +++ b/examples/WPTO_hindcast_example.ipynb @@ -51,7 +51,7 @@ "\n", "### Setting up Access to WPTO Hindcast Data\n", "To access the WPTO hindcast data, you will need to configure h5pyd for data access on HSDS. \n", - " To get your own API key, visit https://developer.nrel.gov/signup/. \n", + " To get your own API key, visit https://developer.nlr.gov/signup/. \n", "\n", "To configure h5phd type:\n", "\n", @@ -59,7 +59,7 @@ " \n", "and enter at the prompt:\n", "\n", - " hs_endpoint = https://developer.nrel.gov/api/hsds\n", + " hs_endpoint = https://developer.nlr.gov/api/hsds\n", " hs_username = None\n", " hs_password = None\n", " hs_api_key = {your key}\n", diff --git a/examples/metocean_example.ipynb b/examples/metocean_example.ipynb index 4f4540ab8..06098aeb2 100644 --- a/examples/metocean_example.ipynb +++ b/examples/metocean_example.ipynb @@ -567,7 +567,7 @@ "source": [ "## 2. Request Surface Wind Data from WIND Toolkit\n", "\n", - "MHKiT can be used to request historical data from [WIND Toolkit](https://www.nrel.gov/grid/wind-toolkit.html). The WIND Toolkit is a high-spatial-resolution dataset of meteorological parameters covering several offshore regions including California, Hawaii, the Northwest Pacific, and the Mid-Atlantic. The offshore datasets span 2000-2019 (or to 2020 for the Mid-Atlantic region). \n", + "MHKiT can be used to request historical data from [WIND Toolkit](https://www.nlr.gov/grid/wind-toolkit). The WIND Toolkit is a high-spatial-resolution dataset of meteorological parameters covering several offshore regions including California, Hawaii, the Northwest Pacific, and the Mid-Atlantic. The offshore datasets span 2000-2019 (or to 2020 for the Mid-Atlantic region). \n", "\n", "\n", "### Included Variables: \n", @@ -588,7 +588,7 @@ "\n", "### Setting up Access to WIND Toolkit\n", "To access the WIND Toolkit, you will need to configure h5pyd for data access on HSDS. \n", - "To get your own API key, visit https://developer.nrel.gov/signup/. \n", + "To get your own API key, visit https://developer.nlr.gov/signup/. \n", "If you have already accessed WPTO Hindcast data, you may skip this step.\n", "\n", "To configure h5pyd type:\n", @@ -597,7 +597,7 @@ " \n", "and enter at the prompt:\n", "\n", - " hs_endpoint = https://developer.nrel.gov/api/hsds\n", + " hs_endpoint = https://developer.nlr.gov/api/hsds\n", " hs_username = None\n", " hs_password = None\n", " hs_api_key = {your key}\n", @@ -615,7 +615,7 @@ "* location\n", "* year\n", "\n", - "A full list of available historical parameters can be found [here](https://github.com/NREL/hsds-examples/blob/master/datasets/WINDToolkit.md#model). Here we are interested in historical surface wind data (`'windspeed_10m'`, `'winddirection_10m'`) and a vertical temperature profile. For demonstration purposes, not all temperature elevations will be used. Additionally, we will use the latitude and longitude of NDBC buoy `'46022'` to later compare to the NDBC dataset. The buoy location can be found [here](https://www.ndbc.noaa.gov/data/stations/station_table.txt). The `elevation_to_string` can be used to easily combine an array of elevations with a parameter name to create the correctly formatted parameter string. This is demonstrated for the array of temperature parameters." + "A full list of available historical parameters can be found [here](https://github.com/NatLabRockies/hsds-examples/blob/master/datasets/WINDToolkit.md#model). Here we are interested in historical surface wind data (`'windspeed_10m'`, `'winddirection_10m'`) and a vertical temperature profile. For demonstration purposes, not all temperature elevations will be used. Additionally, we will use the latitude and longitude of NDBC buoy `'46022'` to later compare to the NDBC dataset. The buoy location can be found [here](https://www.ndbc.noaa.gov/data/stations/station_table.txt). The `elevation_to_string` can be used to easily combine an array of elevations with a parameter name to create the correctly formatted parameter string. This is demonstrated for the array of temperature parameters." ] }, { From e33f4c75c6643479652a2a38bd22b6d1d5ccca92 Mon Sep 17 00:00:00 2001 From: "Simms, Andrew" Date: Mon, 29 Jun 2026 07:51:58 -0600 Subject: [PATCH 03/26] Dev: Lint --- mhkit/wave/io/hindcast/wind_toolkit.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mhkit/wave/io/hindcast/wind_toolkit.py b/mhkit/wave/io/hindcast/wind_toolkit.py index a22cbe7ba..be269e3c6 100644 --- a/mhkit/wave/io/hindcast/wind_toolkit.py +++ b/mhkit/wave/io/hindcast/wind_toolkit.py @@ -210,7 +210,7 @@ def plot_region( supported_regions = ["Offshore_CA", "Hawaii", "Mid_Atlantic", "NW_Pacific"] if region not in supported_regions: raise ValueError( - f'{region} not in list of supported regions: {", ".join(supported_regions)}' + f"{region} not in list of supported regions: {', '.join(supported_regions)}" ) lats, lons = get_region_data(region) From 7479ddd4c260ebdbeca9546ab2cd23c7a57a6c28 Mon Sep 17 00:00:00 2001 From: "Simms, Andrew" Date: Mon, 29 Jun 2026 07:52:06 -0600 Subject: [PATCH 04/26] HSDS: Add note about HSDS domain and endpoint changes --- mhkit/wave/io/hindcast/wind_toolkit.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/mhkit/wave/io/hindcast/wind_toolkit.py b/mhkit/wave/io/hindcast/wind_toolkit.py index be269e3c6..596bf58e8 100644 --- a/mhkit/wave/io/hindcast/wind_toolkit.py +++ b/mhkit/wave/io/hindcast/wind_toolkit.py @@ -427,6 +427,9 @@ def request_wtk_point_data( else: raise TypeError("Coordinates must be within the same region!") + # NOTE: NREL to NLR name changes + # * developer.nlr.gov must be configured via ~/.hscfg + # * As of June 2026 the HSDS data domain is still "/nrel/...". if time_interval == "1-hour": wind_path = f"/nrel/wtk/{region.lower()}/{region}_*.h5" elif time_interval == "5-minute": From 321a7672f303cf85b194ca59665bf46ecd24176f Mon Sep 17 00:00:00 2001 From: "Simms, Andrew" Date: Mon, 29 Jun 2026 07:52:45 -0600 Subject: [PATCH 05/26] HSDS: Set endpoint to developer.nlr.gov --- .hscfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.hscfg b/.hscfg index f9aa99caa..c280dc421 100644 --- a/.hscfg +++ b/.hscfg @@ -1,4 +1,4 @@ -hs_endpoint = https://developer.nrel.gov/api/hsds +hs_endpoint = https://developer.nlr.gov/api/hsds hs_username = hs_password = hs_api_key = jODGciIBnejrYd9GXxgXjbbAjMDLBMWQer05P98N From e7c1e4e11a2c04279d8fdb71e88d40367c32192d Mon Sep 17 00:00:00 2001 From: "Simms, Andrew" Date: Mon, 29 Jun 2026 15:28:02 -0600 Subject: [PATCH 06/26] Fix: Add alternative way to access meta for rex/hsds hindcast calls --- mhkit/wave/io/hindcast/hindcast.py | 76 +++++++++++++++++++++++++++--- 1 file changed, 70 insertions(+), 6 deletions(-) diff --git a/mhkit/wave/io/hindcast/hindcast.py b/mhkit/wave/io/hindcast/hindcast.py index 83119a782..a6481a480 100644 --- a/mhkit/wave/io/hindcast/hindcast.py +++ b/mhkit/wave/io/hindcast/hindcast.py @@ -5,8 +5,8 @@ regions, request point data for various parameters, and request directional spectrum data. -Author: rpauly, aidanbharath, ssolson -Date: 2023-09-26 +Author: rpauly, aidanbharath, ssolson, simmsa +Date: 2026-06-29 """ import os @@ -16,11 +16,77 @@ import pandas as pd import xarray as xr import numpy as np +import h5pyd from rex import MultiYearWaveX, WaveX +from rex.utilities.exceptions import ResourceRuntimeError from mhkit.utils.cache import handle_caching from mhkit.utils.type_handling import convert_to_dataset +def _meta_s3_uri(rex_waves: Union[WaveX, MultiYearWaveX]) -> str: + """ + Extract the object store (AWS S3) uri from the metadata of rex h5/HSDS file object + + HSDS serves a virtual file whose bytes live in a separate S3 .h5 object. + In rex the domain/dataset name does not reveal that object's S3 uri but + HSDS records the S3 path per dataset as metadata. + + Parameters + ---------- + rex_waves : WaveX or MultiYearWaveX + Open rex resource on HSDS + + Returns + ------- + s3_uri : string + s3:// path of the underlying .h5 file + """ + # rex exposes the h5 object differentely by resource type: a + # single-year WaveX is the h5pyd file itself (has `.filename`), while + # MultiYearWaveX wraps one WaveResource per year in a MultiYearH5 (has + # `.h5_files`). This works with both types of inputs to yield a single h5py filename + h5 = rex_waves.h5 + domain = h5.h5_files[0] if hasattr(h5, "h5_files") else h5.filename + with h5pyd.File(domain, mode="r") as h5_file: + # The h5 object metadata holds file information in the id.dcpl_json object + # DCPL = Dataset Creation Property List + # dcpl_json spec layout.file_uri spec: + # https://github.com/HDFGroup/hsds/blob/master/docs/design/single_object/SingleObject.md + return h5_file["meta"].id.dcpl_json["layout"]["file_uri"] + + +def _meta_for_gids(rex_waves: Union[WaveX, MultiYearWaveX], gids) -> pd.DataFrame: + """ + Returns meta rows for the given grid ids. + + Reads meta over HSDS, falling back to reading directly from the AWS .h5 + file on S3 when the HSDS read fails. + + Parameters + ---------- + rex_waves : WaveX or MultiYearWaveX + Open rex resource on HSDS + gids : int, list, or array of int + Grid ids to read + + Returns + ------- + meta : pandas.DataFrame + Meta rows for the given grid ids, indexed from 0 + """ + gids = [int(g) for g in np.atleast_1d(gids)] + try: + meta = rex_waves.meta + except ResourceRuntimeError: + # Reading the meta dataset over HSDS returns an empty response for the + # West_Coast and Atlantic regions, raised here as ResourceRuntimeError. + # Their data reads fine over HSDS, only the meta read fails. Fall back + # to reading the requested rows directly from the AWS .h5 file on S3. + with WaveX(_meta_s3_uri(rex_waves), hsds=False) as s3_waves: + return s3_waves["meta", gids].reset_index(drop=True) + return meta.loc[gids, :].reset_index(drop=True) + + def region_selection(lat_lon: Union[List[float], Tuple[float, float]]) -> str: """ Returns the name of the predefined region in which the given @@ -250,8 +316,7 @@ def request_wpto_point_data( temp = f"{parameter}_{i}" data = data.rename(columns={col: temp}) - meta = rex_waves.meta.loc[cols, :] - meta = meta.reset_index(drop=True) + meta = _meta_for_gids(rex_waves, cols) gid = rex_waves.lat_lon_gid(lat_lon) meta["gid"] = gid @@ -449,8 +514,7 @@ def request_wpto_directional_spectrum( data["time_index"] = pd.to_datetime(data.time_index) # Get metadata - meta = rex_waves.meta.loc[columns, :] - meta = meta.reset_index(drop=True) + meta = _meta_for_gids(rex_waves, columns) meta["gid"] = gid # Convert gid to integer or list of integers From 9c06a39d4827a3731be30bfa6bf016428b3dd31f Mon Sep 17 00:00:00 2001 From: "Simms, Andrew" Date: Mon, 29 Jun 2026 16:21:37 -0600 Subject: [PATCH 07/26] Fix: hindcast, Use s3 fallback for any HSDS read --- mhkit/wave/io/hindcast/hindcast.py | 107 ++++++++++++++++++----------- 1 file changed, 66 insertions(+), 41 deletions(-) diff --git a/mhkit/wave/io/hindcast/hindcast.py b/mhkit/wave/io/hindcast/hindcast.py index a6481a480..700a766b0 100644 --- a/mhkit/wave/io/hindcast/hindcast.py +++ b/mhkit/wave/io/hindcast/hindcast.py @@ -77,16 +77,64 @@ def _meta_for_gids(rex_waves: Union[WaveX, MultiYearWaveX], gids) -> pd.DataFram gids = [int(g) for g in np.atleast_1d(gids)] try: meta = rex_waves.meta - except ResourceRuntimeError: + except (ResourceRuntimeError, IOError): # Reading the meta dataset over HSDS returns an empty response for the - # West_Coast and Atlantic regions, raised here as ResourceRuntimeError. - # Their data reads fine over HSDS, only the meta read fails. Fall back - # to reading the requested rows directly from the AWS .h5 file on S3. + # West_Coast and Atlantic regions. rex and h5pyd surface this as either + # ResourceRuntimeError or IOError depending on the installed version. + # The data reads fine over HSDS, only the meta read fails. Fall back to + # reading the requested rows directly from the AWS .h5 file on S3. with WaveX(_meta_s3_uri(rex_waves), hsds=False) as s3_waves: return s3_waves["meta", gids].reset_index(drop=True) return meta.loc[gids, :].reset_index(drop=True) +def _read_point_data( + rex_waves: Union[WaveX, MultiYearWaveX], + parameter: Union[str, List[str]], + lat_lon, +) -> Tuple[pd.DataFrame, pd.DataFrame]: + """ + Read renamed point time-series data and meta (with gid) from an open rex + resource. Works against either an HSDS-backed or S3-backed resource. + + Parameters + ---------- + rex_waves : WaveX or MultiYearWaveX + Open rex resource + parameter : string or list of strings + Parameter(s) to read + lat_lon : tuple or list of tuples + Latitude/longitude point(s) to read + + Returns + ------- + data : pandas.DataFrame + Time-series data with columns renamed to ``{parameter}_{i}`` + meta : pandas.DataFrame + Meta rows for the read columns, with a ``gid`` column + """ + if isinstance(parameter, list): + data_list = [] + for param in parameter: + temp_data = rex_waves.get_lat_lon_df(param, lat_lon) + cols = temp_data.columns[:] + temp_data = temp_data.rename( + columns={col: f"{param}_{i}" for i, col in enumerate(cols)} + ) + data_list.append(temp_data) + data = pd.concat(data_list, axis=1) + else: + data = rex_waves.get_lat_lon_df(parameter, lat_lon) + cols = data.columns[:] + data = data.rename( + columns={col: f"{parameter}_{i}" for i, col in enumerate(cols)} + ) + + meta = _meta_for_gids(rex_waves, cols) + meta["gid"] = rex_waves.lat_lon_gid(lat_lon) + return data, meta + + def region_selection(lat_lon: Union[List[float], Tuple[float, float]]) -> str: """ Returns the name of the predefined region in which the given @@ -293,48 +341,25 @@ def request_wpto_point_data( "hsds": hsds, "years": years, } - data_list = [] - with MultiYearWaveX(wave_path, **wave_kwargs) as rex_waves: - if isinstance(parameter, list): - for param in parameter: - temp_data = rex_waves.get_lat_lon_df(param, lat_lon) - gid = rex_waves.lat_lon_gid(lat_lon) - cols = temp_data.columns[:] - for i, col in zip(range(len(cols)), cols): - temp = f"{param}_{i}" - temp_data = temp_data.rename(columns={col: temp}) + data, meta = _read_point_data(rex_waves, parameter, lat_lon) - data_list.append(temp_data) - data = pd.concat(data_list, axis=1) - - else: - data = rex_waves.get_lat_lon_df(parameter, lat_lon) - cols = data.columns[:] - - for i, col in zip(range(len(cols)), cols): - temp = f"{parameter}_{i}" - data = data.rename(columns={col: temp}) - - meta = _meta_for_gids(rex_waves, cols) - gid = rex_waves.lat_lon_gid(lat_lon) - meta["gid"] = gid - - if not to_pandas: - data = convert_to_dataset(data) - data["time_index"] = pd.to_datetime(data.time_index) + if not to_pandas: + data = convert_to_dataset(data) + data["time_index"] = pd.to_datetime(data.time_index) - if isinstance(parameter, list): - param_coords = [f"{param}_{i}" for param in parameter] - data.coords["parameter"] = xr.DataArray(param_coords, dims="parameter") + if isinstance(parameter, list): + n_loc = 1 if isinstance(lat_lon[0], float) else len(lat_lon) + param_coords = [f"{param}_{n_loc - 1}" for param in parameter] + data.coords["parameter"] = xr.DataArray(param_coords, dims="parameter") - data.coords["year"] = xr.DataArray(years, dims="year") + data.coords["year"] = xr.DataArray(years, dims="year") - meta_ds = meta.to_xarray() - data = xr.merge([data, meta_ds]) + meta_ds = meta.to_xarray() + data = xr.merge([data, meta_ds]) - # Remove the 'index' coordinate - data = data.drop_vars("index") + # Remove the 'index' coordinate + data = data.drop_vars("index") # save_to_cache(hash_params, data, meta) handle_caching( @@ -494,7 +519,7 @@ def request_wpto_directional_spectrum( try: data_array = rex_waves[parameter, bins[i] : bins[i + 1], :, :, gid] str_error = None - except OSError as err: + except IOError as err: str_error = str(err) if str_error: From cec82ef0182578f6ee0a0acb240e2a60d32035e3 Mon Sep 17 00:00:00 2001 From: "Simms, Andrew" Date: Mon, 29 Jun 2026 17:00:57 -0600 Subject: [PATCH 08/26] Deps: Update rex to NLR-rex with hsds feature --- environment-dev.yml | 7 ++++++- pyproject.toml | 2 +- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/environment-dev.yml b/environment-dev.yml index e624a7286..455c8eaeb 100644 --- a/environment-dev.yml +++ b/environment-dev.yml @@ -26,5 +26,10 @@ dependencies: - notebook - matplotlib>=3.9.1 - fatpack - - nrel-rex - cartopy + # rex is installed from PyPI (NLR-rex), not conda-forge. The conda-forge + # `nrel-rex` is frozen at 0.2.84, which predates the s3:// fsspec support + # needed to read the wave hindcast .h5 files directly from S3. The [hsds] extra + # pulls the HSDS client dependencies the hindcast reads use. + - pip: + - NLR-rex[hsds]>=0.5.0 diff --git a/pyproject.toml b/pyproject.toml index c9805d480..02386a2b8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -47,7 +47,7 @@ wave = [ "scikit-learn>=1.5.1", "statsmodels>=0.14.2", "pytz", - "NREL-rex>=0.2.63", + "NLR-rex[hsds]>=0.5.0", "beautifulsoup4", "bottleneck", "lxml", From 84fcd864d6f3ce1e39b3ec7ed02507b1b71390e5 Mon Sep 17 00:00:00 2001 From: "Simms, Andrew" Date: Mon, 29 Jun 2026 17:40:24 -0600 Subject: [PATCH 09/26] HSDS: Store and use HSDS api key as Github Secret --- .github/workflows/main.yml | 4 ++++ .hscfg | 6 +++++- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 7ca82b44d..ddd6a427e 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -23,6 +23,10 @@ env: # Coveralls: Set to false to ignore upload failures during outages. # Check status: https://status.coveralls.io/ COVERALLS_FAIL_ON_ERROR: true + # HSDS API key for the wave/wind hindcast reads. h5pyd reads HS_API_KEY from + # the environment and it overrides the (blank) hs_api_key in .hscfg, so the + # key stays out of the repo. Set the HSDS_API_KEY secret in repo settings. + HS_API_KEY: ${{ secrets.HSDS_API_KEY }} jobs: # Set the operating system matrix for the following jobs/tests diff --git a/.hscfg b/.hscfg index c280dc421..c0040489d 100644 --- a/.hscfg +++ b/.hscfg @@ -1,4 +1,8 @@ hs_endpoint = https://developer.nlr.gov/api/hsds hs_username = hs_password = -hs_api_key = jODGciIBnejrYd9GXxgXjbbAjMDLBMWQer05P98N +# hs_api_key is intentionally blank. It is provided at runtime via the +# HS_API_KEY environment variable (the HSDS_API_KEY GitHub secret in CI, or +# your own HS_API_KEY / ~/.hscfg locally), which h5pyd reads in preference to +# this file. Do not commit a real key here. +hs_api_key = From 846419795c24ed527152b88488814fa6db8e1158 Mon Sep 17 00:00:00 2001 From: "Simms, Andrew" Date: Mon, 29 Jun 2026 18:47:08 -0600 Subject: [PATCH 10/26] Revert "HSDS: Store and use HSDS api key as Github Secret" This reverts commit 84fcd864d6f3ce1e39b3ec7ed02507b1b71390e5. --- .github/workflows/main.yml | 4 ---- .hscfg | 6 +----- 2 files changed, 1 insertion(+), 9 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index ddd6a427e..7ca82b44d 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -23,10 +23,6 @@ env: # Coveralls: Set to false to ignore upload failures during outages. # Check status: https://status.coveralls.io/ COVERALLS_FAIL_ON_ERROR: true - # HSDS API key for the wave/wind hindcast reads. h5pyd reads HS_API_KEY from - # the environment and it overrides the (blank) hs_api_key in .hscfg, so the - # key stays out of the repo. Set the HSDS_API_KEY secret in repo settings. - HS_API_KEY: ${{ secrets.HSDS_API_KEY }} jobs: # Set the operating system matrix for the following jobs/tests diff --git a/.hscfg b/.hscfg index c0040489d..c280dc421 100644 --- a/.hscfg +++ b/.hscfg @@ -1,8 +1,4 @@ hs_endpoint = https://developer.nlr.gov/api/hsds hs_username = hs_password = -# hs_api_key is intentionally blank. It is provided at runtime via the -# HS_API_KEY environment variable (the HSDS_API_KEY GitHub secret in CI, or -# your own HS_API_KEY / ~/.hscfg locally), which h5pyd reads in preference to -# this file. Do not commit a real key here. -hs_api_key = +hs_api_key = jODGciIBnejrYd9GXxgXjbbAjMDLBMWQer05P98N From 973ba90268053ead973ff4690184405efd526a0b Mon Sep 17 00:00:00 2001 From: "Simms, Andrew" Date: Tue, 30 Jun 2026 08:21:05 -0600 Subject: [PATCH 11/26] Hindcast: Add fallback to read wave hindcast h5 data directly from S3 --- mhkit/wave/io/hindcast/hindcast.py | 519 +++++++++++++++++++++++++---- 1 file changed, 459 insertions(+), 60 deletions(-) diff --git a/mhkit/wave/io/hindcast/hindcast.py b/mhkit/wave/io/hindcast/hindcast.py index 700a766b0..99a06384b 100644 --- a/mhkit/wave/io/hindcast/hindcast.py +++ b/mhkit/wave/io/hindcast/hindcast.py @@ -11,61 +11,217 @@ import os import sys +import warnings from time import sleep from typing import List, Tuple, Union, Optional, Dict import pandas as pd import xarray as xr import numpy as np -import h5pyd +import fsspec +from scipy.spatial import cKDTree from rex import MultiYearWaveX, WaveX from rex.utilities.exceptions import ResourceRuntimeError from mhkit.utils.cache import handle_caching from mhkit.utils.type_handling import convert_to_dataset +# Public AWS S3 bucket mirroring the WPTO US Wave hindcast .h5 files served by +# HSDS. Used as a fallback when HSDS is unavailable: the same data is read +# directly from S3. https://registry.opendata.aws/wpto-pds-us-wave/ +_WAVE_S3_BUCKET = "wpto-pds-us-wave" -def _meta_s3_uri(rex_waves: Union[WaveX, MultiYearWaveX]) -> str: +# Retry count for the initial HSDS open in the hybrid read. Kept low so an +# unresponsive HSDS server fails over to the S3 fallback in seconds rather than +# waiting out h5pyd's default of 10 retries with exponential backoff. +_HSDS_OPEN_RETRIES = 2 + + +def _latest_s3_version(region_path: str) -> str: """ - Extract the object store (AWS S3) uri from the metadata of rex h5/HSDS file object + Returns the newest version directory in the wave S3 bucket that contains + the given region path. - HSDS serves a virtual file whose bytes live in a separate S3 .h5 object. - In rex the domain/dataset name does not reveal that object's S3 uri but - HSDS records the S3 path per dataset as metadata. + The bucket stores each release under a version directory (e.g. "v1.0.1") + and not every region exists in every version, so the version is resolved by + listing the bucket rather than assumed. Parameters ---------- - rex_waves : WaveX or MultiYearWaveX - Open rex resource on HSDS + region_path : string + Region path under a version directory, e.g. "West_Coast" or + "virtual_buoy/West_Coast" + + Returns + ------- + version : string + Newest version directory containing the region, e.g. "v1.0.1" + """ + file_system = fsspec.filesystem("s3", anon=True) + versions = sorted( + ( + entry.rsplit("/", 1)[-1] + for entry in file_system.ls(_WAVE_S3_BUCKET) + if entry.rsplit("/", 1)[-1].startswith("v") + ), + reverse=True, + ) + for version in versions: + if file_system.exists(f"{_WAVE_S3_BUCKET}/{version}/{region_path}"): + return version + raise FileNotFoundError( + f"{region_path} not found in any version of s3://{_WAVE_S3_BUCKET}" + ) + + +def _s3_wave_path(region: str, data_type: str, year: str = "*") -> str: + """ + Builds the s3:// path for a wave region, mirroring the HSDS domain layout. + + Parameters + ---------- + region : string + Region name, e.g. "West_Coast" + data_type : string + "3-hour" for the regular wave files or "1-hour" for the virtual_buoy + files + year : string or int + Year to read, or "*" for a multi-year wildcard glob. Default "*". Returns ------- - s3_uri : string - s3:// path of the underlying .h5 file + s3_path : string + s3:// path (or wildcard glob) of the underlying .h5 file(s) """ - # rex exposes the h5 object differentely by resource type: a - # single-year WaveX is the h5pyd file itself (has `.filename`), while - # MultiYearWaveX wraps one WaveResource per year in a MultiYearH5 (has - # `.h5_files`). This works with both types of inputs to yield a single h5py filename - h5 = rex_waves.h5 - domain = h5.h5_files[0] if hasattr(h5, "h5_files") else h5.filename - with h5pyd.File(domain, mode="r") as h5_file: - # The h5 object metadata holds file information in the id.dcpl_json object - # DCPL = Dataset Creation Property List - # dcpl_json spec layout.file_uri spec: - # https://github.com/HDFGroup/hsds/blob/master/docs/design/single_object/SingleObject.md - return h5_file["meta"].id.dcpl_json["layout"]["file_uri"] - - -def _meta_for_gids(rex_waves: Union[WaveX, MultiYearWaveX], gids) -> pd.DataFrame: + if data_type == "1-hour": + region_path = f"virtual_buoy/{region}" + filename = f"{region}_virtual_buoy_{year}.h5" + else: + region_path = region + filename = f"{region}_wave_{year}.h5" + version = _latest_s3_version(region_path) + return f"s3://{_WAVE_S3_BUCKET}/{version}/{region_path}/{filename}" + + +def _open_wave_resource( + resource_cls, + hsds_path: str, + wave_kwargs: dict, + s3_region: str, + s3_data_type: str, + s3_year: str = "*", + s3_fallback: bool = True, +): """ - Returns meta rows for the given grid ids. + Opens a rex wave resource on HSDS, falling back to a direct S3 read when the + HSDS open fails (e.g. the HSDS server is unavailable). - Reads meta over HSDS, falling back to reading directly from the AWS .h5 - file on S3 when the HSDS read fails. + Returns an open resource. The caller closes it, typically with a `with` + block. The HSDS and S3 reads return the same data, so the caller's read + logic does not change with the source. Parameters ---------- + resource_cls : type + rex resource class to open, WaveX or MultiYearWaveX + hsds_path : string + HSDS domain path to open + wave_kwargs : dict + Keyword arguments for the resource, including hsds and (for + MultiYearWaveX) years + s3_region : string + Region name used to build the S3 path for the fallback + s3_data_type : string + "3-hour" or "1-hour", used to build the S3 path for the fallback + s3_year : string or int + Year used to build the S3 path for the fallback. Default "*". + s3_fallback : bool + Whether to fall back to S3 when the HSDS open fails. Disabled when the + caller is not reading the default HSDS files (custom path or hsds=False). + + Returns + ------- rex_waves : WaveX or MultiYearWaveX - Open rex resource on HSDS + Open rex resource, HSDS-backed when available else S3-backed + """ + hsds_open_kwargs = dict(wave_kwargs) + if wave_kwargs.get("hsds"): + hsds_open_kwargs["hsds_kwargs"] = {"retries": _HSDS_OPEN_RETRIES} + try: + return resource_cls(hsds_path, **hsds_open_kwargs) + except (ResourceRuntimeError, IOError): + if not s3_fallback: + raise + # HSDS is unavailable. Read the same data directly from the public S3 + # bucket. rex reads s3:// paths with h5py and fsspec when hsds=False. + warnings.warn( + "HSDS is unavailable; falling back to reading the wave data " + "directly from S3, which is slower.", + UserWarning, + ) + s3_path = _s3_wave_path(s3_region, s3_data_type, s3_year) + return resource_cls(s3_path, **{**wave_kwargs, "hsds": False}) + + +def _meta_cache_path(region: str, data_type: str) -> str: + """Local parquet path for a wave dataset's cached coordinate metadata.""" + kind = "virtual_buoy" if data_type == "1-hour" else "wave" + return os.path.join(_get_cache_dir(), "meta", f"{region}_{kind}.parquet") + + +def _cached_meta(region: str, data_type: str) -> pd.DataFrame: + """ + Returns the location metadata (latitude, longitude, ...) for a wave dataset, + indexed by grid id (gid), from a local parquet cache. + + The coordinates are fixed per region across all years, so they are read from + S3 once and cached. Every later spatial query reuses the local copy instead + of re-reading hundreds of thousands of points from S3. + + Parameters + ---------- + region : string + Region name, e.g. "West_Coast" + data_type : string + "3-hour" for the regular wave files or "1-hour" for the virtual_buoy + files + + Returns + ------- + meta : pandas.DataFrame + Location metadata indexed by gid + """ + cache_path = _meta_cache_path(region, data_type) + if os.path.isfile(cache_path): + return pd.read_parquet(cache_path) + + region_path = f"virtual_buoy/{region}" if data_type == "1-hour" else region + version = _latest_s3_version(region_path) + file_system = fsspec.filesystem("s3", anon=True) + h5_files = [ + entry + for entry in file_system.ls(f"{_WAVE_S3_BUCKET}/{version}/{region_path}") + if entry.endswith(".h5") + ] + # The coordinates are identical across years, so any year's file works. + with WaveX(f"s3://{h5_files[0]}", hsds=False) as s3_waves: + meta = s3_waves.meta + os.makedirs(os.path.dirname(cache_path), exist_ok=True) + meta.to_parquet(cache_path) + return meta + + +def _meta_tree(meta: pd.DataFrame) -> cKDTree: + """Returns a cKDTree of the (latitude, longitude) coordinates in meta.""" + return cKDTree(meta[["latitude", "longitude"]].to_numpy()) + + +def _meta_for_gids(meta: pd.DataFrame, gids) -> pd.DataFrame: + """ + Returns cached meta rows for the given grid ids, indexed from 0. + + Parameters + ---------- + meta : pandas.DataFrame + Cached location metadata indexed by gid gids : int, list, or array of int Grid ids to read @@ -75,28 +231,213 @@ def _meta_for_gids(rex_waves: Union[WaveX, MultiYearWaveX], gids) -> pd.DataFram Meta rows for the given grid ids, indexed from 0 """ gids = [int(g) for g in np.atleast_1d(gids)] - try: - meta = rex_waves.meta - except (ResourceRuntimeError, IOError): - # Reading the meta dataset over HSDS returns an empty response for the - # West_Coast and Atlantic regions. rex and h5pyd surface this as either - # ResourceRuntimeError or IOError depending on the installed version. - # The data reads fine over HSDS, only the meta read fails. Fall back to - # reading the requested rows directly from the AWS .h5 file on S3. - with WaveX(_meta_s3_uri(rex_waves), hsds=False) as s3_waves: - return s3_waves["meta", gids].reset_index(drop=True) return meta.loc[gids, :].reset_index(drop=True) +def cache_wave_meta( + lat_lon: Optional[Union[Tuple[float, float], List[Tuple[float, float]]]] = None, + data_type: str = "3-hour", + datasets: Optional[List[Tuple[str, str]]] = None, +) -> None: + """ + Pre-builds the local coordinate metadata cache so spatial queries never read + coordinates from S3 at request time. + + This is optional. The request functions already cache each region's + coordinates lazily on first use, downloading only the region a request + falls in. Use this to warm the cache ahead of time, scoped to what is + needed: pass lat_lon to cache only the region(s) those points fall in, pass + datasets to cache an explicit list, or pass nothing to cache every region. + + Parameters + ---------- + lat_lon : tuple or list of tuples, optional + Point(s) whose region(s) to cache. The region is inferred from each + point, so only the regions to be queried are downloaded. + data_type : string, optional + "3-hour" or "1-hour", paired with lat_lon. Default "3-hour". + datasets : list of (region, data_type) tuples, optional + Explicit datasets to cache, instead of inferring from lat_lon. + """ + if datasets is None: + if lat_lon is not None: + points = lat_lon if isinstance(lat_lon[0], (list, tuple)) else [lat_lon] + regions = {region_selection(point) for point in points} + datasets = [(region, data_type) for region in regions] + else: + datasets = [ + ("West_Coast", "3-hour"), + ("Atlantic", "3-hour"), + ("Hawaii", "3-hour"), + ("West_Coast", "1-hour"), + ] + for region, dataset_type in datasets: + _cached_meta(region, dataset_type) + + +def _block_cache_path( + region: str, data_type: str, years: List[int], parameter: str, block: int +) -> str: + """Local parquet path for one cached location chunk (a block of gids).""" + kind = "virtual_buoy" if data_type == "1-hour" else "wave" + span = "_".join(str(y) for y in sorted(years)) + name = f"{region}_{kind}_{span}_{parameter}_block{block}.parquet" + return os.path.join(_get_cache_dir(), "blocks", name) + + +def _data_block_size(rex_waves: Union[WaveX, MultiYearWaveX], parameter: str) -> int: + """ + Returns the number of locations stored per chunk (the gid-dimension chunk + size) for a data variable. + + The data is chunked across locations, so reading a whole chunk costs the + same fetch as reading one location in it. This is the natural block size for + the location cache. + """ + for obj in (getattr(rex_waves, "resource", None), rex_waves): + chunks = getattr(obj, "chunks", None) + if isinstance(chunks, dict) and chunks.get(parameter): + return int(chunks[parameter][-1]) + raise ValueError(f"Could not determine the chunk size for {parameter}") + + +def _gid_block( + rex_waves: Union[WaveX, MultiYearWaveX], + parameter: str, + gid: int, + block_size: int, + n_gids: int, + region: str, + data_type: str, + years: List[int], +) -> pd.DataFrame: + """ + Returns the time series for the whole chunk of locations containing gid, + from a local cache, reading and caching it from the resource on a miss. + + One location's chunk also holds its neighbors, so reading and caching the + whole chunk costs the same fetch as a single location but serves every + location in the chunk on later queries. Columns are the gids in the block, + as strings. + """ + block = gid // block_size + cache_path = _block_cache_path(region, data_type, years, parameter, block) + if os.path.isfile(cache_path): + return pd.read_parquet(cache_path) + + start = block * block_size + stop = min(start + block_size, n_gids) + block_df = rex_waves.get_gid_df(parameter, list(range(start, stop))) + block_df.columns = block_df.columns.astype(str) + os.makedirs(os.path.dirname(cache_path), exist_ok=True) + block_df.to_parquet(cache_path) + return block_df + + +# A network read (HSDS or S3) can fail transiently, for example when the HSDS +# server is busy. Retry a failed read this many times, doubling the wait each +# time, before giving up. +_READ_ATTEMPTS = 4 +_READ_BACKOFF_SECONDS = 2 + + +def _read_with_retry(read): + """ + Calls read, retrying a transient network failure with exponential backoff. + + Parameters + ---------- + read : callable + Zero-argument function that performs the read and returns its result + + Returns + ------- + result + The value returned by read + """ + delay = _READ_BACKOFF_SECONDS + for attempt in range(_READ_ATTEMPTS): + try: + return read() + except (ResourceRuntimeError, IOError): + if attempt == _READ_ATTEMPTS - 1: + raise + sleep(delay) + delay *= 2 + + +def _read_point_blocked( + rex_waves: Union[WaveX, MultiYearWaveX], + parameter: Union[str, List[str]], + lat_lon, + meta_cache: pd.DataFrame, + region: str, + data_type: str, + years: List[int], +) -> Tuple[pd.DataFrame, pd.DataFrame]: + """ + Like _read_point_data, but reads and caches the whole location chunk around + each requested point. Nearby and repeat queries are then served from the + local cache. See the cache_block option of request_wpto_point_data. + + Parameters + ---------- + rex_waves : WaveX or MultiYearWaveX + Open rex resource + parameter : string or list of strings + Parameter(s) to read + lat_lon : tuple or list of tuples + Latitude/longitude point(s) to read + meta_cache : pandas.DataFrame + Cached location metadata indexed by gid + region : string + Region name, used in the block cache path + data_type : string + "3-hour" or "1-hour", used in the block cache path + years : list of int + Years read, used in the block cache path + + Returns + ------- + data : pandas.DataFrame + Time-series data with columns renamed to ``{parameter}_{i}`` + meta : pandas.DataFrame + Meta rows for the read columns, with a ``gid`` column + """ + parameters = parameter if isinstance(parameter, list) else [parameter] + gids = [int(g) for g in np.atleast_1d(rex_waves.lat_lon_gid(lat_lon))] + n_gids = len(meta_cache) + block_size = _data_block_size(rex_waves, parameters[0]) + time_index = rex_waves.time_index + + columns = {} + for param in parameters: + for i, gid in enumerate(gids): + block_df = _gid_block( + rex_waves, param, gid, block_size, n_gids, region, data_type, years + ) + columns[f"{param}_{i}"] = block_df[str(gid)].to_numpy() + data = pd.DataFrame(columns, index=time_index) + + meta = _meta_for_gids(meta_cache, gids) + meta["gid"] = rex_waves.lat_lon_gid(lat_lon) + return data, meta + + def _read_point_data( rex_waves: Union[WaveX, MultiYearWaveX], parameter: Union[str, List[str]], lat_lon, + meta_cache: Optional[pd.DataFrame], ) -> Tuple[pd.DataFrame, pd.DataFrame]: """ Read renamed point time-series data and meta (with gid) from an open rex resource. Works against either an HSDS-backed or S3-backed resource. + The spatial lookup uses the resource's pre-built tree and the meta comes + from the cached coordinate metadata, so no coordinates are read from the + resource. + Parameters ---------- rex_waves : WaveX or MultiYearWaveX @@ -105,6 +446,8 @@ def _read_point_data( Parameter(s) to read lat_lon : tuple or list of tuples Latitude/longitude point(s) to read + meta_cache : pandas.DataFrame + Cached location metadata indexed by gid Returns ------- @@ -130,7 +473,11 @@ def _read_point_data( columns={col: f"{parameter}_{i}" for i, col in enumerate(cols)} ) - meta = _meta_for_gids(rex_waves, cols) + # A custom-path read has no cached coordinate metadata, so read the meta + # from the resource itself in that case. + if meta_cache is None: + meta_cache = rex_waves.meta + meta = _meta_for_gids(meta_cache, cols) meta["gid"] = rex_waves.lat_lon_gid(lat_lon) return data, meta @@ -199,6 +546,7 @@ def request_wpto_point_data( hsds: bool = True, path: Optional[str] = None, to_pandas: bool = True, + cache_block: bool = False, ) -> Tuple[Union[pd.DataFrame, xr.Dataset], pd.DataFrame]: """ Returns data from the WPTO wave hindcast hosted on AWS at the @@ -254,6 +602,20 @@ def request_wpto_point_data( `hsds=False`. to_pandas: bool (optional) Flag to output pandas instead of xarray. Default = True. + cache_block : bool (optional) + Read and cache the whole location chunk around each requested point, + rather than only the requested location. Default = False. + + The hindcast data is chunked across locations, so fetching one location + already transfers a chunk of roughly a thousand neighboring locations. + With ``cache_block=True`` that whole chunk is kept and cached locally + (under the hindcast cache directory, one parquet file per + region/years/parameter/chunk), so later requests for any nearby location + and the same years and parameters are served from disk without another + download. This trades local disk for far fewer downloads and is useful + when analyzing many points in the same area. The returned data is + identical to ``cache_block=False``; only the caching differs. The cached + chunk files can be inspected or pruned by hand. Returns --------- @@ -291,6 +653,10 @@ def request_wpto_point_data( raise TypeError( f"If specified, to_pandas must be bool type. Got: {type(to_pandas)}" ) + if not isinstance(cache_block, bool): + raise TypeError( + f"If specified, cache_block must be bool type. Got: {type(cache_block)}" + ) # Attempt to load data from cache # Construct a string representation of the function parameters @@ -334,6 +700,13 @@ def request_wpto_point_data( f"Invalid data_type: {data_type}. Must be '3-hour' or '1-hour'" ) + # Use the cached coordinate metadata for the spatial lookup so the + # coordinates are not read from HSDS or S3 on every request. Custom-path + # reads have no cached metadata and let rex build the tree from the file. + meta_cache = None if path else _cached_meta(region, data_type) + if tree is None and meta_cache is not None: + tree = _meta_tree(meta_cache) + wave_kwargs = { "tree": tree, "unscale": unscale, @@ -341,8 +714,26 @@ def request_wpto_point_data( "hsds": hsds, "years": years, } - with MultiYearWaveX(wave_path, **wave_kwargs) as rex_waves: - data, meta = _read_point_data(rex_waves, parameter, lat_lon) + + def read_point(): + rex_waves = _open_wave_resource( + MultiYearWaveX, + wave_path, + wave_kwargs, + s3_region=region, + s3_data_type=data_type, + s3_fallback=hsds and path is None, + ) + with rex_waves: + if cache_block and meta_cache is not None: + return _read_point_blocked( + rex_waves, parameter, lat_lon, meta_cache, region, data_type, years + ) + return _read_point_data(rex_waves, parameter, lat_lon, meta_cache) + + # Retry the open and read together so a transient HSDS or S3 error (for + # example a busy server during the multi-year setup) is recovered. + data, meta = _read_with_retry(read_point) if not to_pandas: data = convert_to_dataset(data) @@ -478,6 +869,12 @@ def request_wpto_directional_spectrum( f"/nrel/US_wave/virtual_buoy/{region}/{region}_virtual_buoy_{year}.h5" ) parameter = "directional_wave_spectrum" + # Use the cached coordinate metadata for the spatial lookup so coordinates + # are not read from HSDS or S3 on every request. Custom-path reads have no + # cached metadata and let rex build the tree from the file. + meta_cache = None if path else _cached_meta(region, "1-hour") + if tree is None and meta_cache is not None: + tree = _meta_tree(meta_cache) wave_kwargs = { "tree": tree, "unscale": unscale, @@ -485,7 +882,16 @@ def request_wpto_directional_spectrum( "hsds": hsds, } - with WaveX(wave_path, **wave_kwargs) as rex_waves: + rex_waves = _open_wave_resource( + WaveX, + wave_path, + wave_kwargs, + s3_region=region, + s3_data_type="1-hour", + s3_year=year, + s3_fallback=hsds and path is None, + ) + with rex_waves: # Get graphical identifier gid = rex_waves.lat_lon_gid(lat_lon) @@ -512,21 +918,11 @@ def request_wpto_directional_spectrum( for i in range(len(bins) - 1): idx = index[index_bins[i] : index_bins[i + 1]] - # Request with exponential back off wait time - sleep_time = 2 - num_retries = 4 - for _ in range(num_retries): - try: - data_array = rex_waves[parameter, bins[i] : bins[i + 1], :, :, gid] - str_error = None - except IOError as err: - str_error = str(err) - - if str_error: - sleep(sleep_time) - sleep_time *= 2 - else: - break + # Read each bin with a retry so a transient network error is + # recovered without failing the whole request. + data_array = _read_with_retry( + lambda i=i: rex_waves[parameter, bins[i] : bins[i + 1], :, :, gid] + ) ax1 = np.prod(data_array.shape[:3]) ax2 = data_array.shape[-1] if len(data_array.shape) == 4 else 1 @@ -538,8 +934,11 @@ def request_wpto_directional_spectrum( data = data_raw.to_xarray() data["time_index"] = pd.to_datetime(data.time_index) - # Get metadata - meta = _meta_for_gids(rex_waves, columns) + # Get metadata from the cached coordinates, or the resource itself for + # a custom path. + if meta_cache is None: + meta_cache = rex_waves.meta + meta = _meta_for_gids(meta_cache, columns) meta["gid"] = gid # Convert gid to integer or list of integers From b048bb8ecf8e96fb781f0b6bd99094614c2dd22c Mon Sep 17 00:00:00 2001 From: "Simms, Andrew" Date: Tue, 30 Jun 2026 11:05:18 -0600 Subject: [PATCH 12/26] Hindcast: Revert out of scope hindcast fixes --- mhkit/wave/io/hindcast/hindcast.py | 606 +++-------------------------- 1 file changed, 59 insertions(+), 547 deletions(-) diff --git a/mhkit/wave/io/hindcast/hindcast.py b/mhkit/wave/io/hindcast/hindcast.py index 99a06384b..83119a782 100644 --- a/mhkit/wave/io/hindcast/hindcast.py +++ b/mhkit/wave/io/hindcast/hindcast.py @@ -5,482 +5,21 @@ regions, request point data for various parameters, and request directional spectrum data. -Author: rpauly, aidanbharath, ssolson, simmsa -Date: 2026-06-29 +Author: rpauly, aidanbharath, ssolson +Date: 2023-09-26 """ import os import sys -import warnings from time import sleep from typing import List, Tuple, Union, Optional, Dict import pandas as pd import xarray as xr import numpy as np -import fsspec -from scipy.spatial import cKDTree from rex import MultiYearWaveX, WaveX -from rex.utilities.exceptions import ResourceRuntimeError from mhkit.utils.cache import handle_caching from mhkit.utils.type_handling import convert_to_dataset -# Public AWS S3 bucket mirroring the WPTO US Wave hindcast .h5 files served by -# HSDS. Used as a fallback when HSDS is unavailable: the same data is read -# directly from S3. https://registry.opendata.aws/wpto-pds-us-wave/ -_WAVE_S3_BUCKET = "wpto-pds-us-wave" - -# Retry count for the initial HSDS open in the hybrid read. Kept low so an -# unresponsive HSDS server fails over to the S3 fallback in seconds rather than -# waiting out h5pyd's default of 10 retries with exponential backoff. -_HSDS_OPEN_RETRIES = 2 - - -def _latest_s3_version(region_path: str) -> str: - """ - Returns the newest version directory in the wave S3 bucket that contains - the given region path. - - The bucket stores each release under a version directory (e.g. "v1.0.1") - and not every region exists in every version, so the version is resolved by - listing the bucket rather than assumed. - - Parameters - ---------- - region_path : string - Region path under a version directory, e.g. "West_Coast" or - "virtual_buoy/West_Coast" - - Returns - ------- - version : string - Newest version directory containing the region, e.g. "v1.0.1" - """ - file_system = fsspec.filesystem("s3", anon=True) - versions = sorted( - ( - entry.rsplit("/", 1)[-1] - for entry in file_system.ls(_WAVE_S3_BUCKET) - if entry.rsplit("/", 1)[-1].startswith("v") - ), - reverse=True, - ) - for version in versions: - if file_system.exists(f"{_WAVE_S3_BUCKET}/{version}/{region_path}"): - return version - raise FileNotFoundError( - f"{region_path} not found in any version of s3://{_WAVE_S3_BUCKET}" - ) - - -def _s3_wave_path(region: str, data_type: str, year: str = "*") -> str: - """ - Builds the s3:// path for a wave region, mirroring the HSDS domain layout. - - Parameters - ---------- - region : string - Region name, e.g. "West_Coast" - data_type : string - "3-hour" for the regular wave files or "1-hour" for the virtual_buoy - files - year : string or int - Year to read, or "*" for a multi-year wildcard glob. Default "*". - - Returns - ------- - s3_path : string - s3:// path (or wildcard glob) of the underlying .h5 file(s) - """ - if data_type == "1-hour": - region_path = f"virtual_buoy/{region}" - filename = f"{region}_virtual_buoy_{year}.h5" - else: - region_path = region - filename = f"{region}_wave_{year}.h5" - version = _latest_s3_version(region_path) - return f"s3://{_WAVE_S3_BUCKET}/{version}/{region_path}/{filename}" - - -def _open_wave_resource( - resource_cls, - hsds_path: str, - wave_kwargs: dict, - s3_region: str, - s3_data_type: str, - s3_year: str = "*", - s3_fallback: bool = True, -): - """ - Opens a rex wave resource on HSDS, falling back to a direct S3 read when the - HSDS open fails (e.g. the HSDS server is unavailable). - - Returns an open resource. The caller closes it, typically with a `with` - block. The HSDS and S3 reads return the same data, so the caller's read - logic does not change with the source. - - Parameters - ---------- - resource_cls : type - rex resource class to open, WaveX or MultiYearWaveX - hsds_path : string - HSDS domain path to open - wave_kwargs : dict - Keyword arguments for the resource, including hsds and (for - MultiYearWaveX) years - s3_region : string - Region name used to build the S3 path for the fallback - s3_data_type : string - "3-hour" or "1-hour", used to build the S3 path for the fallback - s3_year : string or int - Year used to build the S3 path for the fallback. Default "*". - s3_fallback : bool - Whether to fall back to S3 when the HSDS open fails. Disabled when the - caller is not reading the default HSDS files (custom path or hsds=False). - - Returns - ------- - rex_waves : WaveX or MultiYearWaveX - Open rex resource, HSDS-backed when available else S3-backed - """ - hsds_open_kwargs = dict(wave_kwargs) - if wave_kwargs.get("hsds"): - hsds_open_kwargs["hsds_kwargs"] = {"retries": _HSDS_OPEN_RETRIES} - try: - return resource_cls(hsds_path, **hsds_open_kwargs) - except (ResourceRuntimeError, IOError): - if not s3_fallback: - raise - # HSDS is unavailable. Read the same data directly from the public S3 - # bucket. rex reads s3:// paths with h5py and fsspec when hsds=False. - warnings.warn( - "HSDS is unavailable; falling back to reading the wave data " - "directly from S3, which is slower.", - UserWarning, - ) - s3_path = _s3_wave_path(s3_region, s3_data_type, s3_year) - return resource_cls(s3_path, **{**wave_kwargs, "hsds": False}) - - -def _meta_cache_path(region: str, data_type: str) -> str: - """Local parquet path for a wave dataset's cached coordinate metadata.""" - kind = "virtual_buoy" if data_type == "1-hour" else "wave" - return os.path.join(_get_cache_dir(), "meta", f"{region}_{kind}.parquet") - - -def _cached_meta(region: str, data_type: str) -> pd.DataFrame: - """ - Returns the location metadata (latitude, longitude, ...) for a wave dataset, - indexed by grid id (gid), from a local parquet cache. - - The coordinates are fixed per region across all years, so they are read from - S3 once and cached. Every later spatial query reuses the local copy instead - of re-reading hundreds of thousands of points from S3. - - Parameters - ---------- - region : string - Region name, e.g. "West_Coast" - data_type : string - "3-hour" for the regular wave files or "1-hour" for the virtual_buoy - files - - Returns - ------- - meta : pandas.DataFrame - Location metadata indexed by gid - """ - cache_path = _meta_cache_path(region, data_type) - if os.path.isfile(cache_path): - return pd.read_parquet(cache_path) - - region_path = f"virtual_buoy/{region}" if data_type == "1-hour" else region - version = _latest_s3_version(region_path) - file_system = fsspec.filesystem("s3", anon=True) - h5_files = [ - entry - for entry in file_system.ls(f"{_WAVE_S3_BUCKET}/{version}/{region_path}") - if entry.endswith(".h5") - ] - # The coordinates are identical across years, so any year's file works. - with WaveX(f"s3://{h5_files[0]}", hsds=False) as s3_waves: - meta = s3_waves.meta - os.makedirs(os.path.dirname(cache_path), exist_ok=True) - meta.to_parquet(cache_path) - return meta - - -def _meta_tree(meta: pd.DataFrame) -> cKDTree: - """Returns a cKDTree of the (latitude, longitude) coordinates in meta.""" - return cKDTree(meta[["latitude", "longitude"]].to_numpy()) - - -def _meta_for_gids(meta: pd.DataFrame, gids) -> pd.DataFrame: - """ - Returns cached meta rows for the given grid ids, indexed from 0. - - Parameters - ---------- - meta : pandas.DataFrame - Cached location metadata indexed by gid - gids : int, list, or array of int - Grid ids to read - - Returns - ------- - meta : pandas.DataFrame - Meta rows for the given grid ids, indexed from 0 - """ - gids = [int(g) for g in np.atleast_1d(gids)] - return meta.loc[gids, :].reset_index(drop=True) - - -def cache_wave_meta( - lat_lon: Optional[Union[Tuple[float, float], List[Tuple[float, float]]]] = None, - data_type: str = "3-hour", - datasets: Optional[List[Tuple[str, str]]] = None, -) -> None: - """ - Pre-builds the local coordinate metadata cache so spatial queries never read - coordinates from S3 at request time. - - This is optional. The request functions already cache each region's - coordinates lazily on first use, downloading only the region a request - falls in. Use this to warm the cache ahead of time, scoped to what is - needed: pass lat_lon to cache only the region(s) those points fall in, pass - datasets to cache an explicit list, or pass nothing to cache every region. - - Parameters - ---------- - lat_lon : tuple or list of tuples, optional - Point(s) whose region(s) to cache. The region is inferred from each - point, so only the regions to be queried are downloaded. - data_type : string, optional - "3-hour" or "1-hour", paired with lat_lon. Default "3-hour". - datasets : list of (region, data_type) tuples, optional - Explicit datasets to cache, instead of inferring from lat_lon. - """ - if datasets is None: - if lat_lon is not None: - points = lat_lon if isinstance(lat_lon[0], (list, tuple)) else [lat_lon] - regions = {region_selection(point) for point in points} - datasets = [(region, data_type) for region in regions] - else: - datasets = [ - ("West_Coast", "3-hour"), - ("Atlantic", "3-hour"), - ("Hawaii", "3-hour"), - ("West_Coast", "1-hour"), - ] - for region, dataset_type in datasets: - _cached_meta(region, dataset_type) - - -def _block_cache_path( - region: str, data_type: str, years: List[int], parameter: str, block: int -) -> str: - """Local parquet path for one cached location chunk (a block of gids).""" - kind = "virtual_buoy" if data_type == "1-hour" else "wave" - span = "_".join(str(y) for y in sorted(years)) - name = f"{region}_{kind}_{span}_{parameter}_block{block}.parquet" - return os.path.join(_get_cache_dir(), "blocks", name) - - -def _data_block_size(rex_waves: Union[WaveX, MultiYearWaveX], parameter: str) -> int: - """ - Returns the number of locations stored per chunk (the gid-dimension chunk - size) for a data variable. - - The data is chunked across locations, so reading a whole chunk costs the - same fetch as reading one location in it. This is the natural block size for - the location cache. - """ - for obj in (getattr(rex_waves, "resource", None), rex_waves): - chunks = getattr(obj, "chunks", None) - if isinstance(chunks, dict) and chunks.get(parameter): - return int(chunks[parameter][-1]) - raise ValueError(f"Could not determine the chunk size for {parameter}") - - -def _gid_block( - rex_waves: Union[WaveX, MultiYearWaveX], - parameter: str, - gid: int, - block_size: int, - n_gids: int, - region: str, - data_type: str, - years: List[int], -) -> pd.DataFrame: - """ - Returns the time series for the whole chunk of locations containing gid, - from a local cache, reading and caching it from the resource on a miss. - - One location's chunk also holds its neighbors, so reading and caching the - whole chunk costs the same fetch as a single location but serves every - location in the chunk on later queries. Columns are the gids in the block, - as strings. - """ - block = gid // block_size - cache_path = _block_cache_path(region, data_type, years, parameter, block) - if os.path.isfile(cache_path): - return pd.read_parquet(cache_path) - - start = block * block_size - stop = min(start + block_size, n_gids) - block_df = rex_waves.get_gid_df(parameter, list(range(start, stop))) - block_df.columns = block_df.columns.astype(str) - os.makedirs(os.path.dirname(cache_path), exist_ok=True) - block_df.to_parquet(cache_path) - return block_df - - -# A network read (HSDS or S3) can fail transiently, for example when the HSDS -# server is busy. Retry a failed read this many times, doubling the wait each -# time, before giving up. -_READ_ATTEMPTS = 4 -_READ_BACKOFF_SECONDS = 2 - - -def _read_with_retry(read): - """ - Calls read, retrying a transient network failure with exponential backoff. - - Parameters - ---------- - read : callable - Zero-argument function that performs the read and returns its result - - Returns - ------- - result - The value returned by read - """ - delay = _READ_BACKOFF_SECONDS - for attempt in range(_READ_ATTEMPTS): - try: - return read() - except (ResourceRuntimeError, IOError): - if attempt == _READ_ATTEMPTS - 1: - raise - sleep(delay) - delay *= 2 - - -def _read_point_blocked( - rex_waves: Union[WaveX, MultiYearWaveX], - parameter: Union[str, List[str]], - lat_lon, - meta_cache: pd.DataFrame, - region: str, - data_type: str, - years: List[int], -) -> Tuple[pd.DataFrame, pd.DataFrame]: - """ - Like _read_point_data, but reads and caches the whole location chunk around - each requested point. Nearby and repeat queries are then served from the - local cache. See the cache_block option of request_wpto_point_data. - - Parameters - ---------- - rex_waves : WaveX or MultiYearWaveX - Open rex resource - parameter : string or list of strings - Parameter(s) to read - lat_lon : tuple or list of tuples - Latitude/longitude point(s) to read - meta_cache : pandas.DataFrame - Cached location metadata indexed by gid - region : string - Region name, used in the block cache path - data_type : string - "3-hour" or "1-hour", used in the block cache path - years : list of int - Years read, used in the block cache path - - Returns - ------- - data : pandas.DataFrame - Time-series data with columns renamed to ``{parameter}_{i}`` - meta : pandas.DataFrame - Meta rows for the read columns, with a ``gid`` column - """ - parameters = parameter if isinstance(parameter, list) else [parameter] - gids = [int(g) for g in np.atleast_1d(rex_waves.lat_lon_gid(lat_lon))] - n_gids = len(meta_cache) - block_size = _data_block_size(rex_waves, parameters[0]) - time_index = rex_waves.time_index - - columns = {} - for param in parameters: - for i, gid in enumerate(gids): - block_df = _gid_block( - rex_waves, param, gid, block_size, n_gids, region, data_type, years - ) - columns[f"{param}_{i}"] = block_df[str(gid)].to_numpy() - data = pd.DataFrame(columns, index=time_index) - - meta = _meta_for_gids(meta_cache, gids) - meta["gid"] = rex_waves.lat_lon_gid(lat_lon) - return data, meta - - -def _read_point_data( - rex_waves: Union[WaveX, MultiYearWaveX], - parameter: Union[str, List[str]], - lat_lon, - meta_cache: Optional[pd.DataFrame], -) -> Tuple[pd.DataFrame, pd.DataFrame]: - """ - Read renamed point time-series data and meta (with gid) from an open rex - resource. Works against either an HSDS-backed or S3-backed resource. - - The spatial lookup uses the resource's pre-built tree and the meta comes - from the cached coordinate metadata, so no coordinates are read from the - resource. - - Parameters - ---------- - rex_waves : WaveX or MultiYearWaveX - Open rex resource - parameter : string or list of strings - Parameter(s) to read - lat_lon : tuple or list of tuples - Latitude/longitude point(s) to read - meta_cache : pandas.DataFrame - Cached location metadata indexed by gid - - Returns - ------- - data : pandas.DataFrame - Time-series data with columns renamed to ``{parameter}_{i}`` - meta : pandas.DataFrame - Meta rows for the read columns, with a ``gid`` column - """ - if isinstance(parameter, list): - data_list = [] - for param in parameter: - temp_data = rex_waves.get_lat_lon_df(param, lat_lon) - cols = temp_data.columns[:] - temp_data = temp_data.rename( - columns={col: f"{param}_{i}" for i, col in enumerate(cols)} - ) - data_list.append(temp_data) - data = pd.concat(data_list, axis=1) - else: - data = rex_waves.get_lat_lon_df(parameter, lat_lon) - cols = data.columns[:] - data = data.rename( - columns={col: f"{parameter}_{i}" for i, col in enumerate(cols)} - ) - - # A custom-path read has no cached coordinate metadata, so read the meta - # from the resource itself in that case. - if meta_cache is None: - meta_cache = rex_waves.meta - meta = _meta_for_gids(meta_cache, cols) - meta["gid"] = rex_waves.lat_lon_gid(lat_lon) - return data, meta - def region_selection(lat_lon: Union[List[float], Tuple[float, float]]) -> str: """ @@ -546,7 +85,6 @@ def request_wpto_point_data( hsds: bool = True, path: Optional[str] = None, to_pandas: bool = True, - cache_block: bool = False, ) -> Tuple[Union[pd.DataFrame, xr.Dataset], pd.DataFrame]: """ Returns data from the WPTO wave hindcast hosted on AWS at the @@ -602,20 +140,6 @@ def request_wpto_point_data( `hsds=False`. to_pandas: bool (optional) Flag to output pandas instead of xarray. Default = True. - cache_block : bool (optional) - Read and cache the whole location chunk around each requested point, - rather than only the requested location. Default = False. - - The hindcast data is chunked across locations, so fetching one location - already transfers a chunk of roughly a thousand neighboring locations. - With ``cache_block=True`` that whole chunk is kept and cached locally - (under the hindcast cache directory, one parquet file per - region/years/parameter/chunk), so later requests for any nearby location - and the same years and parameters are served from disk without another - download. This trades local disk for far fewer downloads and is useful - when analyzing many points in the same area. The returned data is - identical to ``cache_block=False``; only the caching differs. The cached - chunk files can be inspected or pruned by hand. Returns --------- @@ -653,10 +177,6 @@ def request_wpto_point_data( raise TypeError( f"If specified, to_pandas must be bool type. Got: {type(to_pandas)}" ) - if not isinstance(cache_block, bool): - raise TypeError( - f"If specified, cache_block must be bool type. Got: {type(cache_block)}" - ) # Attempt to load data from cache # Construct a string representation of the function parameters @@ -700,13 +220,6 @@ def request_wpto_point_data( f"Invalid data_type: {data_type}. Must be '3-hour' or '1-hour'" ) - # Use the cached coordinate metadata for the spatial lookup so the - # coordinates are not read from HSDS or S3 on every request. Custom-path - # reads have no cached metadata and let rex build the tree from the file. - meta_cache = None if path else _cached_meta(region, data_type) - if tree is None and meta_cache is not None: - tree = _meta_tree(meta_cache) - wave_kwargs = { "tree": tree, "unscale": unscale, @@ -714,43 +227,49 @@ def request_wpto_point_data( "hsds": hsds, "years": years, } + data_list = [] - def read_point(): - rex_waves = _open_wave_resource( - MultiYearWaveX, - wave_path, - wave_kwargs, - s3_region=region, - s3_data_type=data_type, - s3_fallback=hsds and path is None, - ) - with rex_waves: - if cache_block and meta_cache is not None: - return _read_point_blocked( - rex_waves, parameter, lat_lon, meta_cache, region, data_type, years - ) - return _read_point_data(rex_waves, parameter, lat_lon, meta_cache) - - # Retry the open and read together so a transient HSDS or S3 error (for - # example a busy server during the multi-year setup) is recovered. - data, meta = _read_with_retry(read_point) - - if not to_pandas: - data = convert_to_dataset(data) - data["time_index"] = pd.to_datetime(data.time_index) - + with MultiYearWaveX(wave_path, **wave_kwargs) as rex_waves: if isinstance(parameter, list): - n_loc = 1 if isinstance(lat_lon[0], float) else len(lat_lon) - param_coords = [f"{param}_{n_loc - 1}" for param in parameter] - data.coords["parameter"] = xr.DataArray(param_coords, dims="parameter") + for param in parameter: + temp_data = rex_waves.get_lat_lon_df(param, lat_lon) + gid = rex_waves.lat_lon_gid(lat_lon) + cols = temp_data.columns[:] + for i, col in zip(range(len(cols)), cols): + temp = f"{param}_{i}" + temp_data = temp_data.rename(columns={col: temp}) + + data_list.append(temp_data) + data = pd.concat(data_list, axis=1) + + else: + data = rex_waves.get_lat_lon_df(parameter, lat_lon) + cols = data.columns[:] + + for i, col in zip(range(len(cols)), cols): + temp = f"{parameter}_{i}" + data = data.rename(columns={col: temp}) + + meta = rex_waves.meta.loc[cols, :] + meta = meta.reset_index(drop=True) + gid = rex_waves.lat_lon_gid(lat_lon) + meta["gid"] = gid - data.coords["year"] = xr.DataArray(years, dims="year") + if not to_pandas: + data = convert_to_dataset(data) + data["time_index"] = pd.to_datetime(data.time_index) - meta_ds = meta.to_xarray() - data = xr.merge([data, meta_ds]) + if isinstance(parameter, list): + param_coords = [f"{param}_{i}" for param in parameter] + data.coords["parameter"] = xr.DataArray(param_coords, dims="parameter") - # Remove the 'index' coordinate - data = data.drop_vars("index") + data.coords["year"] = xr.DataArray(years, dims="year") + + meta_ds = meta.to_xarray() + data = xr.merge([data, meta_ds]) + + # Remove the 'index' coordinate + data = data.drop_vars("index") # save_to_cache(hash_params, data, meta) handle_caching( @@ -869,12 +388,6 @@ def request_wpto_directional_spectrum( f"/nrel/US_wave/virtual_buoy/{region}/{region}_virtual_buoy_{year}.h5" ) parameter = "directional_wave_spectrum" - # Use the cached coordinate metadata for the spatial lookup so coordinates - # are not read from HSDS or S3 on every request. Custom-path reads have no - # cached metadata and let rex build the tree from the file. - meta_cache = None if path else _cached_meta(region, "1-hour") - if tree is None and meta_cache is not None: - tree = _meta_tree(meta_cache) wave_kwargs = { "tree": tree, "unscale": unscale, @@ -882,16 +395,7 @@ def request_wpto_directional_spectrum( "hsds": hsds, } - rex_waves = _open_wave_resource( - WaveX, - wave_path, - wave_kwargs, - s3_region=region, - s3_data_type="1-hour", - s3_year=year, - s3_fallback=hsds and path is None, - ) - with rex_waves: + with WaveX(wave_path, **wave_kwargs) as rex_waves: # Get graphical identifier gid = rex_waves.lat_lon_gid(lat_lon) @@ -918,11 +422,21 @@ def request_wpto_directional_spectrum( for i in range(len(bins) - 1): idx = index[index_bins[i] : index_bins[i + 1]] - # Read each bin with a retry so a transient network error is - # recovered without failing the whole request. - data_array = _read_with_retry( - lambda i=i: rex_waves[parameter, bins[i] : bins[i + 1], :, :, gid] - ) + # Request with exponential back off wait time + sleep_time = 2 + num_retries = 4 + for _ in range(num_retries): + try: + data_array = rex_waves[parameter, bins[i] : bins[i + 1], :, :, gid] + str_error = None + except OSError as err: + str_error = str(err) + + if str_error: + sleep(sleep_time) + sleep_time *= 2 + else: + break ax1 = np.prod(data_array.shape[:3]) ax2 = data_array.shape[-1] if len(data_array.shape) == 4 else 1 @@ -934,11 +448,9 @@ def request_wpto_directional_spectrum( data = data_raw.to_xarray() data["time_index"] = pd.to_datetime(data.time_index) - # Get metadata from the cached coordinates, or the resource itself for - # a custom path. - if meta_cache is None: - meta_cache = rex_waves.meta - meta = _meta_for_gids(meta_cache, columns) + # Get metadata + meta = rex_waves.meta.loc[columns, :] + meta = meta.reset_index(drop=True) meta["gid"] = gid # Convert gid to integer or list of integers From bf926a4b70020486af07c4657016e1f45efc9624 Mon Sep 17 00:00:00 2001 From: "Simms, Andrew" Date: Tue, 30 Jun 2026 11:26:02 -0600 Subject: [PATCH 13/26] Actions: Disable hindcast tests due to non functional api --- .github/workflows/main.yml | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 7ca82b44d..1f307cbea 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -64,11 +64,15 @@ jobs: - id: hindcast-logic run: | - if [[ "${{ github.event.pull_request.base.ref }}" == "main" || "${{ steps.changes.outputs.wave_io_hindcast }}" == "true" ]]; then - echo "should-run-hindcast=true" >> "$GITHUB_OUTPUT" - else - echo "should-run-hindcast=false" >> "$GITHUB_OUTPUT" - fi + # TODO(restore-hindcast): NLR HSDS endpoint is down for v1.1, so hindcast + # jobs are disabled. Restore the logic below when the service is back online. + # See https://github.com/MHKiT-Software/MHKiT-Python/issues/450 + echo "should-run-hindcast=false" >> "$GITHUB_OUTPUT" + # if [[ "${{ github.event.pull_request.base.ref }}" == "main" || "${{ steps.changes.outputs.wave_io_hindcast }}" == "true" ]]; then + # echo "should-run-hindcast=true" >> "$GITHUB_OUTPUT" + # else + # echo "should-run-hindcast=false" >> "$GITHUB_OUTPUT" + # fi prepare-nonhindcast-cache: runs-on: ubuntu-latest From d2d4e339695c317dcb638190ff66fcb456f01a4d Mon Sep 17 00:00:00 2001 From: "Simms, Andrew" Date: Tue, 30 Jun 2026 12:27:17 -0600 Subject: [PATCH 14/26] Hindcast: Add generic hindcast exception decorator --- mhkit/wave/io/hindcast/hindcast_exceptions.py | 92 +++++++++++++++++++ 1 file changed, 92 insertions(+) create mode 100644 mhkit/wave/io/hindcast/hindcast_exceptions.py diff --git a/mhkit/wave/io/hindcast/hindcast_exceptions.py b/mhkit/wave/io/hindcast/hindcast_exceptions.py new file mode 100644 index 000000000..d54dbdcf1 --- /dev/null +++ b/mhkit/wave/io/hindcast/hindcast_exceptions.py @@ -0,0 +1,92 @@ +""" +Exceptions for the hindcast API and a decorator that turns low-level request +failures into clear, actionable errors. + +A 502 or 503 response means the NLR HSDS service is down, which is tracked in +the MHKiT issue below. Any other failure asks the user to open a new issue and +paste the stack trace. To handle a new failure mode, add an exception class and +a branch to :func:`hindcast_guard`. +""" + +import functools +import traceback +from typing import Callable + +# Existing MHKiT issue tracking the known NLR HSDS service outage. +HINDCAST_UNAVALIABLE_ISSUE_URL = ( + "https://github.com/MHKiT-Software/MHKiT-Python/issues/450" +) +# Where to open a new MHKiT issue for any other hindcast request failure. +NEW_ISSUE_URL = "https://github.com/MHKiT-Software/MHKiT-Python/issues/new" +# HTTP status codes that mean the HSDS service is down (tracked in #450). +_SERVICE_DOWN_CODES = ("502", "503") + + +class HindcastApiError(RuntimeError): + """Base class for hindcast API errors, also raised for unexpected failures.""" + + +class HindcastApiUnavailableError(HindcastApiError): + """Raised when the hindcast API is down (HSDS 502 or 503).""" + + +def _is_service_down(err: Exception) -> bool: + """Return True when the error looks like an HSDS 502 or 503 response.""" + text = str(err) + return any(code in text for code in _SERVICE_DOWN_CODES) + + +def _unavailable_message(trace: str) -> str: + return ( + "Could not retrieve the hindcast data. The NLR HSDS endpoint " + "(developer.nlr.gov) returned a 502 or 503 error and is likely down.\n\n" + "This is a known outage tracked here:\n" + f" {HINDCAST_UNAVALIABLE_ISSUE_URL}\n\n" + f"Stack trace:\n{trace}" + ) + + +def _unexpected_message(trace: str) -> str: + return ( + "An unexpected error occurred while retrieving the hindcast data.\n\n" + "Please open an issue and paste the stack trace below:\n" + f" {NEW_ISSUE_URL}\n\n" + f"Stack trace:\n{trace}" + ) + + +def hindcast_guard(func: Callable) -> Callable: + """ + Convert a failed hindcast API request into a clear HindcastApiError. + + The wrapped function runs normally, so cached results and a recovered + service still work. Input validation errors (TypeError, ValueError) and + existing HindcastApiError subclasses pass through unchanged. An HSDS 502 + or 503 raises HindcastApiUnavailableError pointing to the tracking issue. + Any other failure raises HindcastApiError asking the user to open an + issue. Both include the stack trace for the user to copy into the issue. + + Parameters + ---------- + func : callable + Hindcast data-request function to wrap. + + Returns + ------- + callable + The wrapped function. + """ + + @functools.wraps(func) + def wrapper(*args, **kwargs): + try: + return func(*args, **kwargs) + except (TypeError, ValueError, HindcastApiError): + raise + except Exception as err: + trace = traceback.format_exc() + if _is_service_down(err): + raise HindcastApiUnavailableError(_unavailable_message(trace)) from err + raise HindcastApiError(_unexpected_message(trace)) from err + + return wrapper From 686851a5086ef993320781640a1f594c8fdf174b Mon Sep 17 00:00:00 2001 From: "Simms, Andrew" Date: Tue, 30 Jun 2026 12:27:57 -0600 Subject: [PATCH 15/26] Hindcast: Wire up hindcast exception guard decorator --- mhkit/wave/io/hindcast/hindcast.py | 4 ++++ mhkit/wave/io/hindcast/wind_toolkit.py | 5 ++++- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/mhkit/wave/io/hindcast/hindcast.py b/mhkit/wave/io/hindcast/hindcast.py index 83119a782..5be667683 100644 --- a/mhkit/wave/io/hindcast/hindcast.py +++ b/mhkit/wave/io/hindcast/hindcast.py @@ -20,6 +20,8 @@ from mhkit.utils.cache import handle_caching from mhkit.utils.type_handling import convert_to_dataset +from mhkit.wave.io.hindcast.hindcast_exceptions import hindcast_guard + def region_selection(lat_lon: Union[List[float], Tuple[float, float]]) -> str: """ @@ -74,6 +76,7 @@ def region_search( # pylint: disable=too-many-locals # pylint: disable=too-many-branches # pylint: disable=too-many-statements +@hindcast_guard def request_wpto_point_data( data_type: str, parameter: Union[str, List[str]], @@ -283,6 +286,7 @@ def request_wpto_point_data( # pylint: disable=too-many-branches # pylint: disable=too-many-statements +@hindcast_guard def request_wpto_directional_spectrum( lat_lon: Union[Tuple[float, float], List[Tuple[float, float]]], year: str, diff --git a/mhkit/wave/io/hindcast/wind_toolkit.py b/mhkit/wave/io/hindcast/wind_toolkit.py index 596bf58e8..cfc55b8a1 100644 --- a/mhkit/wave/io/hindcast/wind_toolkit.py +++ b/mhkit/wave/io/hindcast/wind_toolkit.py @@ -30,6 +30,8 @@ from mhkit.utils.cache import handle_caching from mhkit.utils.type_handling import convert_to_dataset +from mhkit.wave.io.hindcast.hindcast_exceptions import hindcast_guard + def region_selection(lat_lon: Tuple[float, float], preferred_region: str = "") -> str: """ @@ -108,6 +110,7 @@ def region_search(x: str) -> bool: return region[0] +@hindcast_guard def get_region_data(region: str) -> Tuple[np.ndarray, np.ndarray]: """ Retrieves the latitude and longitude data points for the specified region @@ -278,6 +281,7 @@ def elevation_to_string( # pylint: disable=too-many-statements # pylint: disable=too-many-positional-arguments # pylint: disable=duplicate-code +@hindcast_guard def request_wtk_point_data( time_interval: str, parameter: Union[str, List[str]], @@ -369,7 +373,6 @@ def request_wtk_point_data( meta: DataFrame Location metadata for the requested data location """ - if not isinstance(parameter, (str, list)): raise TypeError("parameter must be of type string or list") if not isinstance(lat_lon, (list, tuple)): From a56651625a97b9e6a89a7983fe5a955bfcd0a129 Mon Sep 17 00:00:00 2001 From: "Simms, Andrew" Date: Tue, 30 Jun 2026 12:28:30 -0600 Subject: [PATCH 16/26] Tests: Temporarily disable hindcast tests --- mhkit/tests/wave/io/hindcast/test_hindcast.py | 7 ++++++- mhkit/tests/wave/io/hindcast/test_wind_toolkit.py | 9 +++++++-- 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/mhkit/tests/wave/io/hindcast/test_hindcast.py b/mhkit/tests/wave/io/hindcast/test_hindcast.py index 456ea05b7..44cbc1aed 100644 --- a/mhkit/tests/wave/io/hindcast/test_hindcast.py +++ b/mhkit/tests/wave/io/hindcast/test_hindcast.py @@ -26,7 +26,6 @@ import unittest from os.path import abspath, dirname, join, normpath from pandas.testing import assert_frame_equal -import xarray.testing as xrt import pandas as pd import mhkit.wave as wave import xarray as xr @@ -37,6 +36,12 @@ ) +# TODO(restore-hindcast): 6/30/2026, NLR HSDS endpoint is non functional +# https://github.com/MHKiT-Software/MHKiT-Python/issues/450 +@unittest.skip( + "TODO(restore-hindcast): NLR HSDS endpoint unavailable. See " + "https://github.com/MHKiT-Software/MHKiT-Python/issues/450" +) class TestWPTOhindcast(unittest.TestCase): """ A test call designed to check the WPTO hindcast retrival diff --git a/mhkit/tests/wave/io/hindcast/test_wind_toolkit.py b/mhkit/tests/wave/io/hindcast/test_wind_toolkit.py index 0113a0cab..ad9dd7141 100644 --- a/mhkit/tests/wave/io/hindcast/test_wind_toolkit.py +++ b/mhkit/tests/wave/io/hindcast/test_wind_toolkit.py @@ -1,4 +1,4 @@ -from os.path import abspath, dirname, join, isfile, normpath, relpath +from os.path import abspath, dirname, join, normpath from pandas.testing import assert_frame_equal import matplotlib.pylab as plt import mhkit.wave.io.hindcast.wind_toolkit as wtk @@ -6,7 +6,6 @@ import unittest import pytest - testdir = dirname(abspath(__file__)) datadir = normpath( join( @@ -24,6 +23,12 @@ ) +# TODO(restore-hindcast): 6/30/2026, NLR HSDS endpoint is non functional +# https://github.com/MHKiT-Software/MHKiT-Python/issues/450 +@unittest.skip( + "TODO(restore-hindcast): NLR HSDS endpoint unavailable. See " + "https://github.com/MHKiT-Software/MHKiT-Python/issues/450" +) class TestWINDToolkit(unittest.TestCase): @classmethod def setUpClass(self): From a0e4caf5533b35d762ea8316ae5b60edadc60641 Mon Sep 17 00:00:00 2001 From: "Simms, Andrew" Date: Tue, 30 Jun 2026 12:47:26 -0600 Subject: [PATCH 17/26] NOAA: Add retries to api calls that can intermittently fail --- mhkit/tidal/io/noaa.py | 48 ++++++++++++++++++++++++++++-------------- 1 file changed, 32 insertions(+), 16 deletions(-) diff --git a/mhkit/tidal/io/noaa.py b/mhkit/tidal/io/noaa.py index 8622e110e..5a042416b 100644 --- a/mhkit/tidal/io/noaa.py +++ b/mhkit/tidal/io/noaa.py @@ -9,6 +9,7 @@ """ import os +import time import xml.etree.ElementTree as ET import datetime import json @@ -402,16 +403,30 @@ def _build_data_url( return f"https://tidesandcurrents.noaa.gov/api/datagetter?{api_query}" -def _make_request(data_url: str, proxy: dict) -> requests.Response: +def _make_request( + data_url: str, + proxy: dict, + max_retries: int = 5, + retry_delay: float = 2.0, +) -> requests.Response: """ Makes an HTTP request to the specified data URL using optional proxy settings. + The NOAA Tides and Currents API intermittently fails with transient errors + (e.g. 504 Gateway Timeout). To make requests resilient to these temporary + outages, failed requests are retried with exponential backoff. + Parameters ---------- data_url : str The URL to request data from. proxy : dict Proxy settings for the request. + max_retries : int, optional + Maximum number of attempts before giving up. + retry_delay : float, optional + Base delay in seconds between retries. The delay grows exponentially + (retry_delay * 2 ** attempt) with each successive retry. Returns ------- @@ -421,22 +436,23 @@ def _make_request(data_url: str, proxy: dict) -> requests.Response: Raises ------ requests.exceptions.RequestException - If an error occurs during the request. + If an error occurs during the request and all retries are exhausted. """ - try: - response = requests.get(url=data_url, proxies=proxy, timeout=60) - response.raise_for_status() - if "error" in response.content.decode(): - raise requests.exceptions.RequestException(response.content.decode()) - except requests.exceptions.HTTPError as http_err: - print(f"HTTP error occurred: {http_err}") - print(f"Error message: {response.content.decode()}\n") - raise - except requests.exceptions.RequestException as req_err: - print(f"Requests error occurred: {req_err}") - print(f"Error message: {response.content.decode()}\n") - raise - return response + for attempt in range(max_retries): + try: + response = requests.get(url=data_url, proxies=proxy, timeout=60) + response.raise_for_status() + if "error" in response.content.decode(): + raise requests.exceptions.RequestException(response.content.decode()) + return response + except requests.exceptions.RequestException as err: + print(f"Request error occurred: {err}") + # Last attempt failed, give up and re-raise. + if attempt == max_retries - 1: + raise + delay = retry_delay * (2**attempt) + print(f"Retrying in {delay:.1f}s...\n") + time.sleep(delay) def _concatenate_data_frames(data_frames: list[pd.DataFrame]) -> pd.DataFrame: From 888682ee9985a04ee5f8e9b5347f6d65270899dc Mon Sep 17 00:00:00 2001 From: "Simms, Andrew" Date: Tue, 30 Jun 2026 13:12:32 -0600 Subject: [PATCH 18/26] NOAA: fix lints issues in _make_request --- mhkit/tidal/io/noaa.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/mhkit/tidal/io/noaa.py b/mhkit/tidal/io/noaa.py index 5a042416b..daff0e35c 100644 --- a/mhkit/tidal/io/noaa.py +++ b/mhkit/tidal/io/noaa.py @@ -438,6 +438,7 @@ def _make_request( requests.exceptions.RequestException If an error occurs during the request and all retries are exhausted. """ + last_err = None for attempt in range(max_retries): try: response = requests.get(url=data_url, proxies=proxy, timeout=60) @@ -446,13 +447,14 @@ def _make_request( raise requests.exceptions.RequestException(response.content.decode()) return response except requests.exceptions.RequestException as err: + last_err = err print(f"Request error occurred: {err}") - # Last attempt failed, give up and re-raise. - if attempt == max_retries - 1: - raise - delay = retry_delay * (2**attempt) - print(f"Retrying in {delay:.1f}s...\n") - time.sleep(delay) + if attempt < max_retries - 1: + delay = retry_delay * (2**attempt) + print(f"Retrying in {delay:.1f}s...\n") + time.sleep(delay) + # All attempts failed; re-raise the last error. + raise last_err def _concatenate_data_frames(data_frames: list[pd.DataFrame]) -> pd.DataFrame: From f71a2bb4e46764681afe406be13e383a734ce0e4 Mon Sep 17 00:00:00 2001 From: "Simms, Andrew" Date: Tue, 30 Jun 2026 13:13:46 -0600 Subject: [PATCH 19/26] Tests: Disable NOAA tidal api calls because endpoint is returning 504 https://github.com/MHKiT-Software/MHKiT-Python/issues/451 --- mhkit/tests/tidal/test_io.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/mhkit/tests/tidal/test_io.py b/mhkit/tests/tidal/test_io.py index be34cd444..72e5159db 100644 --- a/mhkit/tests/tidal/test_io.py +++ b/mhkit/tests/tidal/test_io.py @@ -68,6 +68,11 @@ def test_load_noaa_data_xarray(self): self.assertEqual(len(data["index"]), 18890) self.assertEqual(data.attrs["id"], "s08010") + # TODO(noaa_api_failures): NOAA Tides and Currents API intermittently + # returns 504 Gateway Timeout, causing this test to fail in CI. Re-enable + # once the upstream API is reliable. See + # https://github.com/MHKiT-Software/MHKiT-Python/issues/451 + @unittest.skip("NOAA API intermittently unavailable; see issue #451") def test_request_noaa_data_basic(self): """ Test the request_noaa_data function with basic input parameters @@ -89,6 +94,11 @@ def test_request_noaa_data_basic(self): self.assertEqual(data.shape, (183, 3)) self.assertEqual(metadata["id"], "s08010") + # TODO(noaa_api_failures): NOAA Tides and Currents API intermittently + # returns 504 Gateway Timeout, causing this test to fail in CI. Re-enable + # once the upstream API is reliable. See + # https://github.com/MHKiT-Software/MHKiT-Python/issues/451 + @unittest.skip("NOAA API intermittently unavailable; see issue #451") def test_request_noaa_data_basic_xarray(self): """ Test the request_noaa_data function with basic input parameters @@ -116,6 +126,11 @@ def test_request_noaa_data_basic_xarray(self): self.assertEqual(len(data["index"]), 183) self.assertEqual(data.attrs["id"], "s08010") + # TODO(noaa_api_failures): NOAA Tides and Currents API intermittently + # returns 504 Gateway Timeout, causing this test to fail in CI. Re-enable + # once the upstream API is reliable. See + # https://github.com/MHKiT-Software/MHKiT-Python/issues/451 + @unittest.skip("NOAA API intermittently unavailable; see issue #451") def test_request_noaa_data_write_json(self): """ Test the request_noaa_data function with the write_json parameter From a5ea29303bd756f300c47c4c9a2200017ad435eb Mon Sep 17 00:00:00 2001 From: "Simms, Andrew" Date: Tue, 30 Jun 2026 13:19:10 -0600 Subject: [PATCH 20/26] Dev: Lint --- mhkit/tests/tidal/test_io.py | 1 - 1 file changed, 1 deletion(-) diff --git a/mhkit/tests/tidal/test_io.py b/mhkit/tests/tidal/test_io.py index 72e5159db..d614b5e16 100644 --- a/mhkit/tests/tidal/test_io.py +++ b/mhkit/tests/tidal/test_io.py @@ -21,7 +21,6 @@ import numpy as np import mhkit.tidal as tidal - testdir = dirname(abspath(__file__)) plotdir = join(testdir, "plots") isdir = os.path.isdir(plotdir) From 02b1bad1cbc0b0e71ef023365c7daa46390cedb8 Mon Sep 17 00:00:00 2001 From: "Simms, Andrew" Date: Tue, 30 Jun 2026 13:33:45 -0600 Subject: [PATCH 21/26] Revert: NOAA request changes: The reverted changes don't fix any underlying NOAA api issues and add unnecessary complexity. --- mhkit/tidal/io/noaa.py | 50 ++++++++++++++---------------------------- 1 file changed, 16 insertions(+), 34 deletions(-) diff --git a/mhkit/tidal/io/noaa.py b/mhkit/tidal/io/noaa.py index daff0e35c..8622e110e 100644 --- a/mhkit/tidal/io/noaa.py +++ b/mhkit/tidal/io/noaa.py @@ -9,7 +9,6 @@ """ import os -import time import xml.etree.ElementTree as ET import datetime import json @@ -403,30 +402,16 @@ def _build_data_url( return f"https://tidesandcurrents.noaa.gov/api/datagetter?{api_query}" -def _make_request( - data_url: str, - proxy: dict, - max_retries: int = 5, - retry_delay: float = 2.0, -) -> requests.Response: +def _make_request(data_url: str, proxy: dict) -> requests.Response: """ Makes an HTTP request to the specified data URL using optional proxy settings. - The NOAA Tides and Currents API intermittently fails with transient errors - (e.g. 504 Gateway Timeout). To make requests resilient to these temporary - outages, failed requests are retried with exponential backoff. - Parameters ---------- data_url : str The URL to request data from. proxy : dict Proxy settings for the request. - max_retries : int, optional - Maximum number of attempts before giving up. - retry_delay : float, optional - Base delay in seconds between retries. The delay grows exponentially - (retry_delay * 2 ** attempt) with each successive retry. Returns ------- @@ -436,25 +421,22 @@ def _make_request( Raises ------ requests.exceptions.RequestException - If an error occurs during the request and all retries are exhausted. + If an error occurs during the request. """ - last_err = None - for attempt in range(max_retries): - try: - response = requests.get(url=data_url, proxies=proxy, timeout=60) - response.raise_for_status() - if "error" in response.content.decode(): - raise requests.exceptions.RequestException(response.content.decode()) - return response - except requests.exceptions.RequestException as err: - last_err = err - print(f"Request error occurred: {err}") - if attempt < max_retries - 1: - delay = retry_delay * (2**attempt) - print(f"Retrying in {delay:.1f}s...\n") - time.sleep(delay) - # All attempts failed; re-raise the last error. - raise last_err + try: + response = requests.get(url=data_url, proxies=proxy, timeout=60) + response.raise_for_status() + if "error" in response.content.decode(): + raise requests.exceptions.RequestException(response.content.decode()) + except requests.exceptions.HTTPError as http_err: + print(f"HTTP error occurred: {http_err}") + print(f"Error message: {response.content.decode()}\n") + raise + except requests.exceptions.RequestException as req_err: + print(f"Requests error occurred: {req_err}") + print(f"Error message: {response.content.decode()}\n") + raise + return response def _concatenate_data_frames(data_frames: list[pd.DataFrame]) -> pd.DataFrame: From 8f0afc412b36acc80719460da7bb1ac93f7ad3de Mon Sep 17 00:00:00 2001 From: "Simms, Andrew" Date: Tue, 30 Jun 2026 13:37:08 -0600 Subject: [PATCH 22/26] Actions: Run tests in this PR on all operating systems as a sanity check --- .github/workflows/main.yml | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 1f307cbea..a69f9d365 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -36,11 +36,14 @@ jobs: steps: - id: set-matrix run: | - if [[ "${{ github.event_name }}" == "pull_request" && "${{ github.base_ref }}" == "develop" ]]; then - echo "matrix_os=[ \"ubuntu-latest\"]" >> $GITHUB_OUTPUT - else - echo "matrix_os=[\"windows-latest\", \"ubuntu-latest\", \"macos-latest\"]" >> $GITHUB_OUTPUT - fi + # TEMP(all-os): Force all operating systems for this PR. Restore the + # block below to go back to the develop-only ubuntu shortcut. + echo "matrix_os=[\"windows-latest\", \"ubuntu-latest\", \"macos-latest\"]" >> $GITHUB_OUTPUT + # if [[ "${{ github.event_name }}" == "pull_request" && "${{ github.base_ref }}" == "develop" ]]; then + # echo "matrix_os=[ \"ubuntu-latest\"]" >> $GITHUB_OUTPUT + # else + # echo "matrix_os=[\"windows-latest\", \"ubuntu-latest\", \"macos-latest\"]" >> $GITHUB_OUTPUT + # fi # This job decides if the hindcast test suite should run. From 7bcbf370b1f592d906f276c913587431be53de54 Mon Sep 17 00:00:00 2001 From: "Simms, Andrew" Date: Tue, 30 Jun 2026 14:42:32 -0600 Subject: [PATCH 23/26] Actions: Only upload to coveralls on pr's into main and pushes to main Historically mhkit ran coveralls on every PR and push. Lately coveralls has been unreliable and the coveralls step is failing to upload causing the actions to fail. Coveralls is used for badges, and the badge used in MHKiT is from the main branch. This change reduces the number of uploads to coveralls to the bare minimum to determine if the coverage will upload before a merge into main, and when code is merged into main --- .github/workflows/main.yml | 36 ++++++++++++++++++++++++++++++++++-- 1 file changed, 34 insertions(+), 2 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index a69f9d365..f99b581cf 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -22,6 +22,7 @@ env: CONDA_ENVIRONMENT_NAME: mhkit-dev-env # Coveralls: Set to false to ignore upload failures during outages. # Check status: https://status.coveralls.io/ + # Note this only affects the code coverage percentage badge in the README, nothing else COVERALLS_FAIL_ON_ERROR: true jobs: @@ -46,6 +47,31 @@ jobs: # fi + # `coveralls-gate` controls whether code coverage data is uploaded to coveralls. + # The README code coverage badge is main-only. Because this is a simple metric we don't need to + # calculate this for every code push, just the ones that can change the badge coverage percentage + # + # Specific code changes that require coveralls are: + # * a PR targeting main + # * a push to main + # Coveralls badge: + # + coveralls-gate: + runs-on: ubuntu-latest + outputs: + should-upload: ${{ steps.gate.outputs.should-upload }} + steps: + - id: gate + run: | + should_upload=false + if [[ "${{ github.event_name }}" == "pull_request" && "${{ github.base_ref }}" == "main" ]]; then + should_upload=true + fi + if [[ "${{ github.event_name }}" == "push" && "${{ github.ref }}" == "refs/heads/main" ]]; then + should_upload=true + fi + echo "should-upload=$should_upload" >> "$GITHUB_OUTPUT" + # This job decides if the hindcast test suite should run. check-changes: runs-on: ubuntu-latest @@ -208,7 +234,7 @@ jobs: conda-forge-build: name: conda-forge-${{ matrix.os }}/${{ matrix.python-version }} - needs: [set-os, prepare-nonhindcast-cache] + needs: [set-os, prepare-nonhindcast-cache, coveralls-gate] runs-on: ${{ matrix.os }} strategy: fail-fast: false @@ -256,6 +282,7 @@ jobs: coverage lcov - name: Upload coverage data to coveralls.io + if: needs.coveralls-gate.outputs.should-upload == 'true' uses: coverallsapp/github-action@v2 with: github-token: ${{ secrets.GITHUB_TOKEN }} @@ -266,7 +293,7 @@ jobs: pip-build: name: pip-${{ matrix.os }}/${{ matrix.python-version }} - needs: [set-os, prepare-nonhindcast-cache] + needs: [set-os, prepare-nonhindcast-cache, coveralls-gate] runs-on: ${{ matrix.os }} strategy: fail-fast: false @@ -317,6 +344,7 @@ jobs: coverage lcov - name: Upload coverage data to coveralls.io + if: needs.coveralls-gate.outputs.should-upload == 'true' uses: coverallsapp/github-action@v2 with: github-token: ${{ secrets.GITHUB_TOKEN }} @@ -333,6 +361,7 @@ jobs: prepare-wave-hindcast-cache, prepare-wind-hindcast-cache, set-os, + coveralls-gate, ] if: (needs.check-changes.outputs.should-run-hindcast == 'true') @@ -395,6 +424,7 @@ jobs: coverage lcov - name: Upload coverage data to coveralls.io + if: needs.coveralls-gate.outputs.should-upload == 'true' uses: coverallsapp/github-action@v2 with: github-token: ${{ secrets.GITHUB_TOKEN }} @@ -643,9 +673,11 @@ jobs: conda-forge-build, pip-build, hindcast-calls, + coveralls-gate, ] if: | always() && + needs.coveralls-gate.outputs.should-upload == 'true' && ( ( needs.conda-forge-build.result == 'success' && From b866a5586dd1499be30fa9c5df0b657e207fa768 Mon Sep 17 00:00:00 2001 From: "Simms, Andrew" Date: Tue, 30 Jun 2026 14:59:10 -0600 Subject: [PATCH 24/26] Actions: Add single action for coveralls uploads --- .github/actions/upload-coverage/action.yml | 32 ++++++++++++++++++++++ .github/workflows/main.yml | 12 ++------ 2 files changed, 35 insertions(+), 9 deletions(-) create mode 100644 .github/actions/upload-coverage/action.yml diff --git a/.github/actions/upload-coverage/action.yml b/.github/actions/upload-coverage/action.yml new file mode 100644 index 000000000..b7083d0b9 --- /dev/null +++ b/.github/actions/upload-coverage/action.yml @@ -0,0 +1,32 @@ +name: Upload coverage to Coveralls +description: Install the macOS reporter if needed and upload lcov coverage to Coveralls. + +inputs: + github-token: + description: Token used to authenticate with Coveralls. + required: true + flag-name: + description: Unique flag identifying this coverage job. + required: true + fail-on-error: + description: Whether an upload failure should fail the job. + required: false + default: "true" + +runs: + using: composite + steps: + # Fix for coveralls on homebrew 6+: https://github.com/coverallsapp/github-action/pull/265 + - name: Install coveralls reporter (macOS) + if: runner.os == 'macOS' + shell: bash + run: brew install coverallsapp/coveralls/coveralls --quiet + + - name: Upload coverage data to coveralls.io + uses: coverallsapp/github-action@v2 + with: + github-token: ${{ inputs.github-token }} + flag-name: ${{ inputs.flag-name }} + parallel: true + path-to-lcov: ./coverage.lcov + fail-on-error: ${{ inputs.fail-on-error }} diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index f99b581cf..c122c4b06 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -283,12 +283,10 @@ jobs: - name: Upload coverage data to coveralls.io if: needs.coveralls-gate.outputs.should-upload == 'true' - uses: coverallsapp/github-action@v2 + uses: ./.github/actions/upload-coverage with: github-token: ${{ secrets.GITHUB_TOKEN }} flag-name: conda-forge-${{ runner.os }}-py${{ matrix.python-version }} - parallel: true - path-to-lcov: ./coverage.lcov fail-on-error: ${{ env.COVERALLS_FAIL_ON_ERROR }} pip-build: @@ -345,12 +343,10 @@ jobs: - name: Upload coverage data to coveralls.io if: needs.coveralls-gate.outputs.should-upload == 'true' - uses: coverallsapp/github-action@v2 + uses: ./.github/actions/upload-coverage with: github-token: ${{ secrets.GITHUB_TOKEN }} flag-name: pip-${{ runner.os }}-py${{ matrix.python-version }} - parallel: true - path-to-lcov: ./coverage.lcov fail-on-error: ${{ env.COVERALLS_FAIL_ON_ERROR }} hindcast-calls: @@ -425,12 +421,10 @@ jobs: - name: Upload coverage data to coveralls.io if: needs.coveralls-gate.outputs.should-upload == 'true' - uses: coverallsapp/github-action@v2 + uses: ./.github/actions/upload-coverage with: github-token: ${{ secrets.GITHUB_TOKEN }} flag-name: hindcast-${{ runner.os }}-py${{ matrix.python-version }} - parallel: true - path-to-lcov: ./coverage.lcov fail-on-error: ${{ env.COVERALLS_FAIL_ON_ERROR }} test-wheel-packaging: From c230467a80c4796fcf8031c8ce4e5d0a8f30253d Mon Sep 17 00:00:00 2001 From: "Simms, Andrew" Date: Tue, 30 Jun 2026 15:01:42 -0600 Subject: [PATCH 25/26] Actions: Verify that coveralls still works on all operating systems --- .github/workflows/main.yml | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index c122c4b06..4d9066988 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -63,14 +63,17 @@ jobs: steps: - id: gate run: | - should_upload=false - if [[ "${{ github.event_name }}" == "pull_request" && "${{ github.base_ref }}" == "main" ]]; then - should_upload=true - fi - if [[ "${{ github.event_name }}" == "push" && "${{ github.ref }}" == "refs/heads/main" ]]; then - should_upload=true - fi - echo "should-upload=$should_upload" >> "$GITHUB_OUTPUT" + # TEMP(test-coveralls): Force coveralls upload on every run to test the + # macOS reporter fix. Restore the gate logic below when done. + echo "should-upload=true" >> "$GITHUB_OUTPUT" + # should_upload=false + # if [[ "${{ github.event_name }}" == "pull_request" && "${{ github.base_ref }}" == "main" ]]; then + # should_upload=true + # fi + # if [[ "${{ github.event_name }}" == "push" && "${{ github.ref }}" == "refs/heads/main" ]]; then + # should_upload=true + # fi + # echo "should-upload=$should_upload" >> "$GITHUB_OUTPUT" # This job decides if the hindcast test suite should run. check-changes: From afc771fa918fc89b2478994240604c371050cb78 Mon Sep 17 00:00:00 2001 From: "Simms, Andrew" Date: Tue, 30 Jun 2026 17:11:26 -0600 Subject: [PATCH 26/26] Actions: Migrate macos-latest to macos-26: Actions warning: ``` The macos-latest label will migrate to macOS 26 beginning June 15, 2026. For more information see https://github.com/actions/runner-images/issues/14167 ``` --- .github/workflows/main.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 4d9066988..1a4ebede6 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -39,11 +39,11 @@ jobs: run: | # TEMP(all-os): Force all operating systems for this PR. Restore the # block below to go back to the develop-only ubuntu shortcut. - echo "matrix_os=[\"windows-latest\", \"ubuntu-latest\", \"macos-latest\"]" >> $GITHUB_OUTPUT + echo "matrix_os=[\"windows-latest\", \"ubuntu-latest\", \"macos-26\"]" >> $GITHUB_OUTPUT # if [[ "${{ github.event_name }}" == "pull_request" && "${{ github.base_ref }}" == "develop" ]]; then # echo "matrix_os=[ \"ubuntu-latest\"]" >> $GITHUB_OUTPUT # else - # echo "matrix_os=[\"windows-latest\", \"ubuntu-latest\", \"macos-latest\"]" >> $GITHUB_OUTPUT + # echo "matrix_os=[\"windows-latest\", \"ubuntu-latest\", \"macos-26\"]" >> $GITHUB_OUTPUT # fi