From ec7a26ca8f8d1c92b574a032848bd63b5a820bde Mon Sep 17 00:00:00 2001 From: Eric Novotny Date: Wed, 3 Jun 2026 12:40:19 -0700 Subject: [PATCH 1/4] fixes to shef exporter --- pyproject.toml | 4 + shef/exporters/abstract_exporter.py | 2 - shef/exporters/cda_exporter.py | 66 ++-- shef/loaders/cda_loader.py | 99 +++--- shef/loaders/dss_loader.py | 13 +- shef/shef_parser.py | 294 ++++++++++++------ tests/test_cda_exporter_empty_series.py | 127 ++++++++ .../test_cda_loader_make_export_transforms.py | 135 ++++++++ tests/test_export_time_window.py | 150 +++++++++ 9 files changed, 724 insertions(+), 166 deletions(-) create mode 100644 tests/test_cda_exporter_empty_series.py create mode 100644 tests/test_cda_loader_make_export_transforms.py create mode 100644 tests/test_export_time_window.py diff --git a/pyproject.toml b/pyproject.toml index 5e6b213..08e7333 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -17,12 +17,16 @@ authors = ["Hydrologic Engineering Center"] [tool.poetry.dependencies] python = "^3.9" click = "^8.1" +cwms-python = { version = "^1.0.8", optional = true } +hecdss = { version = "^0.1.29", optional = true } sphinx = { version = "^7.0", optional = true } sphinx_rtd_theme = { version = "^2.0", optional = true } sphinx-design = { version = "^0.5", optional = true } [tool.poetry.extras] docs = ["sphinx", "sphinx_rtd_theme", "sphinx-design"] +cda = ["cwms-python"] +dss = ["hecdss"] [tool.poetry.group.dev.dependencies] black = "^24.2.0" diff --git a/shef/exporters/abstract_exporter.py b/shef/exporters/abstract_exporter.py index 7cb0a8a..d2747f1 100644 --- a/shef/exporters/abstract_exporter.py +++ b/shef/exporters/abstract_exporter.py @@ -5,8 +5,6 @@ from io import BufferedRandom, StringIO from typing import Optional, TextIO, Union -import cwms # type: ignore - from shef.loaders import abstract_loader, shared diff --git a/shef/exporters/cda_exporter.py b/shef/exporters/cda_exporter.py index 72e5c8b..6438ecc 100644 --- a/shef/exporters/cda_exporter.py +++ b/shef/exporters/cda_exporter.py @@ -35,7 +35,6 @@ def __init__(self, cda_url: str, office: str): self._office = office self._cda_loader = loaders.cda_loader.CdaLoader(self.logger, sys.stdout) self._cda_loader.set_options(f"[{cda_url}][][{office}]") - self._cda_loader.make_export_transforms() def export(self, timeseries_or_group: str) -> None: """ @@ -45,35 +44,50 @@ def export(self, timeseries_or_group: str) -> None: timeseries_or_group (str): If a time series ID, export that time series; If a time series group ID, export each time series in that group """ if len(timeseries_or_group.split(".")) == 6: + self._cda_loader.make_export_transforms() timeseries_ids = [timeseries_or_group] - elif timeseries_or_group in self._cda_loader._export_groups: + else: + self._cda_loader.make_export_transforms(group_id=timeseries_or_group) + if timeseries_or_group not in self._cda_loader._export_groups: + raise shared.LoaderException( + f"Time series group [{timeseries_or_group}] not found under the SHEF Export category for office [{self._office}]" + ) timeseries_ids = self._cda_loader._export_groups[timeseries_or_group][ "timeseries" ] total_value_count: int = 0 - first = True - data = StringIO() - data.write("[") + ts_payloads: list[Any] = [] for tsid in timeseries_ids: - unit = self._cda_loader._transforms[tsid].units - ts = cwms.get_timeseries( - ts_id=tsid, - office_id=self._cda_loader._office_id, - unit=unit, - begin=self._start_time, - end=self._end_time, - ) - value_count = len(ts.json["values"]) - if value_count > 0: - if not first: - data.write(",") - data.write(json.dumps(ts.json)) - total_value_count += value_count - first = False - data.write("]") - to_unload = data.getvalue() - data.close() - if (total_value_count) > 0: + try: + unit = self._cda_loader._transforms[tsid].units + ts = cwms.get_timeseries( + ts_id=tsid, + office_id=self._cda_loader._office_id, + unit=unit, + begin=self._start_time, + end=self._end_time, + ) + except Exception as e: + self.logger.warning( + f"Skipping time series [{tsid}]: error fetching from CDA: {e}" + ) + continue + ts_json = ts.json if ts is not None else None + if not isinstance(ts_json, dict): + self.logger.warning( + f"Skipping time series [{tsid}]: CDA response is not a JSON object" + ) + continue + values = ts_json.get("values") or [] + if not values: + self.logger.info( + f"Skipping time series [{tsid}]: no values in window {self._start_time} to {self._end_time}" + ) + continue + ts_payloads.append(ts_json) + total_value_count += len(values) + to_unload = json.dumps(ts_payloads) + if total_value_count > 0: try: old_output = self._cda_loader._output self._cda_loader._output = self._output @@ -90,6 +104,7 @@ def get_groups(self) -> dict[str, str]: Returns: dict[str, str]: A dictionary of time series group descriptions keyed by time series group IDs """ + self._cda_loader.make_export_transforms() return { group: self._cda_loader._export_groups[group]["description"] for group in self._cda_loader._export_groups @@ -105,6 +120,7 @@ def get_time_series(self, group: str) -> list[str]: Returns: list[str]: The assigned time series IDs """ + self._cda_loader.make_export_transforms(group_id=group) return [ts for ts in self._cda_loader._export_groups[group]["timeseries"]] def get_unit(self, tsid: str) -> Optional[str]: @@ -117,6 +133,8 @@ def get_unit(self, tsid: str) -> Optional[str]: Returns: Optional[str]: The unit as specified in the time series alias """ + if tsid not in self._cda_loader._transforms: + self._cda_loader.make_export_transforms() return self._cda_loader._transforms[tsid].units diff --git a/shef/loaders/cda_loader.py b/shef/loaders/cda_loader.py index 501f44e..6e1df67 100644 --- a/shef/loaders/cda_loader.py +++ b/shef/loaders/cda_loader.py @@ -106,6 +106,8 @@ def __init__( self._value_error_count: int = 0 self._write_tasks: list[Coroutine[Any, Any, Any]] = [] self._configured_pe_codes = set() + self._loaded_export_group_ids: set[str] = set() + self._loaded_all_export_groups: bool = False def make_shef_transform(self, crit: dict[str, Any]) -> ShefTransform: """ @@ -462,55 +464,72 @@ def set_input(self, input_object: Union[StringIO, TextIO, str]) -> None: f"Expected TextIOWrapper or str object, got [{input_object.__class__.__name__}]" ) - def make_export_transforms(self) -> None: + def make_export_transforms(self, group_id: Optional[str] = None) -> None: if not self._office_id: raise shared.LoaderException( f"Cannot unload without office specified, use options [api_root][api_key][office]" ) - if not self._transforms: - tsids_used: dict[str, list[str]] = {} - group_list = cwms.get_timeseries_groups( - office_id=self._office_id, - include_assigned=True, - timeseries_category_like="SHEF Export", - timeseries_group_like="^.+$", - category_office_id="CWMS", - ).json - for shef_group in group_list: - group_id = shef_group["id"] - if "description" not in shef_group: - shef_group["description"] = "" - self._export_groups[group_id] = { - "description": shef_group["description"], - "timeseries": [], - } - try: - for time_series in shef_group["assigned-time-series"]: - transform = self.make_shef_transform(time_series) - transform_key = ( - f"{transform.location}.{transform.parameter_code}" - ) - self._transforms[transform_key] = transform - if transform.timeseries_id in tsids_used: - if self._logger: - self._logger.warning( - f"Tranform for time seires {transform.timeseries_id} specified in group(s) " - f"{','.join(tsids_used[transform.timeseries_id])} is/are overriden by transform specified in group {group_id}" - ) - self._export_groups[group_id]["timeseries"].append( - transform.timeseries_id - ) - tsids_used.setdefault(transform.timeseries_id, []).append( - group_id - ) - self._transforms[transform.timeseries_id] = ( - transform # to be able to retrieve by time series ID + if self._loaded_all_export_groups: + return + if group_id is None: + group_filter = "^.+$" + elif group_id in self._loaded_export_group_ids: + return + else: + group_filter = f"^{re.escape(group_id)}$" + tsids_used: dict[str, list[str]] = {} + group_list = cwms.get_timeseries_groups( + office_id=self._office_id, + include_assigned=True, + timeseries_category_like="SHEF Export", + timeseries_group_like=group_filter, + category_office_id="CWMS", + group_office_id=self._office_id, + ).json + for shef_group in group_list: + shef_group_id = shef_group["id"] + if "description" not in shef_group: + shef_group["description"] = "" + self._export_groups[shef_group_id] = { + "description": shef_group["description"], + "timeseries": [], + } + for time_series in shef_group["assigned-time-series"]: + if not time_series.get("alias-id"): + if self._logger: + self._logger.warning( + f"Skipping time series {time_series.get('timeseries-id')} in group {shef_group_id}: missing or empty alias-id" ) + continue + try: + transform = self.make_shef_transform(time_series) + transform_key = ( + f"{transform.location}.{transform.parameter_code}" + ) + self._transforms[transform_key] = transform + if transform.timeseries_id in tsids_used: + if self._logger: + self._logger.warning( + f"Tranform for time seires {transform.timeseries_id} specified in group(s) " + f"{','.join(tsids_used[transform.timeseries_id])} is/are overriden by transform specified in group {shef_group_id}" + ) + self._export_groups[shef_group_id]["timeseries"].append( + transform.timeseries_id + ) + tsids_used.setdefault(transform.timeseries_id, []).append( + shef_group_id + ) + self._transforms[transform.timeseries_id] = ( + transform # to be able to retrieve by time series ID + ) except Exception as e: if self._logger: self._logger.warning( - f"{str(e)} occurred while processing SHEF criteria for {time_series['timeseries-id']}" + f"{str(e)} occurred while processing SHEF criteria for {time_series.get('timeseries-id')}" ) + self._loaded_export_group_ids.add(shef_group_id) + if group_id is None: + self._loaded_all_export_groups = True def unload(self) -> None: """ diff --git a/shef/loaders/dss_loader.py b/shef/loaders/dss_loader.py index bd4ee7a..9c95816 100644 --- a/shef/loaders/dss_loader.py +++ b/shef/loaders/dss_loader.py @@ -6,7 +6,14 @@ from logging import Logger from typing import Any, Optional, TextIO, Union, cast -from hecdss import HecDss, IrregularTimeSeries, RegularTimeSeries # type: ignore +try: + from hecdss import HecDss, IrregularTimeSeries, RegularTimeSeries # type: ignore + HECDSS_AVAILABLE = True +except ImportError: + HECDSS_AVAILABLE = False + HecDss = None + IrregularTimeSeries = None + RegularTimeSeries = None from shef.constants import PE_CONVERSIONS from shef.loaders import abstract_loader, shared @@ -68,6 +75,8 @@ def set_options(self, options_str: Optional[str]) -> None: """ Set the sensor and parameter file names """ + if not HECDSS_AVAILABLE: + raise ImportError("The 'hecdss' library is required but not installed. Please install it to use DSS functionality.") if not options_str: raise shared.LoaderException( f"Empty options on {self.loader_name}.set_options()" @@ -617,6 +626,8 @@ def load_time_series(self) -> None: """ Store the time series to HEC-DSS file """ + if not HECDSS_AVAILABLE: + raise ImportError("The 'hecdss' library is required but not installed. Please install it to use DSS functionality.") if self._shef_value and self._time_series: if self._dss_file is None: self._dss_file = HecDss(self._dss_file_name) diff --git a/shef/shef_parser.py b/shef/shef_parser.py index be97a15..8a15834 100644 --- a/shef/shef_parser.py +++ b/shef/shef_parser.py @@ -115,7 +115,8 @@ def configure_logging( ) return log_target else: - logging.basicConfig(stream=log_target, format=fmt, datefmt=datefmt, level=level) + logging.basicConfig(stream=log_target, format=fmt, + datefmt=datefmt, level=level) return log_target.name @@ -898,7 +899,8 @@ def __init__( dt = obstime if relativetime: dt = ( - dt.astimezone("Z" if parser.shefit_times else ShefParser.UTC) + dt.astimezone( + "Z" if parser.shefit_times else ShefParser.UTC) + relativetime ) self._createtime = parser.get_creation_time(dt, createtime_str) @@ -1011,7 +1013,8 @@ def get_output_record( if obst.hour < 7: obst += timedelta( days=-1 - ) # dont use "obst -1 timedelta(days=1)" - it causes mypy to complain + # dont use "obst -1 timedelta(days=1)" - it causes mypy to complain + ) obst = obst.replace(hour=7, minute=0, second=0) elif shift: if isinstance(shift, MonthsDelta): @@ -1027,7 +1030,8 @@ def get_output_record( obst += timedelta(days=days) # 2 - convert to UTC, but keep timezone for later use zi = obst.tzinfo - obst = obst.astimezone("Z" if parser.shefit_times else ShefParser.UTC) + obst = obst.astimezone( + "Z" if parser.shefit_times else ShefParser.UTC) # 3 - adjust to shift hour, minutes, and seconds if shift is not None and isinstance(shift, timedelta): # DON'T use shift.seconds!!! If shift is negative it will be incorrect as shown below. @@ -1049,9 +1053,11 @@ def get_output_record( else: creat = self.createtime if creat: - creat = creat.astimezone("Z" if parser.shefit_times else ShefParser.UTC) + creat = creat.astimezone( + "Z" if parser.shefit_times else ShefParser.UTC) if units_override == "SI": - value = parser.get_english_unit_value(value, self._parameter_code) + value = parser.get_english_unit_value( + value, self._parameter_code) return ShefParser.OutputRecord( parser, @@ -1122,13 +1128,15 @@ def __init__( f"Location [{location}] must be 3 to 8 characters in length" ) if not parameter_code: - raise ShefParser.OutputException("Parameter code must not be empty") + raise ShefParser.OutputException( + "Parameter code must not be empty") if len(parameter_code) != 7: raise ShefParser.OutputException( f"Parameter code [{parameter_code}] must be 7 characters in length" ) if not obstime: - raise ShefParser.OutputException("Observed time must not be empty") + raise ShefParser.OutputException( + "Observed time must not be empty") self._parser = parser self._location = location @@ -1146,7 +1154,8 @@ def __init__( self._creation_time: Union[None, ShefParser.DateTime] = None if create_time and isinstance(create_time, str): - self._creation_time = parser.get_creation_time(obstime, create_time) + self._creation_time = parser.get_creation_time( + obstime, create_time) elif isinstance(create_time, ShefParser.DateTime): self._creation_time = create_time self._observation_time = self._observation_time.astimezone( @@ -1234,7 +1243,8 @@ def format(self, fmt: str) -> str: buf.write(f"{self.time_series_code:2d}") buf.write(" ") buf.write( - self.message_source.ljust(8) if self.message_source else " " + self.message_source.ljust( + 8) if self.message_source else " " ) buf.write(" ") if self.comment: @@ -1276,7 +1286,8 @@ def format(self, fmt: str) -> str: buf.write(f"{self.revised:2d}") buf.write(" ") buf.write( - self.message_source.ljust(8) if self.message_source else " " + self.message_source.ljust( + 8) if self.message_source else " " ) buf.write(f"{self.time_series_code}") if self.comment: @@ -1284,7 +1295,8 @@ def format(self, fmt: str) -> str: rec = buf.getvalue() buf.close() else: - raise ShefParser.OutputException(f'Invalid output format: "[{fmt}]"') + raise ShefParser.OutputException( + f'Invalid output format: "[{fmt}]"') return rec @property @@ -1609,7 +1621,7 @@ def __init__( # 2 = date-time # 6 = time zone # 1 23 4 - r"^\.[AEB]R?\s+(\w{3,8})\s+((\d{2})?(\d{2})?\d{4})" # 5 6 + r"^\.[AEB]R?\s+(\w{3,8})\s+((\d{2})?(\d{2})?\d{4})" # 5 6 r"(\s+([NAECMPYLHB][DS]?|[JZ]))?\s+?", re.I | re.M, ) @@ -1651,7 +1663,8 @@ def __init__( self._create_time_pattern = re.compile(r"DC\d+", re.I) self._unit_system_pattern = re.compile(r"DU[ES]", re.I) self._data_qualifier_pattern = re.compile(r"DQ.", re.I) - self._duration_code_pattern = re.compile(r"(DV[SNHDMY]\d{1,2}|DVZ)", re.I) + self._duration_code_pattern = re.compile( + r"(DV[SNHDMY]\d{1,2}|DVZ)", re.I) self._parameter_code_pattern = re.compile( r"^[A-CE-IL-NP-Y][A-Z](([A-Z]([A-Z0-9]{2})?[A-Z]{1,2})?)?", re.I ) @@ -1670,7 +1683,8 @@ def __init__( self._replacement_strip_pattern = re.compile( "^[" + chr(0) + chr(9) + "]+|[" + chr(0) + chr(9) + "]+$" ) - self._replacement_split_pattern = re.compile("[" + chr(0) + chr(9) + "]") + self._replacement_split_pattern = re.compile( + "[" + chr(0) + chr(9) + "]") if self._shefparm_pathname: self.read_shefparm(self._shefparm_pathname) @@ -1910,7 +1924,8 @@ def set_send_code(self, line: str) -> None: """ Update Send codes from SHEFPARM line """ - key, value = line[0:2], (line[3:10], len(line) > 12 and line[12] == "1") + key, value = line[0:2], (line[3:10], len(line) + > 12 and line[12] == "1") if key not in self._send_codes: self.info( f"{self._shefparm_pathname}: Adding non-standard send code [{key}] with parmameter [{value[0]}] and use-prev-0700 = [{value[1]}]" @@ -1929,7 +1944,8 @@ def set_qualifier_code(self, line: str) -> None: """ key = line[0] if len(key) != 1 or not key.isalpha() or key != key.upper() or key in ("IO"): - self.critical(f"{self._shefparm_pathname}: Invalid ata qualifier [{key}]") + self.critical( + f"{self._shefparm_pathname}: Invalid ata qualifier [{key}]") if key not in self._qualifier_codes: self.info( f"{self._shefparm_pathname}: Adding non-standard data qualifier code [{key}]" @@ -2246,7 +2262,8 @@ def set_output(self, output_object: Union[TextIO, str], append: bool) -> None: if self._output: self.close_output() elif isinstance(output_object, str): - self._output = open(output_object, "a" if append else "w", encoding="utf-8") + self._output = open( + output_object, "a" if append else "w", encoding="utf-8") self._output_name = output_object else: # IO typing is wonky -- see https://github.com/python/typeshed/issues/6077 @@ -2336,7 +2353,8 @@ def get_next_processed_line(self) -> Optional[OutputRecord]: y = int(line[10:14]) m, d, h, n, s = list( map( - int, [line[i : i + 2] for i in (15, 18, 21, 24, 27)] + int, [line[i: i + 2] + for i in (15, 18, 21, 24, 27)] ) ) obstime = ShefParser.DateTime( @@ -2346,7 +2364,7 @@ def get_next_processed_line(self) -> Optional[OutputRecord]: parse_portion = "creation time" _y = line[31:35].strip() _m, _d, _h, _n, _s = [ - line[i : i + 2].strip() for i in (36, 39, 42, 45, 48) + line[i: i + 2].strip() for i in (36, 39, 42, 45, 48) ] if all([_y, _m, _d, _h, _n, _s]): y, m, d, h, n, s = list( @@ -2354,7 +2372,8 @@ def get_next_processed_line(self) -> Optional[OutputRecord]: ) if all([y, m, d, h, n, s]): create_time = ShefParser.DateTime( - y, m, d, h, n, s, tzinfo=ZoneInfo("UTC") + y, m, d, h, n, s, tzinfo=ZoneInfo( + "UTC") ) else: assert not any([y, m, d, h, n, s]) @@ -2408,7 +2427,8 @@ def get_next_processed_line(self) -> Optional[OutputRecord]: y = int(line[8:12]) m, d, h, n, s = list( map( - int, [line[i : i + 2] for i in (12, 14, 16, 18, 20)] + int, [line[i: i + 2] + for i in (12, 14, 16, 18, 20)] ) ) obstime = ShefParser.DateTime( @@ -2418,7 +2438,7 @@ def get_next_processed_line(self) -> Optional[OutputRecord]: parse_portion = "creation time" _y = line[23:27].strip() _m, _d, _h, _n, _s = [ - line[i : i + 2].strip() for i in (27, 29, 31, 33, 35) + line[i: i + 2].strip() for i in (27, 29, 31, 33, 35) ] if all([_y, _m, _d, _h, _n, _s]): y, m, d, h, n, s = list( @@ -2426,7 +2446,8 @@ def get_next_processed_line(self) -> Optional[OutputRecord]: ) if all([y, m, d, h, n, s]): create_time = ShefParser.DateTime( - y, m, d, h, n, s, tzinfo=ZoneInfo("UTC") + y, m, d, h, n, s, tzinfo=ZoneInfo( + "UTC") ) else: assert not any([y, m, d, h, n, s]) @@ -2437,8 +2458,10 @@ def get_next_processed_line(self) -> Optional[OutputRecord]: pe_code = line[38:40] ts_code = line[41:43] extremum_code = line[43] - probability_code = self._probability_ids[float(line[56:62])] - duration_code = self._duration_ids[int(line[62:67])] + probability_code = self._probability_ids[float( + line[56:62])] + duration_code = self._duration_ids[int( + line[62:67])] parameter_code = f"{pe_code}{duration_code}{ts_code}{extremum_code}{probability_code}" assert parameter_code.isascii() @@ -2469,7 +2492,8 @@ def get_next_processed_line(self) -> Optional[OutputRecord]: f"Error parsing {parse_portion} for pre-processed input: {line}" ) else: - self.error(f"Unrecognized line for pre-processed input: {line}") + self.error( + f"Unrecognized line for pre-processed input: {line}") else: output_rec = ShefParser.OutputRecord( self, @@ -2539,7 +2563,8 @@ def get_next_message(self) -> str: self._line_number += 1 self.debug(f"Removed line from input queue [{line}]") message_line = ( - self.remove_comment_fields(line).rstrip("=").rstrip("&").rstrip("=") + self.remove_comment_fields(line).rstrip( + "=").rstrip("&").rstrip("=") ) if not message_line: continue @@ -2583,16 +2608,19 @@ def get_next_message(self) -> str: continue self._line_number -= 1 self._input_lines.appendleft(line) - self.debug(f"Restored line to input queue [{line}]") + self.debug( + f"Restored line to input queue [{line}]") message_lines.pop() raw_message_lines.pop() self._message_location = ( - self._line_number - len(raw_message_lines) + 1 + self._line_number - + len(raw_message_lines) + 1 ) self._message = "\n".join( list(message_lines) + [".END"] ) - self._raw_message = "\n".join(raw_message_lines) + self._raw_message = "\n".join( + raw_message_lines) self.error( '.B message not finished before next message - missing ".END" appended' ) @@ -2611,7 +2639,8 @@ def get_next_message(self) -> str: else: self._line_number -= 1 self._input_lines.appendleft(line) - self.debug(f"Restored line to input queue [{line}]") + self.debug( + f"Restored line to input queue [{line}]") message_type = "" break if message_lines and not message_type: @@ -2721,9 +2750,11 @@ def parse_header_date( y -= 1 elif month_diff == 6 and cd > d: y += 1 - dateval = ShefParser.DateTime(y, m, d, 0, 0, 0, tzinfo=time_zone) + dateval = ShefParser.DateTime( + y, m, d, 0, 0, 0, tzinfo=time_zone) else: - dateval = ShefParser.DateTime(y, m, d, 0, 0, 0, tzinfo=time_zone) + dateval = ShefParser.DateTime( + y, m, d, 0, 0, 0, tzinfo=time_zone) prev_year = dateval - MonthsDelta(12) cur_diff = dateval - cur_date prev_diff = cur_date - prev_year @@ -2740,7 +2771,8 @@ def parse_header_date( ) dateval = prev_year else: - dateval = ShefParser.DateTime(y, m, d, 0, 0, 0, tzinfo=time_zone) + dateval = ShefParser.DateTime( + y, m, d, 0, 0, 0, tzinfo=time_zone) return dateval, century_specified except: raise ShefParser.ParseException(f"Bad date string: [{datestr}]") @@ -3022,7 +3054,8 @@ def get_observation_time( if century_specified: y = bt.year - bt.year % 100 + int(v[0:2]) else: - y = cur_time.year - cur_time.year % 100 + int(v[0:2]) + y = cur_time.year - \ + cur_time.year % 100 + int(v[0:2]) if y - cur_time.year > 10: y -= 100 else: @@ -3216,7 +3249,8 @@ def get_observation_time( v = subtoken[3:] val = int(v) if abs(val) > 99: - raise ShefParser.ParseException("Invalid relative time value") + raise ShefParser.ParseException( + "Invalid relative time value") if subtoken[2] == "S": if dot_b: relativetime = timedelta(seconds=val) @@ -3259,7 +3293,8 @@ def get_observation_time( except ShefParser.Exc: raise except: - raise ShefParser.ParseException(f"Bad observation time: [{subtoken}]") + raise ShefParser.ParseException( + f"Bad observation time: [{subtoken}]") return obstime, relativetime, century_specified def get_creation_time( @@ -3270,7 +3305,8 @@ def get_creation_time( """ if not token: return None - curtime = ShefParser.DateTime.now("Z" if self.shefit_times else ShefParser.UTC) + curtime = ShefParser.DateTime.now( + "Z" if self.shefit_times else ShefParser.UTC) threshold = ShefParser.DateTime( obstime.year, obstime.month, obstime.day, 0, 0, 0, tzinfo=obstime.tzinfo ) + MonthsDelta(120) @@ -3285,11 +3321,14 @@ def get_creation_time( int(s[8:10]), int(s[10:12]), ) - dt = ShefParser.DateTime(y, m, d, h, n, 0, tzinfo=obstime.tzinfo) + dt = ShefParser.DateTime( + y, m, d, h, n, 0, tzinfo=obstime.tzinfo) elif length == 10: # yymmddhhnn y = curtime.year - curtime.year % 100 + int(s[0:2]) - m, d, h, n = int(s[2:4]), int(s[4:6]), int(s[6:8]), int(s[8:10]) - dt = ShefParser.DateTime(y, m, d, h, n, 0, tzinfo=obstime.tzinfo) + m, d, h, n = int(s[2:4]), int( + s[4:6]), int(s[6:8]), int(s[8:10]) + dt = ShefParser.DateTime( + y, m, d, h, n, 0, tzinfo=obstime.tzinfo) while dt > threshold: dt2 = dt - MonthsDelta(1200) if not isinstance(dt2, ShefParser.DateTime): @@ -3305,7 +3344,8 @@ def get_creation_time( int(s[4:6]), int(s[6:8]), ) - dt = ShefParser.DateTime(y, m, d, h, n, 0, tzinfo=obstime.tzinfo) + dt = ShefParser.DateTime( + y, m, d, h, n, 0, tzinfo=obstime.tzinfo) while dt > threshold: dt2 = dt - MonthsDelta(1200) if not isinstance(dt2, ShefParser.DateTime): @@ -3314,8 +3354,10 @@ def get_creation_time( ) dt = dt2 elif length == 6: # mmddhh - y, m, d, h, n = obstime.year, int(s[0:2]), int(s[2:4]), int(s[4:6]), 0 - dt = ShefParser.DateTime(y, m, d, h, n, 0, tzinfo=obstime.tzinfo) + y, m, d, h, n = obstime.year, int( + s[0:2]), int(s[2:4]), int(s[4:6]), 0 + dt = ShefParser.DateTime( + y, m, d, h, n, 0, tzinfo=obstime.tzinfo) while dt > threshold: dt2 = dt - MonthsDelta(1200) if not isinstance(dt2, ShefParser.DateTime): @@ -3326,7 +3368,8 @@ def get_creation_time( elif length == 4: # mmdd hour = 12 if obstime.tzinfo in ("Z", ShefParser.UTC) else 24 y, m, d, h, n = obstime.year, int(s[0:2]), int(s[2:4]), hour, 0 - dt = ShefParser.DateTime(y, m, d, h, n, 0, tzinfo=obstime.tzinfo) + dt = ShefParser.DateTime( + y, m, d, h, n, 0, tzinfo=obstime.tzinfo) while dt > threshold: dt2 = dt - MonthsDelta(1200) if not isinstance(dt2, ShefParser.DateTime): @@ -3335,7 +3378,8 @@ def get_creation_time( ) dt = dt2 else: - raise ShefParser.ParseException(f"Bad creation time: [{token}]") + raise ShefParser.ParseException( + f"Bad creation time: [{token}]") return dt except: raise ShefParser.ParseException(f"Bad creation time: [{token}]") @@ -3402,7 +3446,8 @@ def parse_value_token( # 4 = missing valule # 5 = value qualifier matched_groups = "".join( - map(lambda x: "T" if bool(x) else "F", [m.group(i) for i in (2, 3, 4)]) + map(lambda x: "T" if bool(x) else "F", + [m.group(i) for i in (2, 3, 4)]) ) qualifier = None if matched_groups == "TFF": @@ -3513,7 +3558,8 @@ def retokenize(tokens: list[Any]) -> list[Any]: ): if i < len(tokens) - 1: if ( - self._parameter_code_pattern.match(tokens[i + 1][0]) + self._parameter_code_pattern.match( + tokens[i + 1][0]) and tokens[i + 1][0][0] != "D" ): new_tokens.append(tokens[i] + [chr(0)]) @@ -3648,7 +3694,8 @@ def retokenize(tokens: list[Any]) -> list[Any]: # -------------------------------------------------# default_qualifier = token[2].upper() if default_qualifier not in self._qualifier_codes: - self.error(f"Bad data qualifier: [{default_qualifier}]") + self.error( + f"Bad data qualifier: [{default_qualifier}]") return [] if self._reject_problematic else outrecs elif self._duration_code_pattern.match(token): # ----------------------------------------------------------------# @@ -3677,7 +3724,8 @@ def retokenize(tokens: list[Any]) -> list[Any]: # ------------# code = tokens[i][0].upper() if len(code) < 2: - self.error(f"Invalid PE code: [{code[:min(2, len(code))]}]") + self.error( + f"Invalid PE code: [{code[:min(2, len(code))]}]") return [] elif ( code not in self._send_codes @@ -3688,7 +3736,8 @@ def retokenize(tokens: list[Any]) -> list[Any]: f"Unknown PE code: [{code[:min(2, len(code))]}], value(s) will be untransformed" ) try: - parameter_code, use_prev_7am = self.get_parameter_code(code) + parameter_code, use_prev_7am = self.get_parameter_code( + code) orig_parameter_code = code except ShefParser.Exc as spe: self.error(str(spe)) @@ -3789,7 +3838,8 @@ def retokenize(tokens: list[Any]) -> list[Any]: comment = tokens[i][2] if comment: if comment[0] not in "'\"": - self.error(f"Invalid retained comment [{tokens[i][2]}]") + self.error( + f"Invalid retained comment [{tokens[i][2]}]") comment = None if parameter_code[3] == "F" and not createtime_str: @@ -4055,7 +4105,8 @@ def retokenize(tokens: list[Any]) -> list[Any]: else: duration_id = self._duration_ids[duration_code] except KeyError: - self.error(f"No valid duration code for time interval [{token}]") + self.error( + f"No valid duration code for time interval [{token}]") return [] if self._reject_problematic else outrecs parameter_code = ( f"{parameter_code[:2]}{duration_id}{parameter_code[3:]}" @@ -4069,7 +4120,8 @@ def retokenize(tokens: list[Any]) -> list[Any]: return [] if self._reject_problematic else outrecs code = token.upper() if len(code) < 2: - self.error(f"Invalid PE code: [{code[:min(2, len(code))]}]") + self.error( + f"Invalid PE code: [{code[:min(2, len(code))]}]") return [] if self._reject_problematic else outrecs elif ( code not in self._send_codes @@ -4117,14 +4169,16 @@ def retokenize(tokens: list[Any]) -> list[Any]: comment = tokens[i][1] if comment: if comment[0] not in "'\"": - self.error(f"Invalid retained comment [{tokens[i][2]}]") + self.error( + f"Invalid retained comment [{tokens[i][2]}]") comment = None elif not token: # ------------------------------------# # missing value if in list of values # # ------------------------------------# if not (parameter_code and interval): - raise ShefParser.ParseException("Null field in data definition") + raise ShefParser.ParseException( + "Null field in data definition") obstime += interval time_series_code = 2 elif token[0] in "\"'": @@ -4148,7 +4202,8 @@ def retokenize(tokens: list[Any]) -> list[Any]: "Value encountered before parameter code" ) if not interval: - raise ShefParser.ParseException("Value encountered before interval") + raise ShefParser.ParseException( + "Value encountered before interval") if parameter_code[3] == "F" and not createtime_str: self.warning( @@ -4202,7 +4257,8 @@ def retokenize(tokens: list[Any]) -> list[Any]: else: new_tokens.append(temp_tokens[j]) for i in range(len(new_tokens)): - new_tokens[i] = ShefParser.unhide_quoted_whitespace(new_tokens[i]) + new_tokens[i] = ShefParser.unhide_quoted_whitespace( + new_tokens[i]) return new_tokens # ------------------------------------------------------------------------------------# @@ -4214,12 +4270,12 @@ def retokenize(tokens: list[Any]) -> list[Any]: lines = m.group(0).strip().split("\n") lines[0] = lines[0].strip() for i in range(1, len(lines)): - lines[i] = lines[i][len(lines[i].split()[0]) :].strip() + lines[i] = lines[i][len(lines[i].split()[0]):].strip() if lines[i] and lines[0][-1] != "/" and lines[i][0] != "/": lines[0] += "/" lines[0] += lines[i] header = lines[0] - body = "\n".join(message[m.end() :].strip().split("\n")[:-1]).strip() + body = "\n".join(message[m.end():].strip().split("\n")[:-1]).strip() # ------------------------------------# # parse the header positional fields # # ------------------------------------# @@ -4282,9 +4338,10 @@ def retokenize(tokens: list[Any]) -> list[Any]: # --------------------------------------# # process the parameter control fields # # --------------------------------------# - param_str = header[m.end() :].strip() + param_str = header[m.end():].strip() while self._multiple_obs_time_pattern.search(param_str): - param_str = self._multiple_obs_time_pattern.sub(r"\1@\6\7", param_str) + param_str = self._multiple_obs_time_pattern.sub( + r"\1@\6\7", param_str) param_tokens = list( map(lambda s: s.strip().strip("@"), param_str.strip("/").split("/")) ) @@ -4300,7 +4357,8 @@ def retokenize(tokens: list[Any]) -> list[Any]: while True: m = self._obs_time_pattern2.search(token[pos:]) if not m: - self.error(f"Unexpected data string item: [{token[pos:]}]") + self.error( + f"Unexpected data string item: [{token[pos:]}]") return [] try: obstime, relativetime, century_specified = ( @@ -4373,7 +4431,8 @@ def retokenize(tokens: list[Any]) -> list[Any]: self.warning( f"Unknown PE code: [{code[:min(2, len(code))]}], value(s) will be untransformed" ) - parameter_code, use_prev_7am = self.get_parameter_code(code) + parameter_code, use_prev_7am = self.get_parameter_code( + code) orig_parameter_code = code if obstime_error: raise ShefParser.ParseException(obstime_error) @@ -4469,7 +4528,7 @@ def retokenize(tokens: list[Any]) -> list[Any]: bodytokens = list( map( lambda s: s.strip(), - bodylines[i][len(location) :].strip().split("/"), + bodylines[i][len(location):].strip().split("/"), ) ) bodytokens = retokenize(bodytokens) @@ -4526,7 +4585,8 @@ def retokenize(tokens: list[Any]) -> list[Any]: # ----------------# # units override # # ----------------# - units_override = "EN" if token[2].upper() == "E" else "SI" + units_override = "EN" if token[2].upper( + ) == "E" else "SI" elif self._data_qualifier_pattern.match(token): # ----------------------------# # default qualifier override # @@ -4684,14 +4744,16 @@ def parse( # -------------------------------------------------------# # assign input and output streams if no filenames given # # -------------------------------------------------------# - input: Union[TextIO, str, StringIO] = input_stream or input_name or sys.stdin + input: Union[TextIO, str, + StringIO] = input_stream or input_name or sys.stdin output: Union[TextIO, str] = sys.stdout if not output_name else output_name log: Union[TextIO, str] = sys.stderr if not log_name else log_name # -----------------------------------------------------------------# # get default SHEFPARM file if exists and --default not specified # # -----------------------------------------------------------------# if not shefparm and not use_defaults: - p = Path.joinpath(Path(os.getenv("rfs_sys_dir", Path.cwd())), Path("SHEFPARM")) + p = Path.joinpath( + Path(os.getenv("rfs_sys_dir", Path.cwd())), Path("SHEFPARM")) if p.exists() and not p.is_dir(): shefparm = str(p) elif use_defaults: @@ -4699,7 +4761,8 @@ def parse( # -------------------# # set up the logger # # -------------------# - logfile_name = configure_logging(log, log_level, log_timestamps, append_log) + logfile_name = configure_logging( + log, log_level, log_timestamps, append_log) logger = logging.getLogger(progname) # ------------------# # log startup info # @@ -4725,12 +4788,14 @@ def parse( logger.info( "----------------------------------------------------------------------" ) - logger.debug(f"Input file set to {infile_name} (pre-processed={processed})") + logger.debug( + f"Input file set to {infile_name} (pre-processed={processed})") logger.debug(f"Output file set to {outfile_name}") logger.debug(f"Log file set to {logfile_name}") logger.debug(f"Log level set to {log_level}") if shefparm and not use_defaults: - logger.debug(f"Will modify program defaults with content of file {shefparm}") + logger.debug( + f"Will modify program defaults with content of file {shefparm}") else: logger.debug(f"Will use program defaults") if unload and not loader_spec: @@ -4756,7 +4821,8 @@ def parse( loader_name = loader_spec[:pos] loader_args = loader_spec[pos:] if loader_name in ["abstract", "abstract_loader"]: - raise ShefParser.ParseException("Cannot directly use the base loader") + raise ShefParser.ParseException( + "Cannot directly use the base loader") if loader_name in available_loaders: loader_info = available_loaders[loader_name] elif f"{loader_name}_loader" in available_loaders: @@ -4796,7 +4862,8 @@ def parse( parser.set_output(output, append_output) if loader: parser.set_additional_pe_codes( - loader.get_additional_pe_codes(parser.get_recognized_pe_codes()) + loader.get_additional_pe_codes( + parser.get_recognized_pe_codes()) ) else: if ( @@ -4826,7 +4893,8 @@ def parse( break value_count += 1 if loader: - format_1_str = outrec.format(ShefParser.OutputRecord.SHEFIT_TEXT_V1) + format_1_str = outrec.format( + ShefParser.OutputRecord.SHEFIT_TEXT_V1) loader.set_shef_value(format_1_str) else: parser.output(outrec) @@ -4867,16 +4935,19 @@ def parse( logger.info(f"Program = {progname} version {version}") logger.info(f"SHEFPARM = {shefparm}") logger.info(f"Start Time = {str(start_time)[:-7]}") - logger.info(f"Run Time = {str(datetime.now() - start_time)[:-3]}") + logger.info( + f"Run Time = {str(datetime.now() - start_time)[:-3]}") logger.info( f"{parser._line_number:6d} lines read from {parser._input_name}" ) if not parser.processed: logger.info(f"{message_count:6d} messages processed") if loader: - logger.info(f"{value_count:6d} values passed to {loader.loader_name}") + logger.info( + f"{value_count:6d} values passed to {loader.loader_name}") else: - logger.info(f"{value_count:6d} values output to {parser._output_name}") + logger.info( + f"{value_count:6d} values output to {parser._output_name}") logger.info( f"{parser._warning_count:6d} warnings in {parser._messages_with_warning_count} messages" ) @@ -4898,7 +4969,8 @@ def export( # Require either a timeseries_group (group id) or timeseries_ids (list of ids) if not timeseries_group and not timeseries_ids: - raise ValueError("Either timeseries_group or timeseries_ids must be provided") + raise ValueError( + "Either timeseries_group or timeseries_ids must be provided") if not office: raise ValueError("Office must be provided") @@ -5093,7 +5165,8 @@ def run_parse( "\nArgument --make-shefparm may not be used with any other argument except -o/--out\n" ) raise SystemExit(-1) - ShefParser.write_shefparm_data(output_arg if output_arg else sys.stdout) + ShefParser.write_shefparm_data( + output_arg if output_arg else sys.stdout) raise SystemExit(0) if show_version: @@ -5122,7 +5195,8 @@ def run_parse( import tomli as tomllib else: import tomllib - pyproject_path = Path(__file__).resolve().parent.parent / "pyproject.toml" + pyproject_path = Path(__file__).resolve( + ).parent.parent / "pyproject.toml" with pyproject_path.open("rb") as f: pyproject = tomllib.load(f) date_str = str( @@ -5230,34 +5304,56 @@ def run_export( loglevel, ): """Export timeseries/group from CWMS via CDA and write to SHEF file or stdout.""" - from hec import HecTime, hectime + _relative_pattern = re.compile( + r"^\s*T\s*([+-])\s*(\d+)\s*([SMHDY])\s*$", re.IGNORECASE + ) + _relative_units = { + "S": "seconds", + "M": "minutes", + "H": "hours", + "D": "days", + "Y": "years", + } - def parse_dt(s: Optional[str]): + def parse_dt(s: Optional[str], now: Optional[datetime] = None) -> Optional[datetime]: + """Parse an ISO 8601 datetime or a HEC-style relative time (T, T-1D, T+2H, ...).""" if not s: return None + s = s.strip() + ref = now if now is not None else datetime.utcnow() + if s.upper() == "T": + return ref + m = _relative_pattern.match(s) + if m: + sign = 1 if m.group(1) == "+" else -1 + qty = int(m.group(2)) * sign + unit = m.group(3).upper() + if unit == "Y": + return ref.replace(year=ref.year + qty) + return ref + timedelta(**{_relative_units[unit]: qty}) try: - return datetime.fromisoformat(s).strftime("%m/%d/%Y %H:%M:%S") - except Exception: + return datetime.fromisoformat(s) + except ValueError: try: - return datetime.fromisoformat(s + "T00:00:00").strftime( - "%m/%d/%Y %H:%M:%S" + return datetime.fromisoformat(s + "T00:00:00") + except ValueError: + raise click.BadParameter( + f"Could not parse time [{s}]; expected ISO 8601 (e.g. 2024-01-15T06:00) or relative (e.g. T, T-1D, T+2H)" ) - except Exception: - return s if timeseries_ids is not None: ts_ids = timeseries_ids.replace(" ", "").split(",") else: ts_ids = None - start = HecTime() - end = HecTime() - st = parse_dt(start_time) - et = parse_dt(end_time) - window = st + ", " + et - if hectime.get_time_window(window, start, end) == -1: - click.BadParameter( - f"Invalid time window check start and end times entered: {window}" + now = datetime.utcnow() + start_dt = parse_dt(start_time, now=now) + end_dt = parse_dt(end_time, now=now) + if start_dt is None or end_dt is None: + raise click.BadParameter("Both --start-time and --end-time are required") + if end_dt < start_dt: + raise click.BadParameter( + f"Invalid time window: end [{end_dt}] is before start [{start_dt}]" ) # configure logging for the export command to match parse() behavior @@ -5276,8 +5372,8 @@ def parse_dt(s: Optional[str]): export_file=export_file, timeseries_group=timeseries_group, timeseries_ids=ts_ids, - start_time=start.datetime(), - end_time=end.datetime(), + start_time=start_dt, + end_time=end_dt, ) except ValueError as e: # convert validation errors to Click exceptions so CLI shows a friendly message diff --git a/tests/test_cda_exporter_empty_series.py b/tests/test_cda_exporter_empty_series.py new file mode 100644 index 0000000..ba8ce57 --- /dev/null +++ b/tests/test_cda_exporter_empty_series.py @@ -0,0 +1,127 @@ +"""Regression tests for issue #75: export must not fail when some time series in a group are empty.""" +import json +import types +from io import StringIO + +import pytest + +from shef.exporters import cda_exporter as cda_exporter_mod +from shef.loaders import cda_loader as cda_loader_mod + + +class _FakeResponse: + def __init__(self, payload): + self.json = payload + + +def _build_exporter(monkeypatch, tsids, response_for): + """Construct a CdaExporter wired to canned cwms responses.""" + monkeypatch.setattr(cda_loader_mod.cwms, "init_session", lambda **kw: None) + + # stub the group-fetching call so make_export_transforms returns our tsids + def fake_groups(**kw): + return types.SimpleNamespace( + json=[ + { + "id": "TG", + "description": "", + "assigned-time-series": [ + { + "timeseries-id": tsid, + "office-id": "OFF", + "alias-id": f"LOC{i}.HG.RZ.1:Units=ft", + } + for i, tsid in enumerate(tsids) + ], + } + ] + ) + + monkeypatch.setattr(cda_loader_mod.cwms, "get_timeseries_groups", fake_groups) + + captured_unload = {} + + def fake_get_timeseries(**kw): + return response_for(kw["ts_id"]) + + monkeypatch.setattr(cda_exporter_mod.cwms, "get_timeseries", fake_get_timeseries) + + exporter = cda_exporter_mod.CdaExporter("http://x", "OFF") + + # capture what gets fed to the loader's unload step (parsed back from JSON) + def fake_unload(): + raw = exporter._cda_loader._input.read() + captured_unload["raw"] = raw + captured_unload["parsed"] = json.loads(raw) + + exporter._cda_loader.unload = fake_unload + exporter.set_output(StringIO()) + return exporter, captured_unload + + +def test_export_skips_empty_time_series_and_still_produces_valid_json(monkeypatch): + tsids = [ + "OFF.LOC0.Flow.Inst.1Hour.0.Raw", + "OFF.LOC1.Flow.Inst.1Hour.0.Raw", + "OFF.LOC2.Flow.Inst.1Hour.0.Raw", + ] + + def response_for(tsid): + if tsid.endswith("LOC1.Flow.Inst.1Hour.0.Raw"): + return _FakeResponse( + {"name": tsid, "office-id": "OFF", "units": "ft", "values": []} + ) + return _FakeResponse( + { + "name": tsid, + "office-id": "OFF", + "units": "ft", + "values": [[0, 1.0, 0]], + } + ) + + exporter, captured = _build_exporter(monkeypatch, tsids, response_for) + exporter.export("TG") + + # JSON must be valid (regression for the column-118899 decode failure) + assert "parsed" in captured, "unload was not invoked" + payloads = captured["parsed"] + assert isinstance(payloads, list) + # the empty LOC1 series must be omitted, the other two kept + names = [p["name"] for p in payloads] + assert tsids[0] in names + assert tsids[2] in names + assert tsids[1] not in names + + +def test_export_handles_missing_values_key(monkeypatch): + tsids = ["OFF.LOC0.Flow.Inst.1Hour.0.Raw"] + + def response_for(tsid): + return _FakeResponse({"name": tsid, "office-id": "OFF"}) + + exporter, captured = _build_exporter(monkeypatch, tsids, response_for) + exporter.export("TG") + # no values means unload is never invoked, and the build did not crash + assert "parsed" not in captured + + +def test_export_handles_cwms_exception_per_series(monkeypatch): + tsids = [ + "OFF.LOC0.Flow.Inst.1Hour.0.Raw", + "OFF.LOC1.Flow.Inst.1Hour.0.Raw", + ] + + def response_for(tsid): + if "LOC0" in tsid: + raise RuntimeError("simulated CDA outage for LOC0") + return _FakeResponse( + {"name": tsid, "office-id": "OFF", "units": "ft", "values": [[0, 1.0, 0]]} + ) + + exporter, captured = _build_exporter(monkeypatch, tsids, response_for) + exporter.export("TG") + + parsed = captured["parsed"] + assert len(parsed) == 1 + assert parsed[0]["name"] == tsids[1] diff --git a/tests/test_cda_loader_make_export_transforms.py b/tests/test_cda_loader_make_export_transforms.py new file mode 100644 index 0000000..16b9965 --- /dev/null +++ b/tests/test_cda_loader_make_export_transforms.py @@ -0,0 +1,135 @@ +import types + +from shef.loaders import cda_loader + + +def _fake_groups_response(assigned): + return types.SimpleNamespace( + json=[ + { + "id": "GROUP_A", + "description": "test group", + "assigned-time-series": assigned, + } + ] + ) + + +def _make_loader(): + loader = cda_loader.CdaLoader(logger=None) + loader._office_id = "OFF" + return loader + + +def test_make_export_transforms_skips_missing_alias_and_keeps_others(monkeypatch): + """A time series missing the alias-id key must not abort the rest of the group.""" + assigned = [ + {"timeseries-id": "OFF.BadNoAlias.Flow.Inst.1Hour.0.Raw", "office-id": "OFF"}, + { + "timeseries-id": "OFF.Good1.Flow.Inst.1Hour.0.Raw", + "office-id": "OFF", + "alias-id": "GOOD1.HG.RZ.1:Units=ft", + }, + ] + monkeypatch.setattr( + cda_loader.cwms, + "get_timeseries_groups", + lambda **kw: _fake_groups_response(assigned), + ) + + loader = _make_loader() + loader.make_export_transforms() + + assert loader._export_groups["GROUP_A"]["timeseries"] == [ + "OFF.Good1.Flow.Inst.1Hour.0.Raw" + ] + assert "OFF.Good1.Flow.Inst.1Hour.0.Raw" in loader._transforms + + +def test_make_export_transforms_skips_empty_alias_and_keeps_others(monkeypatch): + """An empty alias-id string must not abort the rest of the group.""" + assigned = [ + { + "timeseries-id": "OFF.BadEmpty.Flow.Inst.1Hour.0.Raw", + "office-id": "OFF", + "alias-id": "", + }, + { + "timeseries-id": "OFF.Good2.Flow.Inst.1Hour.0.Raw", + "office-id": "OFF", + "alias-id": "GOOD2.HG.RZ.1", + }, + ] + monkeypatch.setattr( + cda_loader.cwms, + "get_timeseries_groups", + lambda **kw: _fake_groups_response(assigned), + ) + + loader = _make_loader() + loader.make_export_transforms() + + assert loader._export_groups["GROUP_A"]["timeseries"] == [ + "OFF.Good2.Flow.Inst.1Hour.0.Raw" + ] + + +def test_make_export_transforms_scopes_fetch_to_requested_group(monkeypatch): + """When a group_id is given, only that group should be fetched from cwms (no warnings about other groups).""" + calls = [] + + def fake_get_groups(**kwargs): + calls.append(kwargs) + return _fake_groups_response( + [ + { + "timeseries-id": "OFF.Only.Flow.Inst.1Hour.0.Raw", + "office-id": "OFF", + "alias-id": "ONLY.HG.RZ.1", + } + ] + ) + + monkeypatch.setattr(cda_loader.cwms, "get_timeseries_groups", fake_get_groups) + + loader = _make_loader() + loader.make_export_transforms(group_id="GROUP_A") + + assert len(calls) == 1 + assert calls[0]["timeseries_group_like"] == "^GROUP_A$" + assert calls[0]["group_office_id"] == "OFF" + assert calls[0]["office_id"] == "OFF" + assert "GROUP_A" in loader._loaded_export_group_ids + assert loader._loaded_all_export_groups is False + + # second call for same group is a no-op (no extra fetch) + loader.make_export_transforms(group_id="GROUP_A") + assert len(calls) == 1 + + +def test_make_export_transforms_skips_malformed_alias_and_keeps_others(monkeypatch): + """A malformed alias-id (raises inside make_shef_transform) must not abort the rest.""" + assigned = [ + { + "timeseries-id": "OFF.BadMalformed.Flow.Inst.1Hour.0.Raw", + "office-id": "OFF", + "alias-id": "not-a-valid-shef-alias", + }, + { + "timeseries-id": "OFF.Good3.Flow.Inst.1Hour.0.Raw", + "office-id": "OFF", + "alias-id": "GOOD3.HG.RZ.1", + }, + ] + monkeypatch.setattr( + cda_loader.cwms, + "get_timeseries_groups", + lambda **kw: _fake_groups_response(assigned), + ) + + loader = _make_loader() + loader.make_export_transforms() + + assert "OFF.Good3.Flow.Inst.1Hour.0.Raw" in loader._export_groups["GROUP_A"][ + "timeseries" + ] diff --git a/tests/test_export_time_window.py b/tests/test_export_time_window.py new file mode 100644 index 0000000..4484265 --- /dev/null +++ b/tests/test_export_time_window.py @@ -0,0 +1,150 @@ +"""Tests for the time-window parsing inside run_export (replaces former hec dependency).""" +import sys +import types +from datetime import datetime, timedelta + +import click +import pytest +from click.testing import CliRunner + + +def _install_stub_cda_exporter(monkeypatch): + captured = {} + + class MockExporter: + def __init__(self, api_root, office): + captured["api_root"] = api_root + captured["office"] = office + self.start_time = None + self.end_time = None + + def set_output(self, out): + pass + + def export(self, ts_or_group): + captured["start_time"] = self.start_time + captured["end_time"] = self.end_time + captured["target"] = ts_or_group + + fake_mod = types.SimpleNamespace(CdaExporter=MockExporter) + monkeypatch.setitem(sys.modules, "shef.exporters.cda_exporter", fake_mod) + return captured + + +def _invoke(args): + from shef.shef_parser import cli + + return CliRunner().invoke(cli, args) + + +def test_iso_start_and_end_parse_to_datetimes(monkeypatch, tmp_path): + captured = _install_stub_cda_exporter(monkeypatch) + result = _invoke( + [ + "export", + "--api-root", + "http://x", + "--office", + "OFF", + "--timeseries-group", + "TG", + "--start-time", + "2024-01-15T06:00:00", + "--end-time", + "2024-01-16T06:00:00", + "--export-file", + str(tmp_path / "out.shef"), + ] + ) + assert result.exit_code == 0, result.output + assert captured["start_time"] == datetime(2024, 1, 15, 6, 0, 0) + assert captured["end_time"] == datetime(2024, 1, 16, 6, 0, 0) + + +def test_iso_date_only_parses_as_midnight(monkeypatch, tmp_path): + captured = _install_stub_cda_exporter(monkeypatch) + result = _invoke( + [ + "export", + "--api-root", + "http://x", + "--office", + "OFF", + "--timeseries-group", + "TG", + "--start-time", + "2024-01-15", + "--end-time", + "2024-01-16", + "--export-file", + str(tmp_path / "out.shef"), + ] + ) + assert result.exit_code == 0, result.output + assert captured["start_time"] == datetime(2024, 1, 15, 0, 0, 0) + assert captured["end_time"] == datetime(2024, 1, 16, 0, 0, 0) + + +def test_relative_T_minus_1D_and_T(monkeypatch, tmp_path): + captured = _install_stub_cda_exporter(monkeypatch) + result = _invoke( + [ + "export", + "--api-root", + "http://x", + "--office", + "OFF", + "--timeseries-group", + "GRFT", + "--start-time", + "T-1D", + "--end-time", + "T", + "--export-file", + str(tmp_path / "out.shef"), + ] + ) + assert result.exit_code == 0, result.output + assert captured["end_time"] - captured["start_time"] == timedelta(days=1) + + +def test_invalid_time_string_reports_bad_parameter(monkeypatch, tmp_path): + _install_stub_cda_exporter(monkeypatch) + result = _invoke( + [ + "export", + "--api-root", + "http://x", + "--office", + "OFF", + "--timeseries-group", + "TG", + "--start-time", + "not-a-time", + "--end-time", + "T", + ] + ) + assert result.exit_code != 0 + assert "Could not parse time" in result.output or "not-a-time" in result.output + + +def test_end_before_start_reports_bad_parameter(monkeypatch, tmp_path): + _install_stub_cda_exporter(monkeypatch) + result = _invoke( + [ + "export", + "--api-root", + "http://x", + "--office", + "OFF", + "--timeseries-group", + "TG", + "--start-time", + "2024-01-16", + "--end-time", + "2024-01-15", + ] + ) + assert result.exit_code != 0 + assert "before start" in result.output or "Invalid time window" in result.output From 659be0907113956e8bb3c3d71f0d49cf3141b5c0 Mon Sep 17 00:00:00 2001 From: Eric Novotny Date: Wed, 3 Jun 2026 12:48:56 -0700 Subject: [PATCH 2/4] add automated testing --- .github/workflows/tests.yml | 40 +++++++++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) create mode 100644 .github/workflows/tests.yml diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml new file mode 100644 index 0000000..a3dca81 --- /dev/null +++ b/.github/workflows/tests.yml @@ -0,0 +1,40 @@ +name: Tests + +on: + pull_request: + branches: + - master + workflow_dispatch: + +jobs: + test: + name: Run Tests + runs-on: ubuntu-latest + + strategy: + fail-fast: false + matrix: + python-version: ['3.9', '3.10', '3.11', '3.12'] + + steps: + - uses: actions/checkout@v5 + + - name: Set Up Python + uses: actions/setup-python@v6 + with: + python-version: ${{ matrix.python-version }} + + - name: Install Poetry + uses: abatilo/actions-poetry@v4 + + - name: Cache Virtual Environment + uses: actions/cache@v4 + with: + path: ./.venv + key: venv-${{ matrix.python-version }}-${{ hashFiles('poetry.lock') }} + + - name: Install Dependencies + run: poetry install --all-extras + + - name: Run Tests + run: poetry run pytest From 68d09dd09da7f86336ee0f33874d1ca98db52525 Mon Sep 17 00:00:00 2001 From: Eric Novotny Date: Wed, 3 Jun 2026 12:52:34 -0700 Subject: [PATCH 3/4] fix poetry lock --- poetry.lock | 251 +++++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 249 insertions(+), 2 deletions(-) diff --git a/poetry.lock b/poetry.lock index ed65293..1605791 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry 1.8.4 and should not be changed by hand. +# This file is automatically @generated by Poetry 1.8.2 and should not be changed by hand. [[package]] name = "alabaster" @@ -359,6 +359,22 @@ tomli = {version = "*", optional = true, markers = "python_full_version <= \"3.1 [package.extras] toml = ["tomli"] +[[package]] +name = "cwms-python" +version = "1.0.8" +description = "Corps water management systems (CWMS) REST API for Data Retrieval of USACE water data" +optional = true +python-versions = "<4.0,>=3.9" +files = [ + {file = "cwms_python-1.0.8-py3-none-any.whl", hash = "sha256:9cdc2e6fb60be9561f680900ad2eb7d2cfda2c5c4157fc7fbb94ac3f4dd3f278"}, + {file = "cwms_python-1.0.8.tar.gz", hash = "sha256:1ff75098eec279c51a27a03431c880bf5baaf7ddc0ea3e10b6af0b4b4d35c527"}, +] + +[package.dependencies] +pandas = ">=2.1.3,<3.0.0" +requests = ">=2.31.0,<3.0.0" +requests-toolbelt = ">=1.0.0,<2.0.0" + [[package]] name = "distlib" version = "0.4.0" @@ -431,6 +447,21 @@ files = [ {file = "filelock-3.24.3.tar.gz", hash = "sha256:011a5644dc937c22699943ebbfc46e969cdde3e171470a6e40b9533e5a72affa"}, ] +[[package]] +name = "hecdss" +version = "0.1.29" +description = "Python wrapper for the HEC-DSS file database C library." +optional = true +python-versions = ">=3.8" +files = [ + {file = "hecdss-0.1.29-py3-none-any.whl", hash = "sha256:6884e5a47c98a761c3429ec4aa3ba382e8a6eb55f1d5ac40416ecb0c10757848"}, + {file = "hecdss-0.1.29.tar.gz", hash = "sha256:39289029d0ec6791e6423f0b91617b473c452fe340448452d8134011ad8f8624"}, +] + +[package.dependencies] +numpy = "*" +tzdata = "*" + [[package]] name = "identify" version = "2.6.15" @@ -831,6 +862,60 @@ files = [ {file = "nodeenv-1.10.0.tar.gz", hash = "sha256:996c191ad80897d076bdfba80a41994c2b47c68e224c542b48feba42ba00f8bb"}, ] +[[package]] +name = "numpy" +version = "2.0.2" +description = "Fundamental package for array computing in Python" +optional = true +python-versions = ">=3.9" +files = [ + {file = "numpy-2.0.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:51129a29dbe56f9ca83438b706e2e69a39892b5eda6cedcb6b0c9fdc9b0d3ece"}, + {file = "numpy-2.0.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:f15975dfec0cf2239224d80e32c3170b1d168335eaedee69da84fbe9f1f9cd04"}, + {file = "numpy-2.0.2-cp310-cp310-macosx_14_0_arm64.whl", hash = "sha256:8c5713284ce4e282544c68d1c3b2c7161d38c256d2eefc93c1d683cf47683e66"}, + {file = "numpy-2.0.2-cp310-cp310-macosx_14_0_x86_64.whl", hash = "sha256:becfae3ddd30736fe1889a37f1f580e245ba79a5855bff5f2a29cb3ccc22dd7b"}, + {file = "numpy-2.0.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2da5960c3cf0df7eafefd806d4e612c5e19358de82cb3c343631188991566ccd"}, + {file = "numpy-2.0.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:496f71341824ed9f3d2fd36cf3ac57ae2e0165c143b55c3a035ee219413f3318"}, + {file = "numpy-2.0.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:a61ec659f68ae254e4d237816e33171497e978140353c0c2038d46e63282d0c8"}, + {file = "numpy-2.0.2-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:d731a1c6116ba289c1e9ee714b08a8ff882944d4ad631fd411106a30f083c326"}, + {file = "numpy-2.0.2-cp310-cp310-win32.whl", hash = "sha256:984d96121c9f9616cd33fbd0618b7f08e0cfc9600a7ee1d6fd9b239186d19d97"}, + {file = "numpy-2.0.2-cp310-cp310-win_amd64.whl", hash = "sha256:c7b0be4ef08607dd04da4092faee0b86607f111d5ae68036f16cc787e250a131"}, + {file = "numpy-2.0.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:49ca4decb342d66018b01932139c0961a8f9ddc7589611158cb3c27cbcf76448"}, + {file = "numpy-2.0.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:11a76c372d1d37437857280aa142086476136a8c0f373b2e648ab2c8f18fb195"}, + {file = "numpy-2.0.2-cp311-cp311-macosx_14_0_arm64.whl", hash = "sha256:807ec44583fd708a21d4a11d94aedf2f4f3c3719035c76a2bbe1fe8e217bdc57"}, + {file = "numpy-2.0.2-cp311-cp311-macosx_14_0_x86_64.whl", hash = "sha256:8cafab480740e22f8d833acefed5cc87ce276f4ece12fdaa2e8903db2f82897a"}, + {file = "numpy-2.0.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a15f476a45e6e5a3a79d8a14e62161d27ad897381fecfa4a09ed5322f2085669"}, + {file = "numpy-2.0.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:13e689d772146140a252c3a28501da66dfecd77490b498b168b501835041f951"}, + {file = "numpy-2.0.2-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:9ea91dfb7c3d1c56a0e55657c0afb38cf1eeae4544c208dc465c3c9f3a7c09f9"}, + {file = "numpy-2.0.2-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:c1c9307701fec8f3f7a1e6711f9089c06e6284b3afbbcd259f7791282d660a15"}, + {file = "numpy-2.0.2-cp311-cp311-win32.whl", hash = "sha256:a392a68bd329eafac5817e5aefeb39038c48b671afd242710b451e76090e81f4"}, + {file = "numpy-2.0.2-cp311-cp311-win_amd64.whl", hash = "sha256:286cd40ce2b7d652a6f22efdfc6d1edf879440e53e76a75955bc0c826c7e64dc"}, + {file = "numpy-2.0.2-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:df55d490dea7934f330006d0f81e8551ba6010a5bf035a249ef61a94f21c500b"}, + {file = "numpy-2.0.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:8df823f570d9adf0978347d1f926b2a867d5608f434a7cff7f7908c6570dcf5e"}, + {file = "numpy-2.0.2-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:9a92ae5c14811e390f3767053ff54eaee3bf84576d99a2456391401323f4ec2c"}, + {file = "numpy-2.0.2-cp312-cp312-macosx_14_0_x86_64.whl", hash = "sha256:a842d573724391493a97a62ebbb8e731f8a5dcc5d285dfc99141ca15a3302d0c"}, + {file = "numpy-2.0.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c05e238064fc0610c840d1cf6a13bf63d7e391717d247f1bf0318172e759e692"}, + {file = "numpy-2.0.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0123ffdaa88fa4ab64835dcbde75dcdf89c453c922f18dced6e27c90d1d0ec5a"}, + {file = "numpy-2.0.2-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:96a55f64139912d61de9137f11bf39a55ec8faec288c75a54f93dfd39f7eb40c"}, + {file = "numpy-2.0.2-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:ec9852fb39354b5a45a80bdab5ac02dd02b15f44b3804e9f00c556bf24b4bded"}, + {file = "numpy-2.0.2-cp312-cp312-win32.whl", hash = "sha256:671bec6496f83202ed2d3c8fdc486a8fc86942f2e69ff0e986140339a63bcbe5"}, + {file = "numpy-2.0.2-cp312-cp312-win_amd64.whl", hash = "sha256:cfd41e13fdc257aa5778496b8caa5e856dc4896d4ccf01841daee1d96465467a"}, + {file = "numpy-2.0.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:9059e10581ce4093f735ed23f3b9d283b9d517ff46009ddd485f1747eb22653c"}, + {file = "numpy-2.0.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:423e89b23490805d2a5a96fe40ec507407b8ee786d66f7328be214f9679df6dd"}, + {file = "numpy-2.0.2-cp39-cp39-macosx_14_0_arm64.whl", hash = "sha256:2b2955fa6f11907cf7a70dab0d0755159bca87755e831e47932367fc8f2f2d0b"}, + {file = "numpy-2.0.2-cp39-cp39-macosx_14_0_x86_64.whl", hash = "sha256:97032a27bd9d8988b9a97a8c4d2c9f2c15a81f61e2f21404d7e8ef00cb5be729"}, + {file = "numpy-2.0.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1e795a8be3ddbac43274f18588329c72939870a16cae810c2b73461c40718ab1"}, + {file = "numpy-2.0.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f26b258c385842546006213344c50655ff1555a9338e2e5e02a0756dc3e803dd"}, + {file = "numpy-2.0.2-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:5fec9451a7789926bcf7c2b8d187292c9f93ea30284802a0ab3f5be8ab36865d"}, + {file = "numpy-2.0.2-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:9189427407d88ff25ecf8f12469d4d39d35bee1db5d39fc5c168c6f088a6956d"}, + {file = "numpy-2.0.2-cp39-cp39-win32.whl", hash = "sha256:905d16e0c60200656500c95b6b8dca5d109e23cb24abc701d41c02d74c6b3afa"}, + {file = "numpy-2.0.2-cp39-cp39-win_amd64.whl", hash = "sha256:a3f4ab0caa7f053f6797fcd4e1e25caee367db3112ef2b6ef82d749530768c73"}, + {file = "numpy-2.0.2-pp39-pypy39_pp73-macosx_10_9_x86_64.whl", hash = "sha256:7f0a0c6f12e07fa94133c8a67404322845220c06a9e80e85999afe727f7438b8"}, + {file = "numpy-2.0.2-pp39-pypy39_pp73-macosx_14_0_x86_64.whl", hash = "sha256:312950fdd060354350ed123c0e25a71327d3711584beaef30cdaa93320c392d4"}, + {file = "numpy-2.0.2-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:26df23238872200f63518dd2aa984cfca675d82469535dc7162dc2ee52d9dd5c"}, + {file = "numpy-2.0.2-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:a46288ec55ebbd58947d31d72be2c63cbf839f0a63b49cb755022310792a3385"}, + {file = "numpy-2.0.2.tar.gz", hash = "sha256:883c987dee1880e2a864ab0dc9892292582510604156762362d9326444636e78"}, +] + [[package]] name = "packaging" version = "26.0" @@ -842,6 +927,105 @@ files = [ {file = "packaging-26.0.tar.gz", hash = "sha256:00243ae351a257117b6a241061796684b084ed1c516a08c48a3f7e147a9d80b4"}, ] +[[package]] +name = "pandas" +version = "2.3.3" +description = "Powerful data structures for data analysis, time series, and statistics" +optional = true +python-versions = ">=3.9" +files = [ + {file = "pandas-2.3.3-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:376c6446ae31770764215a6c937f72d917f214b43560603cd60da6408f183b6c"}, + {file = "pandas-2.3.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:e19d192383eab2f4ceb30b412b22ea30690c9e618f78870357ae1d682912015a"}, + {file = "pandas-2.3.3-cp310-cp310-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5caf26f64126b6c7aec964f74266f435afef1c1b13da3b0636c7518a1fa3e2b1"}, + {file = "pandas-2.3.3-cp310-cp310-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:dd7478f1463441ae4ca7308a70e90b33470fa593429f9d4c578dd00d1fa78838"}, + {file = "pandas-2.3.3-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:4793891684806ae50d1288c9bae9330293ab4e083ccd1c5e383c34549c6e4250"}, + {file = "pandas-2.3.3-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:28083c648d9a99a5dd035ec125d42439c6c1c525098c58af0fc38dd1a7a1b3d4"}, + {file = "pandas-2.3.3-cp310-cp310-win_amd64.whl", hash = "sha256:503cf027cf9940d2ceaa1a93cfb5f8c8c7e6e90720a2850378f0b3f3b1e06826"}, + {file = "pandas-2.3.3-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:602b8615ebcc4a0c1751e71840428ddebeb142ec02c786e8ad6b1ce3c8dec523"}, + {file = "pandas-2.3.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:8fe25fc7b623b0ef6b5009149627e34d2a4657e880948ec3c840e9402e5c1b45"}, + {file = "pandas-2.3.3-cp311-cp311-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b468d3dad6ff947df92dcb32ede5b7bd41a9b3cceef0a30ed925f6d01fb8fa66"}, + {file = "pandas-2.3.3-cp311-cp311-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b98560e98cb334799c0b07ca7967ac361a47326e9b4e5a7dfb5ab2b1c9d35a1b"}, + {file = "pandas-2.3.3-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:1d37b5848ba49824e5c30bedb9c830ab9b7751fd049bc7914533e01c65f79791"}, + {file = "pandas-2.3.3-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:db4301b2d1f926ae677a751eb2bd0e8c5f5319c9cb3f88b0becbbb0b07b34151"}, + {file = "pandas-2.3.3-cp311-cp311-win_amd64.whl", hash = "sha256:f086f6fe114e19d92014a1966f43a3e62285109afe874f067f5abbdcbb10e59c"}, + {file = "pandas-2.3.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:6d21f6d74eb1725c2efaa71a2bfc661a0689579b58e9c0ca58a739ff0b002b53"}, + {file = "pandas-2.3.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:3fd2f887589c7aa868e02632612ba39acb0b8948faf5cc58f0850e165bd46f35"}, + {file = "pandas-2.3.3-cp312-cp312-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ecaf1e12bdc03c86ad4a7ea848d66c685cb6851d807a26aa245ca3d2017a1908"}, + {file = "pandas-2.3.3-cp312-cp312-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b3d11d2fda7eb164ef27ffc14b4fcab16a80e1ce67e9f57e19ec0afaf715ba89"}, + {file = "pandas-2.3.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:a68e15f780eddf2b07d242e17a04aa187a7ee12b40b930bfdd78070556550e98"}, + {file = "pandas-2.3.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:371a4ab48e950033bcf52b6527eccb564f52dc826c02afd9a1bc0ab731bba084"}, + {file = "pandas-2.3.3-cp312-cp312-win_amd64.whl", hash = "sha256:a16dcec078a01eeef8ee61bf64074b4e524a2a3f4b3be9326420cabe59c4778b"}, + {file = "pandas-2.3.3-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:56851a737e3470de7fa88e6131f41281ed440d29a9268dcbf0002da5ac366713"}, + {file = "pandas-2.3.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:bdcd9d1167f4885211e401b3036c0c8d9e274eee67ea8d0758a256d60704cfe8"}, + {file = "pandas-2.3.3-cp313-cp313-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e32e7cc9af0f1cc15548288a51a3b681cc2a219faa838e995f7dc53dbab1062d"}, + {file = "pandas-2.3.3-cp313-cp313-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:318d77e0e42a628c04dc56bcef4b40de67918f7041c2b061af1da41dcff670ac"}, + {file = "pandas-2.3.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:4e0a175408804d566144e170d0476b15d78458795bb18f1304fb94160cabf40c"}, + {file = "pandas-2.3.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:93c2d9ab0fc11822b5eece72ec9587e172f63cff87c00b062f6e37448ced4493"}, + {file = "pandas-2.3.3-cp313-cp313-win_amd64.whl", hash = "sha256:f8bfc0e12dc78f777f323f55c58649591b2cd0c43534e8355c51d3fede5f4dee"}, + {file = "pandas-2.3.3-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:75ea25f9529fdec2d2e93a42c523962261e567d250b0013b16210e1d40d7c2e5"}, + {file = "pandas-2.3.3-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:74ecdf1d301e812db96a465a525952f4dde225fdb6d8e5a521d47e1f42041e21"}, + {file = "pandas-2.3.3-cp313-cp313t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6435cb949cb34ec11cc9860246ccb2fdc9ecd742c12d3304989017d53f039a78"}, + {file = "pandas-2.3.3-cp313-cp313t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:900f47d8f20860de523a1ac881c4c36d65efcb2eb850e6948140fa781736e110"}, + {file = "pandas-2.3.3-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:a45c765238e2ed7d7c608fc5bc4a6f88b642f2f01e70c0c23d2224dd21829d86"}, + {file = "pandas-2.3.3-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:c4fc4c21971a1a9f4bdb4c73978c7f7256caa3e62b323f70d6cb80db583350bc"}, + {file = "pandas-2.3.3-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:ee15f284898e7b246df8087fc82b87b01686f98ee67d85a17b7ab44143a3a9a0"}, + {file = "pandas-2.3.3-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:1611aedd912e1ff81ff41c745822980c49ce4a7907537be8692c8dbc31924593"}, + {file = "pandas-2.3.3-cp314-cp314-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6d2cefc361461662ac48810cb14365a365ce864afe85ef1f447ff5a1e99ea81c"}, + {file = "pandas-2.3.3-cp314-cp314-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ee67acbbf05014ea6c763beb097e03cd629961c8a632075eeb34247120abcb4b"}, + {file = "pandas-2.3.3-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:c46467899aaa4da076d5abc11084634e2d197e9460643dd455ac3db5856b24d6"}, + {file = "pandas-2.3.3-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:6253c72c6a1d990a410bc7de641d34053364ef8bcd3126f7e7450125887dffe3"}, + {file = "pandas-2.3.3-cp314-cp314-win_amd64.whl", hash = "sha256:1b07204a219b3b7350abaae088f451860223a52cfb8a6c53358e7948735158e5"}, + {file = "pandas-2.3.3-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:2462b1a365b6109d275250baaae7b760fd25c726aaca0054649286bcfbb3e8ec"}, + {file = "pandas-2.3.3-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:0242fe9a49aa8b4d78a4fa03acb397a58833ef6199e9aa40a95f027bb3a1b6e7"}, + {file = "pandas-2.3.3-cp314-cp314t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a21d830e78df0a515db2b3d2f5570610f5e6bd2e27749770e8bb7b524b89b450"}, + {file = "pandas-2.3.3-cp314-cp314t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:2e3ebdb170b5ef78f19bfb71b0dc5dc58775032361fa188e814959b74d726dd5"}, + {file = "pandas-2.3.3-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:d051c0e065b94b7a3cea50eb1ec32e912cd96dba41647eb24104b6c6c14c5788"}, + {file = "pandas-2.3.3-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:3869faf4bd07b3b66a9f462417d0ca3a9df29a9f6abd5d0d0dbab15dac7abe87"}, + {file = "pandas-2.3.3-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:c503ba5216814e295f40711470446bc3fd00f0faea8a086cbc688808e26f92a2"}, + {file = "pandas-2.3.3-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:a637c5cdfa04b6d6e2ecedcb81fc52ffb0fd78ce2ebccc9ea964df9f658de8c8"}, + {file = "pandas-2.3.3-cp39-cp39-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:854d00d556406bffe66a4c0802f334c9ad5a96b4f1f868adf036a21b11ef13ff"}, + {file = "pandas-2.3.3-cp39-cp39-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:bf1f8a81d04ca90e32a0aceb819d34dbd378a98bf923b6398b9a3ec0bf44de29"}, + {file = "pandas-2.3.3-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:23ebd657a4d38268c7dfbdf089fbc31ea709d82e4923c5ffd4fbd5747133ce73"}, + {file = "pandas-2.3.3-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:5554c929ccc317d41a5e3d1234f3be588248e61f08a74dd17c9eabb535777dc9"}, + {file = "pandas-2.3.3-cp39-cp39-win_amd64.whl", hash = "sha256:d3e28b3e83862ccf4d85ff19cf8c20b2ae7e503881711ff2d534dc8f761131aa"}, + {file = "pandas-2.3.3.tar.gz", hash = "sha256:e05e1af93b977f7eafa636d043f9f94c7ee3ac81af99c13508215942e64c993b"}, +] + +[package.dependencies] +numpy = [ + {version = ">=1.22.4", markers = "python_version < \"3.11\""}, + {version = ">=1.23.2", markers = "python_version == \"3.11\""}, + {version = ">=1.26.0", markers = "python_version >= \"3.12\""}, +] +python-dateutil = ">=2.8.2" +pytz = ">=2020.1" +tzdata = ">=2022.7" + +[package.extras] +all = ["PyQt5 (>=5.15.9)", "SQLAlchemy (>=2.0.0)", "adbc-driver-postgresql (>=0.8.0)", "adbc-driver-sqlite (>=0.8.0)", "beautifulsoup4 (>=4.11.2)", "bottleneck (>=1.3.6)", "dataframe-api-compat (>=0.1.7)", "fastparquet (>=2022.12.0)", "fsspec (>=2022.11.0)", "gcsfs (>=2022.11.0)", "html5lib (>=1.1)", "hypothesis (>=6.46.1)", "jinja2 (>=3.1.2)", "lxml (>=4.9.2)", "matplotlib (>=3.6.3)", "numba (>=0.56.4)", "numexpr (>=2.8.4)", "odfpy (>=1.4.1)", "openpyxl (>=3.1.0)", "pandas-gbq (>=0.19.0)", "psycopg2 (>=2.9.6)", "pyarrow (>=10.0.1)", "pymysql (>=1.0.2)", "pyreadstat (>=1.2.0)", "pytest (>=7.3.2)", "pytest-xdist (>=2.2.0)", "python-calamine (>=0.1.7)", "pyxlsb (>=1.0.10)", "qtpy (>=2.3.0)", "s3fs (>=2022.11.0)", "scipy (>=1.10.0)", "tables (>=3.8.0)", "tabulate (>=0.9.0)", "xarray (>=2022.12.0)", "xlrd (>=2.0.1)", "xlsxwriter (>=3.0.5)", "zstandard (>=0.19.0)"] +aws = ["s3fs (>=2022.11.0)"] +clipboard = ["PyQt5 (>=5.15.9)", "qtpy (>=2.3.0)"] +compression = ["zstandard (>=0.19.0)"] +computation = ["scipy (>=1.10.0)", "xarray (>=2022.12.0)"] +consortium-standard = ["dataframe-api-compat (>=0.1.7)"] +excel = ["odfpy (>=1.4.1)", "openpyxl (>=3.1.0)", "python-calamine (>=0.1.7)", "pyxlsb (>=1.0.10)", "xlrd (>=2.0.1)", "xlsxwriter (>=3.0.5)"] +feather = ["pyarrow (>=10.0.1)"] +fss = ["fsspec (>=2022.11.0)"] +gcp = ["gcsfs (>=2022.11.0)", "pandas-gbq (>=0.19.0)"] +hdf5 = ["tables (>=3.8.0)"] +html = ["beautifulsoup4 (>=4.11.2)", "html5lib (>=1.1)", "lxml (>=4.9.2)"] +mysql = ["SQLAlchemy (>=2.0.0)", "pymysql (>=1.0.2)"] +output-formatting = ["jinja2 (>=3.1.2)", "tabulate (>=0.9.0)"] +parquet = ["pyarrow (>=10.0.1)"] +performance = ["bottleneck (>=1.3.6)", "numba (>=0.56.4)", "numexpr (>=2.8.4)"] +plot = ["matplotlib (>=3.6.3)"] +postgresql = ["SQLAlchemy (>=2.0.0)", "adbc-driver-postgresql (>=0.8.0)", "psycopg2 (>=2.9.6)"] +pyarrow = ["pyarrow (>=10.0.1)"] +spss = ["pyreadstat (>=1.2.0)"] +sql-other = ["SQLAlchemy (>=2.0.0)", "adbc-driver-postgresql (>=0.8.0)", "adbc-driver-sqlite (>=0.8.0)"] +test = ["hypothesis (>=6.46.1)", "pytest (>=7.3.2)", "pytest-xdist (>=2.2.0)"] +xml = ["lxml (>=4.9.2)"] + [[package]] name = "pathspec" version = "1.0.4" @@ -1016,6 +1200,31 @@ pytest = ">=4.6" [package.extras] testing = ["fields", "hunter", "process-tests", "pytest-xdist", "six", "virtualenv"] +[[package]] +name = "python-dateutil" +version = "2.9.0.post0" +description = "Extensions to the standard Python datetime module" +optional = true +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7" +files = [ + {file = "python-dateutil-2.9.0.post0.tar.gz", hash = "sha256:37dd54208da7e1cd875388217d5e00ebd4179249f90fb72437e91a35459a0ad3"}, + {file = "python_dateutil-2.9.0.post0-py2.py3-none-any.whl", hash = "sha256:a8b2bc7bffae282281c8140a97d3aa9c14da0b136dfe83f850eea9a5f7470427"}, +] + +[package.dependencies] +six = ">=1.5" + +[[package]] +name = "pytz" +version = "2026.2" +description = "World timezone definitions, modern and historical" +optional = true +python-versions = "*" +files = [ + {file = "pytz-2026.2-py2.py3-none-any.whl", hash = "sha256:04156e608bee23d3792fd45c94ae47fae1036688e75032eea2e3bf0323d1f126"}, + {file = "pytz-2026.2.tar.gz", hash = "sha256:0e60b47b29f21574376f218fe21abc009894a2321ea16c6754f3cad6eb7cdd6a"}, +] + [[package]] name = "pyyaml" version = "6.0.3" @@ -1119,6 +1328,20 @@ urllib3 = ">=1.21.1,<3" socks = ["PySocks (>=1.5.6,!=1.5.7)"] use-chardet-on-py3 = ["chardet (>=3.0.2,<6)"] +[[package]] +name = "requests-toolbelt" +version = "1.0.0" +description = "A utility belt for advanced users of python-requests" +optional = true +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +files = [ + {file = "requests-toolbelt-1.0.0.tar.gz", hash = "sha256:7681a0a3d047012b5bdc0ee37d7f8f07ebe76ab08caeccfc3921ce23c88d5bc6"}, + {file = "requests_toolbelt-1.0.0-py2.py3-none-any.whl", hash = "sha256:cccfdd665f0a24fcf4726e690f65639d272bb0637b9b92dfd91a5568ccf6bd06"}, +] + +[package.dependencies] +requests = ">=2.0.1,<3.0.0" + [[package]] name = "ruyaml" version = "0.91.0" @@ -1157,6 +1380,17 @@ enabler = ["pytest-enabler (>=2.2)"] test = ["build[virtualenv] (>=1.0.3)", "filelock (>=3.4.0)", "ini2toml[lite] (>=0.14)", "jaraco.develop (>=7.21)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.7.2)", "jaraco.test (>=5.5)", "packaging (>=24.2)", "pip (>=19.1)", "pyproject-hooks (!=1.1)", "pytest (>=6,!=8.1.*)", "pytest-home (>=0.5)", "pytest-perf", "pytest-subprocess", "pytest-timeout", "pytest-xdist (>=3)", "tomli-w (>=1.0.0)", "virtualenv (>=13.0.0)", "wheel (>=0.44.0)"] type = ["importlib_metadata (>=7.0.2)", "jaraco.develop (>=7.21)", "mypy (==1.18.*)", "pytest-mypy"] +[[package]] +name = "six" +version = "1.17.0" +description = "Python 2 and 3 compatibility utilities" +optional = true +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7" +files = [ + {file = "six-1.17.0-py2.py3-none-any.whl", hash = "sha256:4721f391ed90541fddacab5acf947aa0d3dc7d27b2e1e8eda2be8970586c3274"}, + {file = "six-1.17.0.tar.gz", hash = "sha256:ff70335d468e7eb6ec65b95b99d3a2836546063f63acc5171de367e834932a81"}, +] + [[package]] name = "snowballstemmer" version = "3.0.1" @@ -1432,6 +1666,17 @@ files = [ {file = "typing_extensions-4.15.0.tar.gz", hash = "sha256:0cea48d173cc12fa28ecabc3b837ea3cf6f38c6d1136f85cbaaf598984861466"}, ] +[[package]] +name = "tzdata" +version = "2026.2" +description = "Provider of IANA time zone data" +optional = true +python-versions = ">=2" +files = [ + {file = "tzdata-2026.2-py2.py3-none-any.whl", hash = "sha256:bbe9af844f658da81a5f95019480da3a89415801f6cc966806612cc7169bffe7"}, + {file = "tzdata-2026.2.tar.gz", hash = "sha256:9173fde7d80d9018e02a662e168e5a2d04f87c41ea174b139fbef642eda62d10"}, +] + [[package]] name = "urllib3" version = "2.6.3" @@ -1505,9 +1750,11 @@ test = ["big-O", "jaraco.functools", "jaraco.itertools", "jaraco.test", "more_it type = ["pytest-mypy"] [extras] +cda = ["cwms-python"] docs = ["sphinx", "sphinx-design", "sphinx_rtd_theme"] +dss = ["hecdss"] [metadata] lock-version = "2.0" python-versions = "^3.9" -content-hash = "f65a310a25dff39fac54e1029638b00b3c42c0b6ae74772d047d3ad932b7288c" +content-hash = "ac31607b258daeddf40546d97b5ffb17452f729487a15617404e4e249763ebaa" From f13290d6504daf1f732126f012906db65653e150 Mon Sep 17 00:00:00 2001 From: Eric Novotny Date: Wed, 3 Jun 2026 12:58:57 -0700 Subject: [PATCH 4/4] revert formatting-only changes in shef_parser.py --- shef/shef_parser.py | 236 +++++++++++++++----------------------------- 1 file changed, 81 insertions(+), 155 deletions(-) diff --git a/shef/shef_parser.py b/shef/shef_parser.py index 8a15834..0b57321 100644 --- a/shef/shef_parser.py +++ b/shef/shef_parser.py @@ -115,8 +115,7 @@ def configure_logging( ) return log_target else: - logging.basicConfig(stream=log_target, format=fmt, - datefmt=datefmt, level=level) + logging.basicConfig(stream=log_target, format=fmt, datefmt=datefmt, level=level) return log_target.name @@ -899,8 +898,7 @@ def __init__( dt = obstime if relativetime: dt = ( - dt.astimezone( - "Z" if parser.shefit_times else ShefParser.UTC) + dt.astimezone("Z" if parser.shefit_times else ShefParser.UTC) + relativetime ) self._createtime = parser.get_creation_time(dt, createtime_str) @@ -1013,8 +1011,7 @@ def get_output_record( if obst.hour < 7: obst += timedelta( days=-1 - # dont use "obst -1 timedelta(days=1)" - it causes mypy to complain - ) + ) # dont use "obst -1 timedelta(days=1)" - it causes mypy to complain obst = obst.replace(hour=7, minute=0, second=0) elif shift: if isinstance(shift, MonthsDelta): @@ -1030,8 +1027,7 @@ def get_output_record( obst += timedelta(days=days) # 2 - convert to UTC, but keep timezone for later use zi = obst.tzinfo - obst = obst.astimezone( - "Z" if parser.shefit_times else ShefParser.UTC) + obst = obst.astimezone("Z" if parser.shefit_times else ShefParser.UTC) # 3 - adjust to shift hour, minutes, and seconds if shift is not None and isinstance(shift, timedelta): # DON'T use shift.seconds!!! If shift is negative it will be incorrect as shown below. @@ -1053,11 +1049,9 @@ def get_output_record( else: creat = self.createtime if creat: - creat = creat.astimezone( - "Z" if parser.shefit_times else ShefParser.UTC) + creat = creat.astimezone("Z" if parser.shefit_times else ShefParser.UTC) if units_override == "SI": - value = parser.get_english_unit_value( - value, self._parameter_code) + value = parser.get_english_unit_value(value, self._parameter_code) return ShefParser.OutputRecord( parser, @@ -1128,15 +1122,13 @@ def __init__( f"Location [{location}] must be 3 to 8 characters in length" ) if not parameter_code: - raise ShefParser.OutputException( - "Parameter code must not be empty") + raise ShefParser.OutputException("Parameter code must not be empty") if len(parameter_code) != 7: raise ShefParser.OutputException( f"Parameter code [{parameter_code}] must be 7 characters in length" ) if not obstime: - raise ShefParser.OutputException( - "Observed time must not be empty") + raise ShefParser.OutputException("Observed time must not be empty") self._parser = parser self._location = location @@ -1154,8 +1146,7 @@ def __init__( self._creation_time: Union[None, ShefParser.DateTime] = None if create_time and isinstance(create_time, str): - self._creation_time = parser.get_creation_time( - obstime, create_time) + self._creation_time = parser.get_creation_time(obstime, create_time) elif isinstance(create_time, ShefParser.DateTime): self._creation_time = create_time self._observation_time = self._observation_time.astimezone( @@ -1243,8 +1234,7 @@ def format(self, fmt: str) -> str: buf.write(f"{self.time_series_code:2d}") buf.write(" ") buf.write( - self.message_source.ljust( - 8) if self.message_source else " " + self.message_source.ljust(8) if self.message_source else " " ) buf.write(" ") if self.comment: @@ -1286,8 +1276,7 @@ def format(self, fmt: str) -> str: buf.write(f"{self.revised:2d}") buf.write(" ") buf.write( - self.message_source.ljust( - 8) if self.message_source else " " + self.message_source.ljust(8) if self.message_source else " " ) buf.write(f"{self.time_series_code}") if self.comment: @@ -1295,8 +1284,7 @@ def format(self, fmt: str) -> str: rec = buf.getvalue() buf.close() else: - raise ShefParser.OutputException( - f'Invalid output format: "[{fmt}]"') + raise ShefParser.OutputException(f'Invalid output format: "[{fmt}]"') return rec @property @@ -1621,7 +1609,7 @@ def __init__( # 2 = date-time # 6 = time zone # 1 23 4 - r"^\.[AEB]R?\s+(\w{3,8})\s+((\d{2})?(\d{2})?\d{4})" # 5 6 + r"^\.[AEB]R?\s+(\w{3,8})\s+((\d{2})?(\d{2})?\d{4})" # 5 6 r"(\s+([NAECMPYLHB][DS]?|[JZ]))?\s+?", re.I | re.M, ) @@ -1663,8 +1651,7 @@ def __init__( self._create_time_pattern = re.compile(r"DC\d+", re.I) self._unit_system_pattern = re.compile(r"DU[ES]", re.I) self._data_qualifier_pattern = re.compile(r"DQ.", re.I) - self._duration_code_pattern = re.compile( - r"(DV[SNHDMY]\d{1,2}|DVZ)", re.I) + self._duration_code_pattern = re.compile(r"(DV[SNHDMY]\d{1,2}|DVZ)", re.I) self._parameter_code_pattern = re.compile( r"^[A-CE-IL-NP-Y][A-Z](([A-Z]([A-Z0-9]{2})?[A-Z]{1,2})?)?", re.I ) @@ -1683,8 +1670,7 @@ def __init__( self._replacement_strip_pattern = re.compile( "^[" + chr(0) + chr(9) + "]+|[" + chr(0) + chr(9) + "]+$" ) - self._replacement_split_pattern = re.compile( - "[" + chr(0) + chr(9) + "]") + self._replacement_split_pattern = re.compile("[" + chr(0) + chr(9) + "]") if self._shefparm_pathname: self.read_shefparm(self._shefparm_pathname) @@ -1924,8 +1910,7 @@ def set_send_code(self, line: str) -> None: """ Update Send codes from SHEFPARM line """ - key, value = line[0:2], (line[3:10], len(line) - > 12 and line[12] == "1") + key, value = line[0:2], (line[3:10], len(line) > 12 and line[12] == "1") if key not in self._send_codes: self.info( f"{self._shefparm_pathname}: Adding non-standard send code [{key}] with parmameter [{value[0]}] and use-prev-0700 = [{value[1]}]" @@ -1944,8 +1929,7 @@ def set_qualifier_code(self, line: str) -> None: """ key = line[0] if len(key) != 1 or not key.isalpha() or key != key.upper() or key in ("IO"): - self.critical( - f"{self._shefparm_pathname}: Invalid ata qualifier [{key}]") + self.critical(f"{self._shefparm_pathname}: Invalid ata qualifier [{key}]") if key not in self._qualifier_codes: self.info( f"{self._shefparm_pathname}: Adding non-standard data qualifier code [{key}]" @@ -2262,8 +2246,7 @@ def set_output(self, output_object: Union[TextIO, str], append: bool) -> None: if self._output: self.close_output() elif isinstance(output_object, str): - self._output = open( - output_object, "a" if append else "w", encoding="utf-8") + self._output = open(output_object, "a" if append else "w", encoding="utf-8") self._output_name = output_object else: # IO typing is wonky -- see https://github.com/python/typeshed/issues/6077 @@ -2353,8 +2336,7 @@ def get_next_processed_line(self) -> Optional[OutputRecord]: y = int(line[10:14]) m, d, h, n, s = list( map( - int, [line[i: i + 2] - for i in (15, 18, 21, 24, 27)] + int, [line[i : i + 2] for i in (15, 18, 21, 24, 27)] ) ) obstime = ShefParser.DateTime( @@ -2364,7 +2346,7 @@ def get_next_processed_line(self) -> Optional[OutputRecord]: parse_portion = "creation time" _y = line[31:35].strip() _m, _d, _h, _n, _s = [ - line[i: i + 2].strip() for i in (36, 39, 42, 45, 48) + line[i : i + 2].strip() for i in (36, 39, 42, 45, 48) ] if all([_y, _m, _d, _h, _n, _s]): y, m, d, h, n, s = list( @@ -2372,8 +2354,7 @@ def get_next_processed_line(self) -> Optional[OutputRecord]: ) if all([y, m, d, h, n, s]): create_time = ShefParser.DateTime( - y, m, d, h, n, s, tzinfo=ZoneInfo( - "UTC") + y, m, d, h, n, s, tzinfo=ZoneInfo("UTC") ) else: assert not any([y, m, d, h, n, s]) @@ -2427,8 +2408,7 @@ def get_next_processed_line(self) -> Optional[OutputRecord]: y = int(line[8:12]) m, d, h, n, s = list( map( - int, [line[i: i + 2] - for i in (12, 14, 16, 18, 20)] + int, [line[i : i + 2] for i in (12, 14, 16, 18, 20)] ) ) obstime = ShefParser.DateTime( @@ -2438,7 +2418,7 @@ def get_next_processed_line(self) -> Optional[OutputRecord]: parse_portion = "creation time" _y = line[23:27].strip() _m, _d, _h, _n, _s = [ - line[i: i + 2].strip() for i in (27, 29, 31, 33, 35) + line[i : i + 2].strip() for i in (27, 29, 31, 33, 35) ] if all([_y, _m, _d, _h, _n, _s]): y, m, d, h, n, s = list( @@ -2446,8 +2426,7 @@ def get_next_processed_line(self) -> Optional[OutputRecord]: ) if all([y, m, d, h, n, s]): create_time = ShefParser.DateTime( - y, m, d, h, n, s, tzinfo=ZoneInfo( - "UTC") + y, m, d, h, n, s, tzinfo=ZoneInfo("UTC") ) else: assert not any([y, m, d, h, n, s]) @@ -2458,10 +2437,8 @@ def get_next_processed_line(self) -> Optional[OutputRecord]: pe_code = line[38:40] ts_code = line[41:43] extremum_code = line[43] - probability_code = self._probability_ids[float( - line[56:62])] - duration_code = self._duration_ids[int( - line[62:67])] + probability_code = self._probability_ids[float(line[56:62])] + duration_code = self._duration_ids[int(line[62:67])] parameter_code = f"{pe_code}{duration_code}{ts_code}{extremum_code}{probability_code}" assert parameter_code.isascii() @@ -2492,8 +2469,7 @@ def get_next_processed_line(self) -> Optional[OutputRecord]: f"Error parsing {parse_portion} for pre-processed input: {line}" ) else: - self.error( - f"Unrecognized line for pre-processed input: {line}") + self.error(f"Unrecognized line for pre-processed input: {line}") else: output_rec = ShefParser.OutputRecord( self, @@ -2563,8 +2539,7 @@ def get_next_message(self) -> str: self._line_number += 1 self.debug(f"Removed line from input queue [{line}]") message_line = ( - self.remove_comment_fields(line).rstrip( - "=").rstrip("&").rstrip("=") + self.remove_comment_fields(line).rstrip("=").rstrip("&").rstrip("=") ) if not message_line: continue @@ -2608,19 +2583,16 @@ def get_next_message(self) -> str: continue self._line_number -= 1 self._input_lines.appendleft(line) - self.debug( - f"Restored line to input queue [{line}]") + self.debug(f"Restored line to input queue [{line}]") message_lines.pop() raw_message_lines.pop() self._message_location = ( - self._line_number - - len(raw_message_lines) + 1 + self._line_number - len(raw_message_lines) + 1 ) self._message = "\n".join( list(message_lines) + [".END"] ) - self._raw_message = "\n".join( - raw_message_lines) + self._raw_message = "\n".join(raw_message_lines) self.error( '.B message not finished before next message - missing ".END" appended' ) @@ -2639,8 +2611,7 @@ def get_next_message(self) -> str: else: self._line_number -= 1 self._input_lines.appendleft(line) - self.debug( - f"Restored line to input queue [{line}]") + self.debug(f"Restored line to input queue [{line}]") message_type = "" break if message_lines and not message_type: @@ -2750,11 +2721,9 @@ def parse_header_date( y -= 1 elif month_diff == 6 and cd > d: y += 1 - dateval = ShefParser.DateTime( - y, m, d, 0, 0, 0, tzinfo=time_zone) + dateval = ShefParser.DateTime(y, m, d, 0, 0, 0, tzinfo=time_zone) else: - dateval = ShefParser.DateTime( - y, m, d, 0, 0, 0, tzinfo=time_zone) + dateval = ShefParser.DateTime(y, m, d, 0, 0, 0, tzinfo=time_zone) prev_year = dateval - MonthsDelta(12) cur_diff = dateval - cur_date prev_diff = cur_date - prev_year @@ -2771,8 +2740,7 @@ def parse_header_date( ) dateval = prev_year else: - dateval = ShefParser.DateTime( - y, m, d, 0, 0, 0, tzinfo=time_zone) + dateval = ShefParser.DateTime(y, m, d, 0, 0, 0, tzinfo=time_zone) return dateval, century_specified except: raise ShefParser.ParseException(f"Bad date string: [{datestr}]") @@ -3054,8 +3022,7 @@ def get_observation_time( if century_specified: y = bt.year - bt.year % 100 + int(v[0:2]) else: - y = cur_time.year - \ - cur_time.year % 100 + int(v[0:2]) + y = cur_time.year - cur_time.year % 100 + int(v[0:2]) if y - cur_time.year > 10: y -= 100 else: @@ -3249,8 +3216,7 @@ def get_observation_time( v = subtoken[3:] val = int(v) if abs(val) > 99: - raise ShefParser.ParseException( - "Invalid relative time value") + raise ShefParser.ParseException("Invalid relative time value") if subtoken[2] == "S": if dot_b: relativetime = timedelta(seconds=val) @@ -3293,8 +3259,7 @@ def get_observation_time( except ShefParser.Exc: raise except: - raise ShefParser.ParseException( - f"Bad observation time: [{subtoken}]") + raise ShefParser.ParseException(f"Bad observation time: [{subtoken}]") return obstime, relativetime, century_specified def get_creation_time( @@ -3305,8 +3270,7 @@ def get_creation_time( """ if not token: return None - curtime = ShefParser.DateTime.now( - "Z" if self.shefit_times else ShefParser.UTC) + curtime = ShefParser.DateTime.now("Z" if self.shefit_times else ShefParser.UTC) threshold = ShefParser.DateTime( obstime.year, obstime.month, obstime.day, 0, 0, 0, tzinfo=obstime.tzinfo ) + MonthsDelta(120) @@ -3321,14 +3285,11 @@ def get_creation_time( int(s[8:10]), int(s[10:12]), ) - dt = ShefParser.DateTime( - y, m, d, h, n, 0, tzinfo=obstime.tzinfo) + dt = ShefParser.DateTime(y, m, d, h, n, 0, tzinfo=obstime.tzinfo) elif length == 10: # yymmddhhnn y = curtime.year - curtime.year % 100 + int(s[0:2]) - m, d, h, n = int(s[2:4]), int( - s[4:6]), int(s[6:8]), int(s[8:10]) - dt = ShefParser.DateTime( - y, m, d, h, n, 0, tzinfo=obstime.tzinfo) + m, d, h, n = int(s[2:4]), int(s[4:6]), int(s[6:8]), int(s[8:10]) + dt = ShefParser.DateTime(y, m, d, h, n, 0, tzinfo=obstime.tzinfo) while dt > threshold: dt2 = dt - MonthsDelta(1200) if not isinstance(dt2, ShefParser.DateTime): @@ -3344,8 +3305,7 @@ def get_creation_time( int(s[4:6]), int(s[6:8]), ) - dt = ShefParser.DateTime( - y, m, d, h, n, 0, tzinfo=obstime.tzinfo) + dt = ShefParser.DateTime(y, m, d, h, n, 0, tzinfo=obstime.tzinfo) while dt > threshold: dt2 = dt - MonthsDelta(1200) if not isinstance(dt2, ShefParser.DateTime): @@ -3354,10 +3314,8 @@ def get_creation_time( ) dt = dt2 elif length == 6: # mmddhh - y, m, d, h, n = obstime.year, int( - s[0:2]), int(s[2:4]), int(s[4:6]), 0 - dt = ShefParser.DateTime( - y, m, d, h, n, 0, tzinfo=obstime.tzinfo) + y, m, d, h, n = obstime.year, int(s[0:2]), int(s[2:4]), int(s[4:6]), 0 + dt = ShefParser.DateTime(y, m, d, h, n, 0, tzinfo=obstime.tzinfo) while dt > threshold: dt2 = dt - MonthsDelta(1200) if not isinstance(dt2, ShefParser.DateTime): @@ -3368,8 +3326,7 @@ def get_creation_time( elif length == 4: # mmdd hour = 12 if obstime.tzinfo in ("Z", ShefParser.UTC) else 24 y, m, d, h, n = obstime.year, int(s[0:2]), int(s[2:4]), hour, 0 - dt = ShefParser.DateTime( - y, m, d, h, n, 0, tzinfo=obstime.tzinfo) + dt = ShefParser.DateTime(y, m, d, h, n, 0, tzinfo=obstime.tzinfo) while dt > threshold: dt2 = dt - MonthsDelta(1200) if not isinstance(dt2, ShefParser.DateTime): @@ -3378,8 +3335,7 @@ def get_creation_time( ) dt = dt2 else: - raise ShefParser.ParseException( - f"Bad creation time: [{token}]") + raise ShefParser.ParseException(f"Bad creation time: [{token}]") return dt except: raise ShefParser.ParseException(f"Bad creation time: [{token}]") @@ -3446,8 +3402,7 @@ def parse_value_token( # 4 = missing valule # 5 = value qualifier matched_groups = "".join( - map(lambda x: "T" if bool(x) else "F", - [m.group(i) for i in (2, 3, 4)]) + map(lambda x: "T" if bool(x) else "F", [m.group(i) for i in (2, 3, 4)]) ) qualifier = None if matched_groups == "TFF": @@ -3558,8 +3513,7 @@ def retokenize(tokens: list[Any]) -> list[Any]: ): if i < len(tokens) - 1: if ( - self._parameter_code_pattern.match( - tokens[i + 1][0]) + self._parameter_code_pattern.match(tokens[i + 1][0]) and tokens[i + 1][0][0] != "D" ): new_tokens.append(tokens[i] + [chr(0)]) @@ -3694,8 +3648,7 @@ def retokenize(tokens: list[Any]) -> list[Any]: # -------------------------------------------------# default_qualifier = token[2].upper() if default_qualifier not in self._qualifier_codes: - self.error( - f"Bad data qualifier: [{default_qualifier}]") + self.error(f"Bad data qualifier: [{default_qualifier}]") return [] if self._reject_problematic else outrecs elif self._duration_code_pattern.match(token): # ----------------------------------------------------------------# @@ -3724,8 +3677,7 @@ def retokenize(tokens: list[Any]) -> list[Any]: # ------------# code = tokens[i][0].upper() if len(code) < 2: - self.error( - f"Invalid PE code: [{code[:min(2, len(code))]}]") + self.error(f"Invalid PE code: [{code[:min(2, len(code))]}]") return [] elif ( code not in self._send_codes @@ -3736,8 +3688,7 @@ def retokenize(tokens: list[Any]) -> list[Any]: f"Unknown PE code: [{code[:min(2, len(code))]}], value(s) will be untransformed" ) try: - parameter_code, use_prev_7am = self.get_parameter_code( - code) + parameter_code, use_prev_7am = self.get_parameter_code(code) orig_parameter_code = code except ShefParser.Exc as spe: self.error(str(spe)) @@ -3838,8 +3789,7 @@ def retokenize(tokens: list[Any]) -> list[Any]: comment = tokens[i][2] if comment: if comment[0] not in "'\"": - self.error( - f"Invalid retained comment [{tokens[i][2]}]") + self.error(f"Invalid retained comment [{tokens[i][2]}]") comment = None if parameter_code[3] == "F" and not createtime_str: @@ -4105,8 +4055,7 @@ def retokenize(tokens: list[Any]) -> list[Any]: else: duration_id = self._duration_ids[duration_code] except KeyError: - self.error( - f"No valid duration code for time interval [{token}]") + self.error(f"No valid duration code for time interval [{token}]") return [] if self._reject_problematic else outrecs parameter_code = ( f"{parameter_code[:2]}{duration_id}{parameter_code[3:]}" @@ -4120,8 +4069,7 @@ def retokenize(tokens: list[Any]) -> list[Any]: return [] if self._reject_problematic else outrecs code = token.upper() if len(code) < 2: - self.error( - f"Invalid PE code: [{code[:min(2, len(code))]}]") + self.error(f"Invalid PE code: [{code[:min(2, len(code))]}]") return [] if self._reject_problematic else outrecs elif ( code not in self._send_codes @@ -4169,16 +4117,14 @@ def retokenize(tokens: list[Any]) -> list[Any]: comment = tokens[i][1] if comment: if comment[0] not in "'\"": - self.error( - f"Invalid retained comment [{tokens[i][2]}]") + self.error(f"Invalid retained comment [{tokens[i][2]}]") comment = None elif not token: # ------------------------------------# # missing value if in list of values # # ------------------------------------# if not (parameter_code and interval): - raise ShefParser.ParseException( - "Null field in data definition") + raise ShefParser.ParseException("Null field in data definition") obstime += interval time_series_code = 2 elif token[0] in "\"'": @@ -4202,8 +4148,7 @@ def retokenize(tokens: list[Any]) -> list[Any]: "Value encountered before parameter code" ) if not interval: - raise ShefParser.ParseException( - "Value encountered before interval") + raise ShefParser.ParseException("Value encountered before interval") if parameter_code[3] == "F" and not createtime_str: self.warning( @@ -4257,8 +4202,7 @@ def retokenize(tokens: list[Any]) -> list[Any]: else: new_tokens.append(temp_tokens[j]) for i in range(len(new_tokens)): - new_tokens[i] = ShefParser.unhide_quoted_whitespace( - new_tokens[i]) + new_tokens[i] = ShefParser.unhide_quoted_whitespace(new_tokens[i]) return new_tokens # ------------------------------------------------------------------------------------# @@ -4270,12 +4214,12 @@ def retokenize(tokens: list[Any]) -> list[Any]: lines = m.group(0).strip().split("\n") lines[0] = lines[0].strip() for i in range(1, len(lines)): - lines[i] = lines[i][len(lines[i].split()[0]):].strip() + lines[i] = lines[i][len(lines[i].split()[0]) :].strip() if lines[i] and lines[0][-1] != "/" and lines[i][0] != "/": lines[0] += "/" lines[0] += lines[i] header = lines[0] - body = "\n".join(message[m.end():].strip().split("\n")[:-1]).strip() + body = "\n".join(message[m.end() :].strip().split("\n")[:-1]).strip() # ------------------------------------# # parse the header positional fields # # ------------------------------------# @@ -4338,10 +4282,9 @@ def retokenize(tokens: list[Any]) -> list[Any]: # --------------------------------------# # process the parameter control fields # # --------------------------------------# - param_str = header[m.end():].strip() + param_str = header[m.end() :].strip() while self._multiple_obs_time_pattern.search(param_str): - param_str = self._multiple_obs_time_pattern.sub( - r"\1@\6\7", param_str) + param_str = self._multiple_obs_time_pattern.sub(r"\1@\6\7", param_str) param_tokens = list( map(lambda s: s.strip().strip("@"), param_str.strip("/").split("/")) ) @@ -4357,8 +4300,7 @@ def retokenize(tokens: list[Any]) -> list[Any]: while True: m = self._obs_time_pattern2.search(token[pos:]) if not m: - self.error( - f"Unexpected data string item: [{token[pos:]}]") + self.error(f"Unexpected data string item: [{token[pos:]}]") return [] try: obstime, relativetime, century_specified = ( @@ -4431,8 +4373,7 @@ def retokenize(tokens: list[Any]) -> list[Any]: self.warning( f"Unknown PE code: [{code[:min(2, len(code))]}], value(s) will be untransformed" ) - parameter_code, use_prev_7am = self.get_parameter_code( - code) + parameter_code, use_prev_7am = self.get_parameter_code(code) orig_parameter_code = code if obstime_error: raise ShefParser.ParseException(obstime_error) @@ -4528,7 +4469,7 @@ def retokenize(tokens: list[Any]) -> list[Any]: bodytokens = list( map( lambda s: s.strip(), - bodylines[i][len(location):].strip().split("/"), + bodylines[i][len(location) :].strip().split("/"), ) ) bodytokens = retokenize(bodytokens) @@ -4585,8 +4526,7 @@ def retokenize(tokens: list[Any]) -> list[Any]: # ----------------# # units override # # ----------------# - units_override = "EN" if token[2].upper( - ) == "E" else "SI" + units_override = "EN" if token[2].upper() == "E" else "SI" elif self._data_qualifier_pattern.match(token): # ----------------------------# # default qualifier override # @@ -4744,16 +4684,14 @@ def parse( # -------------------------------------------------------# # assign input and output streams if no filenames given # # -------------------------------------------------------# - input: Union[TextIO, str, - StringIO] = input_stream or input_name or sys.stdin + input: Union[TextIO, str, StringIO] = input_stream or input_name or sys.stdin output: Union[TextIO, str] = sys.stdout if not output_name else output_name log: Union[TextIO, str] = sys.stderr if not log_name else log_name # -----------------------------------------------------------------# # get default SHEFPARM file if exists and --default not specified # # -----------------------------------------------------------------# if not shefparm and not use_defaults: - p = Path.joinpath( - Path(os.getenv("rfs_sys_dir", Path.cwd())), Path("SHEFPARM")) + p = Path.joinpath(Path(os.getenv("rfs_sys_dir", Path.cwd())), Path("SHEFPARM")) if p.exists() and not p.is_dir(): shefparm = str(p) elif use_defaults: @@ -4761,8 +4699,7 @@ def parse( # -------------------# # set up the logger # # -------------------# - logfile_name = configure_logging( - log, log_level, log_timestamps, append_log) + logfile_name = configure_logging(log, log_level, log_timestamps, append_log) logger = logging.getLogger(progname) # ------------------# # log startup info # @@ -4788,14 +4725,12 @@ def parse( logger.info( "----------------------------------------------------------------------" ) - logger.debug( - f"Input file set to {infile_name} (pre-processed={processed})") + logger.debug(f"Input file set to {infile_name} (pre-processed={processed})") logger.debug(f"Output file set to {outfile_name}") logger.debug(f"Log file set to {logfile_name}") logger.debug(f"Log level set to {log_level}") if shefparm and not use_defaults: - logger.debug( - f"Will modify program defaults with content of file {shefparm}") + logger.debug(f"Will modify program defaults with content of file {shefparm}") else: logger.debug(f"Will use program defaults") if unload and not loader_spec: @@ -4821,8 +4756,7 @@ def parse( loader_name = loader_spec[:pos] loader_args = loader_spec[pos:] if loader_name in ["abstract", "abstract_loader"]: - raise ShefParser.ParseException( - "Cannot directly use the base loader") + raise ShefParser.ParseException("Cannot directly use the base loader") if loader_name in available_loaders: loader_info = available_loaders[loader_name] elif f"{loader_name}_loader" in available_loaders: @@ -4862,8 +4796,7 @@ def parse( parser.set_output(output, append_output) if loader: parser.set_additional_pe_codes( - loader.get_additional_pe_codes( - parser.get_recognized_pe_codes()) + loader.get_additional_pe_codes(parser.get_recognized_pe_codes()) ) else: if ( @@ -4893,8 +4826,7 @@ def parse( break value_count += 1 if loader: - format_1_str = outrec.format( - ShefParser.OutputRecord.SHEFIT_TEXT_V1) + format_1_str = outrec.format(ShefParser.OutputRecord.SHEFIT_TEXT_V1) loader.set_shef_value(format_1_str) else: parser.output(outrec) @@ -4935,19 +4867,16 @@ def parse( logger.info(f"Program = {progname} version {version}") logger.info(f"SHEFPARM = {shefparm}") logger.info(f"Start Time = {str(start_time)[:-7]}") - logger.info( - f"Run Time = {str(datetime.now() - start_time)[:-3]}") + logger.info(f"Run Time = {str(datetime.now() - start_time)[:-3]}") logger.info( f"{parser._line_number:6d} lines read from {parser._input_name}" ) if not parser.processed: logger.info(f"{message_count:6d} messages processed") if loader: - logger.info( - f"{value_count:6d} values passed to {loader.loader_name}") + logger.info(f"{value_count:6d} values passed to {loader.loader_name}") else: - logger.info( - f"{value_count:6d} values output to {parser._output_name}") + logger.info(f"{value_count:6d} values output to {parser._output_name}") logger.info( f"{parser._warning_count:6d} warnings in {parser._messages_with_warning_count} messages" ) @@ -4969,8 +4898,7 @@ def export( # Require either a timeseries_group (group id) or timeseries_ids (list of ids) if not timeseries_group and not timeseries_ids: - raise ValueError( - "Either timeseries_group or timeseries_ids must be provided") + raise ValueError("Either timeseries_group or timeseries_ids must be provided") if not office: raise ValueError("Office must be provided") @@ -5165,8 +5093,7 @@ def run_parse( "\nArgument --make-shefparm may not be used with any other argument except -o/--out\n" ) raise SystemExit(-1) - ShefParser.write_shefparm_data( - output_arg if output_arg else sys.stdout) + ShefParser.write_shefparm_data(output_arg if output_arg else sys.stdout) raise SystemExit(0) if show_version: @@ -5195,8 +5122,7 @@ def run_parse( import tomli as tomllib else: import tomllib - pyproject_path = Path(__file__).resolve( - ).parent.parent / "pyproject.toml" + pyproject_path = Path(__file__).resolve().parent.parent / "pyproject.toml" with pyproject_path.open("rb") as f: pyproject = tomllib.load(f) date_str = str(