diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index a9778fbe85..f1e6eb042c 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -44,11 +44,24 @@ jobs: distribution: 'temurin' java-version: ${{matrix.jdk}} cache: 'gradle' + - name: setup python + uses: actions/setup-python@v6 + with: + python-version: '3.10' - name: build and test id: thebuild run: ./gradlew clean build --info --init-script init.gradle - name: integration tests run: ./gradlew integrationtest --info --init-script init.gradle -PCDA.oracle.database.image=${{matrix.schema.image}} + - name: Upload Python SDK and documentation + if: matrix.schema.env == 'latest' + uses: actions/upload-artifact@v4 + with: + name: cda-python + path: | + clients/python/build/dist/ + clients/python/build/docs/html/ + if-no-files-found: error - name: Create matrix job badge if: success() || failure() # always run even if the previous step fails uses: ./.github/actions/badge diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index f16620ad7f..fe9f696f29 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -66,6 +66,10 @@ jobs: distribution: 'temurin' java-version: '11' cache: 'gradle' + - name: setup python + uses: actions/setup-python@v6 + with: + python-version: '3.10' - name: Set version id: version env: @@ -93,6 +97,8 @@ jobs: - name: show version run: echo ${{ steps.version.outputs.version }} - name: build war + env: + VERSION: ${{ steps.version.outputs.version }} run: ./gradlew build --info --init-script init.gradle -PversionOverride=$VERSION - name: Create GitHub Release id: create_release @@ -100,7 +106,10 @@ jobs: if: github.event_name != 'pull_request' && (github.event.ref == 'refs/heads/develop' || startsWith(github.event.ref, 'refs/tags')) uses: softprops/action-gh-release@v2.6.1 with: - files: cwms-data-api/build/libs/cwms-data-api-${{steps.version.outputs.version}}.war + files: | + cwms-data-api/build/libs/cwms-data-api-${{steps.version.outputs.version}}.war + clients/python/build/dist/*.whl + clients/python/build/dist/*.tar.gz tag_name: ${{steps.version.outputs.version}} generate_release_notes: true prerelease: ${{steps.version.outputs.prerelease}} diff --git a/README.md b/README.md index 9bdd87e38c..be3ce5540d 100644 --- a/README.md +++ b/README.md @@ -74,6 +74,11 @@ End user documentation available here: [📕 Read the Docs](https://cwms-data-ap ## Development notes +Generated SDKs live alongside the API source: + +- [cda-python](clients/python/README.md): Python SDK, built locally; not yet published on PyPI. +- [cwmsjs](clients/typescript/README.md): TypeScript SDK. + Development and runtime currently requires java 11. JDKs and JREs greater than 11 should work, please report if they don't. diff --git a/clients/python/.gitignore b/clients/python/.gitignore new file mode 100644 index 0000000000..7a60b85e14 --- /dev/null +++ b/clients/python/.gitignore @@ -0,0 +1,2 @@ +__pycache__/ +*.pyc diff --git a/clients/python/README.md b/clients/python/README.md new file mode 100644 index 0000000000..be97973a9f --- /dev/null +++ b/clients/python/README.md @@ -0,0 +1,114 @@ +# cda-python + +Python SDK generated from the CWMS Data API (CDA) OpenAPI specification. Install +the `cda-python` distribution and import `cda`. The SDK provides generated +API methods and models; higher-level workflows can wrap it separately. + +## Build and test + +Use Python 3.10 or newer, the repository's Java/Node build prerequisites, and a +running Docker engine for CDA's existing OpenAPI export task. From the repository +root: + +```sh +./gradlew :clients:python:build --init-script init.gradle +``` + +On Windows use `./gradlew.bat`. To select a Python interpreter, pass +`-PpythonExecutable=/path/to/python`. Gradle creates its own virtual environment +under `clients/python/build/venv`; it does not install into your system Python. + +The build generates source and documentation, builds a wheel and source archive, +installs the wheel, and runs HTTP contract tests against a local test server. +The SDK tests require no external CDA instance or credentials. + +To build against a previously exported specification without Docker, pass +`-PpythonOpenApiSpec=/absolute/path/to/openapi.json`. Relative paths are resolved +from `clients/python`. This explicitly replaces the local export for that build; +the generator never silently downloads a different API version. + +| Output | Location | +| --- | --- | +| Generated project | `clients/python/build/cda-python/` | +| API and model documentation | `clients/python/build/cda-python/docs/` | +| HTML reference and tested examples | `clients/python/build/docs/html/` | +| Wheel and source archive | `clients/python/build/dist/` | + +Use `:clients:python:generatePythonClient` to generate only the source and docs. +Use `:clients:python:clean` to remove all Python build outputs. + +## Install locally + +This package has **not been set up or published on PyPI**. Install the wheel from +`clients/python/build/dist/` by supplying its actual filename: + +```sh +python -m pip install /path/to/cda_python--py3-none-any.whl +``` + +```python +from cda import ApiClient, Configuration +from cda.api.offices_api import OfficesApi + +config = Configuration(host="https://cwms-data.usace.army.mil/cwms-data") +with ApiClient(config) as client: + offices = OfficesApi(client).get_offices(has_data=True) + print([office.name for office in offices]) +``` + +For authenticated requests, configure `config.api_key["ApiKey"]` with your key +and `config.api_key_prefix["ApiKey"] = "apikey"`. The host includes the deployment +context, such as `/cwms-data`; generated operation paths omit that prefix. + +See [read_data.py](examples/read_data.py) for a complete time-series, level, and +location example, including changing the CDA root. Run it with the interpreter +where you installed the wheel. The sample requests one page of time-series data; +use `series.next_page` as the next request's `page` argument for longer windows. + +## Specification adjustments + +Python attributes use snake case while serialization preserves CDA's JSON keys. +The preparation script separates rating inheritance from the rating union to +avoid circular imports, preserves discriminator values, and describes time-series +rows as numeric arrays that can contain null values. These adaptations affect +only the Python generator input. Office types accept both the descriptive labels +in the schema and the codes returned by `has-data=true` (for example, `DIS`). +Time-series intervals use CDA's ISO 8601 duration strings, and level variants +require their distinguishing value fields so responses deserialize unambiguously. + +## Versions and documentation publishing + +The package version follows CDA, with Python's PEP 440 spelling: + +| CDA version | Python package version | +| --- | --- | +| `2026.09.03` | `2026.9.3` | +| `2026.09.03-deva` | `2026.9.3.dev1` | +| `2026.09.03-testa` | `2026.9.3rc1` | +| `2026.09.03-a` | `2026.9.3.post1` | +| `2026.09.03-feature/python-sdk` | `2026.9.3.dev0+feature.python.sdk` | +| Untagged CI commit `7098006` | `0.dev0+g7098006` | + +Unlettered `-dev` and `-test` use suffix number 0. Nightly builds use the build +date and a development label. There is no separate generator-version prefix. +Detached CI checkouts without a release tag use a development version containing +the commit hash; these artifacts are not published as CDA releases. +PyPI publishing remains follow-up work; GitHub releases include the wheel and source archive. + +`buildPythonDocs` runs the SDK tests before building Sphinx HTML from the generated +API/model Markdown and the same example functions executed by the tests. Warnings +fail the documentation build. CI uploads the package and HTML for review. + +The companion SDK Pages workflow publishes the generated HTML at +[CDA Python SDK documentation](https://usace.github.io/cwms-data-api/sdk/python/), +alongside cwmsjs. That URL becomes available after both the Python SDK and shared +Pages workflow changes are merged and deployed. Release-specific documentation is +retained under `/cwms-data-api/releases//sdk/python/`; development docs +live under `/cwms-data-api/development/sdk/python/`. + +## Naming + +Generated libraries use the `cda-*` naming convention to distinguish them from +existing CWMS projects. See the [proposal to use a generated SDK underneath +cwms-python](https://github.com/HydrologicEngineeringCenter/cwms-python/issues/299). +The existing TypeScript package is named `cwmsjs`. diff --git a/clients/python/build.gradle b/clients/python/build.gradle new file mode 100644 index 0000000000..138805e092 --- /dev/null +++ b/clients/python/build.gradle @@ -0,0 +1,120 @@ +import org.openapitools.generator.gradle.plugin.tasks.GenerateTask +import org.openapitools.generator.gradle.plugin.tasks.ValidateTask + +plugins { + id 'base' + id 'org.openapi.generator' version '7.15.0' +} + +def specOverride = providers.gradleProperty('pythonOpenApiSpec') +def specFile = specOverride.isPresent() ? file(specOverride.get()) : rootProject.file('cwms-data-api/build/openapi.json') +def preparedSpec = layout.buildDirectory.file('openapi.json') +def generatedClientDir = layout.buildDirectory.dir('cda-python') +def distDir = layout.buildDirectory.dir('dist') +def venvDir = layout.buildDirectory.dir('venv') +def windows = System.getProperty('os.name').toLowerCase().contains('windows') +def python = providers.gradleProperty('pythonExecutable').getOrElse(windows ? 'python' : 'python3') +def venvPython = venvDir.get().file(windows ? 'Scripts/python.exe' : 'bin/python').asFile.absolutePath +def clientVersion = providers.provider { + def output = new ByteArrayOutputStream() + exec { + commandLine python, 'scripts/package_version.py', project.version.toString() + standardOutput = output + } + output.toString('UTF-8').trim() +} + +tasks.register('prepareOpenApiSpec', Exec) { + group 'openapi' + description 'Prepare the CDA specification for the Python generator.' + if (!specOverride.isPresent()) { + dependsOn ':cwms-data-api:executeOpenAPIConversion' + } + commandLine python, 'scripts/prepare_spec.py', specFile.absolutePath, preparedSpec.get().asFile.absolutePath + inputs.file specFile + inputs.file 'scripts/prepare_spec.py' + outputs.file preparedSpec +} + +tasks.register('validateOpenApiSpec', ValidateTask) { + group 'openapi' + description 'Validate the adjusted Python client OpenAPI specification.' + dependsOn prepareOpenApiSpec + inputSpec.set(preparedSpec.get().asFile.absolutePath) +} + +tasks.register('generatePythonClient', GenerateTask) { + group 'openapi' + description 'Generate the cda-python SDK and API documentation.' + dependsOn validateOpenApiSpec + generatorName = 'python' + inputSpec = preparedSpec.get().asFile.absolutePath + outputDir = generatedClientDir.get().asFile.absolutePath + configFile = layout.projectDirectory.file('openapi.config.json').asFile.absolutePath + templateDir = layout.projectDirectory.dir('templates').asFile.absolutePath + inputs.property 'cdaVersion', project.version.toString() + inputs.file 'scripts/package_version.py' + additionalProperties.put('packageVersion', clientVersion) + globalProperties.set([apiDocs: 'true', modelDocs: 'true', apiTests: 'false', modelTests: 'false']) + inputs.file rootProject.file('LICENSE.md') + // Remove obsolete generated modules when operations or schemas disappear. + doFirst { + delete generatedClientDir + } + doLast { + copy { + from rootProject.file('LICENSE.md') + into generatedClientDir + } + } +} + +tasks.register('createPythonEnvironment', Exec) { + group 'build' + description 'Create an isolated Python environment for building and testing the SDK.' + commandLine python, '-m', 'venv', venvDir.get().asFile.absolutePath + inputs.property 'pythonExecutable', python + outputs.file venvPython +} + +tasks.register('installBuildDependencies', Exec) { + dependsOn createPythonEnvironment + commandLine venvPython, '-m', 'pip', 'install', '-r', file('requirements-build.txt') +} + +tasks.register('buildPythonClient', Exec) { + group 'build' + description 'Build the cda-python wheel and source distribution without publishing.' + dependsOn generatePythonClient, installBuildDependencies + commandLine venvPython, '-m', 'build', '--outdir', distDir.get().asFile.absolutePath, + generatedClientDir.get().asFile.absolutePath + doFirst { delete distDir } +} + +tasks.register('installPythonClient', Exec) { + dependsOn buildPythonClient + doFirst { + def wheel = fileTree(distDir).matching { include '*.whl' }.singleFile + commandLine venvPython, '-m', 'pip', 'install', '--force-reinstall', wheel.absolutePath + } +} + +tasks.register('testPythonClient', Exec) { + group 'verification' + description 'Test the installed wheel against a local HTTP server; no CDA database is needed.' + dependsOn installPythonClient + commandLine venvPython, '-m', 'unittest', 'discover', '-s', 'tests', '-v' + environment 'CDA_VERSION', project.version.toString() +} + +tasks.register('buildPythonDocs', Exec) { + group 'documentation' + description 'Build HTML reference and examples from the tested Python SDK.' + dependsOn testPythonClient + commandLine venvPython, 'scripts/build_docs.py' + environment 'CDA_VERSION', project.version.toString() +} + +tasks.named('assemble') { dependsOn buildPythonClient } +tasks.named('check') { dependsOn testPythonClient } +tasks.named('build') { dependsOn buildPythonDocs } diff --git a/clients/python/examples/read_data.py b/clients/python/examples/read_data.py new file mode 100644 index 0000000000..3460298c30 --- /dev/null +++ b/clients/python/examples/read_data.py @@ -0,0 +1,61 @@ +"""Read a time series, location level, and location using the generated SDK.""" + +from cda import ApiClient, Configuration +from cda.api.time_series_api import TimeSeriesApi +from cda.api.levels_api import LevelsApi +from cda.api.locations_api import LocationsApi + + +# Change this to your CDA deployment, including its context path. +CDA_ROOT = "https://cwms-data.usace.army.mil/cwms-data" +# CDA_ROOT = "http://localhost:7000/cwms-data" + +config = Configuration(host=CDA_ROOT) +json_v2 = {"Accept": "application/json;version=2"} + +def get_time_series(client): + series = TimeSeriesApi(client).get_timeseries( + name="KEYS.Elev.Inst.1Hour.0.Ccp-Rev", + office="SWT", + begin="2026-09-01T00:00:00Z", + end="2026-09-02T00:00:00Z", + units="ft", + _headers=json_v2, + _request_timeout=30.0, + ) + return series + + + +def get_level(client): + level = LevelsApi(client).get_levels_with_level_id( + level_id="KEYS.Elev.Inst.0.Top of Conservation", + office="SWT", + effective_date="2026-09-01T00:00:00", + timezone="UTC", + use_exact_effective_date=False, + unit="ft", + _headers=json_v2, + _request_timeout=30.0, + ) + return level + + + +def get_location(client): + location = LocationsApi(client).get_locations_with_location_id( + location_id="KEYS", + office="SWT", + unit="EN", + _request_timeout=30.0, + ) + return location + + +if __name__ == "__main__": + with ApiClient(config) as client: + series = get_time_series(client) + print(series.name, series.units, series.interval, series.values[:3]) + print(get_level(client).to_dict()) + location = get_location(client) + print(location.public_name, location.latitude, location.longitude) diff --git a/clients/python/openapi.config.json b/clients/python/openapi.config.json new file mode 100644 index 0000000000..e8b4dcb02c --- /dev/null +++ b/clients/python/openapi.config.json @@ -0,0 +1,9 @@ +{ + "packageName": "cda", + "projectName": "cda-python", + "packageUrl": "https://github.com/USACE/cwms-data-api", + "gitUserId": "USACE", + "gitRepoId": "cwms-data-api", + "library": "urllib3", + "hideGenerationTimestamp": true +} diff --git a/clients/python/requirements-build.txt b/clients/python/requirements-build.txt new file mode 100644 index 0000000000..1ae6046ad4 --- /dev/null +++ b/clients/python/requirements-build.txt @@ -0,0 +1,3 @@ +build==1.3.0 +sphinx==8.1.3 +myst-parser==4.0.1 diff --git a/clients/python/scripts/build_docs.py b/clients/python/scripts/build_docs.py new file mode 100644 index 0000000000..61e047481d --- /dev/null +++ b/clients/python/scripts/build_docs.py @@ -0,0 +1,61 @@ +"""Render generated API/model Markdown and the executable examples as one site.""" + +import importlib.metadata +import json +import os +import re +from pathlib import Path +import shutil +import subprocess +import sys + +root = Path(__file__).resolve().parents[1] +source = root / "build/docs-source" +output = root / "build/docs/html" +for directory in (source, output): + if directory.exists(): + shutil.rmtree(directory) +source.mkdir(parents=True) +generated = root / "build/cda-python" +shutil.copytree(generated / "docs", source / "reference") +shutil.copyfile(generated / "README.md", source / "README.md") +readme = (source / 'README.md').read_text(encoding='utf-8') +readme = '# Generated SDK reference\n\n' + readme[readme.index('## Documentation for API Endpoints'):] +(source / 'README.md').write_text(readme, encoding='utf-8') +# Generator links use docs/ and README.md; retain that relative layout. +(source / "reference").rename(source / "docs") +filenames = {page.name.casefold(): page.name for page in source.rglob('*.md')} +for page in source.rglob('*.md'): + text = page.read_text(encoding='utf-8') + # OpenAPI Generator emits multiple H1 method sections and GitHub anchors; + # normalize these for Sphinx's heading hierarchy and anchor spelling. + text = re.sub(r'^# (\*\*.*\*\*)$', r'## \1', text, flags=re.MULTILINE) + text = re.sub(r'\[([^\]]+)\]\(\)', r'\1', text) + text = text.replace('[[Back to top]](#)', '') + text = re.sub(r'\[([^\]]+)\]\(\.md\)', r'\1', text) + text = re.sub(r'\[(.*?)\]\((?:str|int|float|bool)\.md\)', r'\1', text) + text = re.sub(r'(?<=\()([^()\s]*?)([^/()\s]+\.md)(?=[#)])', lambda match: match[1] + filenames.get(match[2].casefold(), match[2]), text) + text = re.sub(r'\]\(([^)\s]*#)([^)]+)\)', lambda match: '](' + match[1] + match[2].lower() + ')', text) + page.write_text(text, encoding='utf-8') +shutil.copyfile(root / "examples/read_data.py", source / "read_data.py") +version = importlib.metadata.version("cda-python") +(source / "conf.py").write_text( + f'project = "cda-python"\nrelease = {version!r}\n' + 'extensions = ["myst_parser"]\nhtml_theme = "alabaster"\n' + 'myst_heading_anchors = 4\nexclude_patterns = []\n', encoding="utf-8") +references = "\n".join(f" docs/{path.stem}" for path in sorted((source / "docs").glob("*.md"))) +(source / "index.rst").write_text( + f'cda-python {version}\n' + '=' * (11 + len(version)) + '\n\n' + 'Python SDK for the CWMS Data API. Install the ``cda-python`` distribution; use ``import cda``.\n\n' + f'CDA version: ``{os.environ["CDA_VERSION"]}``.\n\n' + 'The package is not yet published on PyPI. Install a wheel built from this repository.\n\n' + '.. toctree::\n :maxdepth: 1\n\n examples\n README\n\n' + '.. toctree::\n :maxdepth: 1\n :caption: API and model reference\n\n' + references + '\n', encoding="utf-8") +(source / "examples.rst").write_text( + 'Tested examples\n===============\n\n' + 'These functions are executed against a local HTTP server by the SDK tests before this site builds.\n' + 'Change ``CDA_ROOT`` to select your deployment, including its context path.\n\n' + '.. literalinclude:: read_data.py\n :language: python\n\n' + 'The time-series example retrieves one page. Pass ``series.next_page`` as ``page`` to fetch the next page.\n', encoding="utf-8") +subprocess.run([sys.executable, '-m', 'sphinx', '-W', '--keep-going', '-b', 'html', str(source), str(output)], check=True) +(output / "sdk.json").write_text(json.dumps({"name": "cda-python", "version": version, "cda_version": os.environ["CDA_VERSION"]}) + '\n', encoding="utf-8") diff --git a/clients/python/scripts/package_version.py b/clients/python/scripts/package_version.py new file mode 100644 index 0000000000..cada91070b --- /dev/null +++ b/clients/python/scripts/package_version.py @@ -0,0 +1,45 @@ +"""Translate CDA CalVer tags to Python's PEP 440 spelling.""" + +from datetime import date +import re +import sys + + +def package_version(value): + # Detached shallow CI checkouts have no release tag; Gradle uses the SHA. + # Keep these artifacts identifiable without inventing a CDA release version. + if re.fullmatch(r"[0-9a-fA-F]{7,40}", value): + return f"0.dev0+g{value.lower()}" + if re.fullmatch(r"[a-zA-Z0-9._/-]+-nightly", value): + value = f"{date.today():%Y.%m.%d}-{value}" + match = re.fullmatch(r"(\d{4})\.(\d{2})\.(\d{2})(?:-(.+))?", value) + if not match: + raise ValueError("Expected a CDA version YYYY.MM.DD with an optional suffix") + year, month, day = map(int, match.group(1, 2, 3)) + date(year, month, day) # Reject invalid calendar dates. + base = f"{year}.{month}.{day}" + suffix = match.group(4) + if not suffix: + return base + suffix = suffix.lower() + release = re.fullmatch(r"(dev|test)([a-z]*)", suffix) + if release: + number = letter_number(release.group(2)) + return f"{base}{'.dev' if release.group(1) == 'dev' else 'rc'}{number}" + if re.fullmatch(r"[a-z]", suffix): + return f"{base}.post{letter_number(suffix)}" + label = re.sub(r"[^a-z0-9]+", ".", suffix).strip(".") + if not label: + raise ValueError("The CDA development suffix must contain letters or numbers") + return f"{base}.dev0+{label}" + + +def letter_number(value): + number = 0 + for letter in value: + number = number * 26 + ord(letter) - ord('a') + 1 + return number + + +if __name__ == "__main__": + print(package_version(sys.argv[1])) diff --git a/clients/python/scripts/prepare_spec.py b/clients/python/scripts/prepare_spec.py new file mode 100644 index 0000000000..562a772504 --- /dev/null +++ b/clients/python/scripts/prepare_spec.py @@ -0,0 +1,80 @@ +"""Apply Python generator compatibility changes without changing CDA wire names.""" + +import copy +import json +from pathlib import Path +import re +import sys + + +def prepare_spec(source): + spec = copy.deepcopy(source) + spec["paths"] = { + re.sub(r"^/cwms-data(?=/|$)", "", path) or "/": item + for path, item in spec["paths"].items() + } + # Local exports describe the test server; provide a useful public default. + spec["servers"] = [{"url": "https://cwms-data.usace.army.mil/cwms-data"}] + for item in spec["paths"].values(): + for operation in item.values(): + if isinstance(operation, dict) and "operationId" in operation: + operation["operationId"] = re.sub( + r"^(get|post|patch|put|delete)CwmsData", r"\1", operation["operationId"] + ) + + schemas = spec["components"]["schemas"] + # /offices?has-data=true returns database codes (DIS, MSC, MSCR), while + # the published schema lists expanded office type labels. + schemas.get("Office", {}).get("properties", {}).get("type", {}).pop("enum", None) + parent = schemas.get("AbstractRatingMetadata", {}) + if "oneOf" in parent: + # The union imports its children. Give those children a separate base + # containing the common fields so they do not import the union back. + schemas["BaseRatingMetadata"] = { + key: copy.deepcopy(value) + for key, value in parent.items() + if key not in ("oneOf", "discriminator") + } + for schema in schemas.values(): + for part in schema.get("allOf", []): + if part.get("$ref") == "#/components/schemas/AbstractRatingMetadata": + part["$ref"] = "#/components/schemas/BaseRatingMetadata" + # Required literal discriminator values keep the oneOf alternatives + # exclusive when the Python models deserialize a rating response. + discriminator = parent["discriminator"] + for value, ref in discriminator["mapping"].items(): + child = schemas[ref.rsplit("/", 1)[-1]] + child.setdefault("properties", {})[discriminator["propertyName"]] = { + "type": "string", "enum": [value] + } + child.setdefault("required", []).append(discriminator["propertyName"]) + + time_series = schemas.get("TimeSeries", {}).get("properties", {}) + if "interval" in time_series: + time_series["interval"] = { + "type": "string", "description": "Time-series interval as an ISO 8601 duration, such as PT1H.", + "readOnly": True, + } + # CDA identifies level variants by these mutually exclusive payload fields. + # Without them being required, a constant response matches several oneOf models. + for name, field in { + "ConstantLocationLevel": "constant-value", + "SeasonalLocationLevel": "seasonal-values", + "TimeSeriesLocationLevel": "seasonal-time-series-id", + "VirtualLocationLevel": "constituents", + }.items(): + if name in schemas: + required = schemas[name].setdefault("required", []) + if field not in required: + required.append(field) + values = time_series.get("values", {}) + if values.get("items", {}).get("type") == "array": + # CDA returns [epoch_millis, value_or_null, quality], not objects. + values["items"]["items"] = {"type": "number", "nullable": True} + return spec + + +if __name__ == "__main__": + source, target = map(Path, sys.argv[1:]) + target.parent.mkdir(parents=True, exist_ok=True) + target.write_text(json.dumps(prepare_spec(json.loads(source.read_text(encoding="utf-8"))), indent=2) + "\n", encoding="utf-8") diff --git a/clients/python/templates/pyproject.mustache b/clients/python/templates/pyproject.mustache new file mode 100644 index 0000000000..627973640d --- /dev/null +++ b/clients/python/templates/pyproject.mustache @@ -0,0 +1,20 @@ +[project] +name = "cda-python" +version = "{{packageVersion}}" +description = "Generated Python SDK for the CWMS Data API" +readme = "README.md" +requires-python = ">=3.10" +license = {file = "LICENSE.md"} +authors = [{name = "US Army Corps of Engineers"}] +dynamic = ["dependencies"] + +[tool.setuptools.dynamic] +dependencies = {file = ["requirements.txt"]} + +[project.urls] +Repository = "https://github.com/USACE/cwms-data-api" +Documentation = "https://github.com/USACE/cwms-data-api/tree/develop/clients/python" + +[build-system] +requires = ["setuptools>=77"] +build-backend = "setuptools.build_meta" diff --git a/clients/python/tests/test_client.py b/clients/python/tests/test_client.py new file mode 100644 index 0000000000..de10c09751 --- /dev/null +++ b/clients/python/tests/test_client.py @@ -0,0 +1,145 @@ +"""Exercise the installed distribution, including its generated HTTP transport.""" + +import importlib +import importlib.metadata +import json +import os +import pkgutil +import threading +import unittest +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from urllib.parse import parse_qs, urlsplit + +import cda +from cda import ApiClient, Configuration +from cda.api.offices_api import OfficesApi +from cda.exceptions import ApiException +from cda.models.office import Office +from cda.models.time_series import TimeSeries +from cda.models.abstract_rating_metadata import AbstractRatingMetadata +from cda.models.expression_rating import ExpressionRating +from cda.models.location_level import LocationLevel +from scripts.package_version import package_version +from examples.read_data import get_time_series, get_level, get_location + + +class ClientTest(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.requests = [] + cls.response_status = 200 + cls.response_body = [] + + class Handler(BaseHTTPRequestHandler): + def do_GET(self): + cls.requests.append((self.path, self.headers)) + body = json.dumps(cls.response_body).encode() + self.send_response(cls.response_status) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + + def log_message(self, *_args): + pass + + cls.server = ThreadingHTTPServer(("127.0.0.1", 0), Handler) + cls.thread = threading.Thread(target=cls.server.serve_forever, daemon=True) + cls.thread.start() + + @classmethod + def tearDownClass(cls): + cls.server.shutdown() + cls.server.server_close() + cls.thread.join() + + def setUp(self): + self.requests.clear() + type(self).response_status = 200 + type(self).response_body = [] + self.configuration = Configuration( + host=f"http://127.0.0.1:{self.server.server_port}/cwms-data", + api_key={"ApiKey": "test-key"}, + api_key_prefix={"ApiKey": "apikey"}, + ) + + def test_installed_distribution_and_all_modules(self): + distribution = importlib.metadata.distribution("cda-python") + self.assertEqual(distribution.metadata["Name"].replace("_", "-"), "cda-python") + if os.environ.get("CDA_VERSION"): + self.assertEqual(distribution.version, package_version(os.environ["CDA_VERSION"])) + self.assertIn("site-packages", cda.__file__) + self.assertTrue(any(item.startswith("pydantic") for item in distribution.requires)) + for module in pkgutil.walk_packages(cda.__path__, "cda."): + with self.subTest(module=module.name): + importlib.import_module(module.name) + + def test_time_series_numeric_rows_and_missing_values(self): + values = [[1509654000000, 54.3, 0], [1509657600000, None, 5]] + series = TimeSeries.from_dict({"name": "TEST", "office-id": "SWT", "units": "ft", "interval": "PT1H", "values": values}) + self.assertEqual(series.to_dict()["values"], values) + self.assertEqual(series.interval, "PT1H") + + def test_location_level_variants_are_unambiguous(self): + for field, value, model in [ + ("constant-value", 723.0, "ConstantLocationLevel"), + ("seasonal-values", [], "SeasonalLocationLevel"), + ("seasonal-time-series-id", "TEST.Elev.Inst.1Hour.0.Level", "TimeSeriesLocationLevel"), + ("constituents", [], "VirtualLocationLevel"), + ]: + with self.subTest(model=model): + level = LocationLevel.from_dict({"office-id": "SWT", "location-level-id": "TEST.Elev.Inst.0.Normal", field: value}) + self.assertEqual(type(level.actual_instance).__name__, model) + self.assertEqual(level.to_dict()[field], value) + + def test_rating_discriminator_and_common_fields(self): + rating = AbstractRatingMetadata.from_dict({ + "rating-type": "expression-rating", "expression": "I1 * 2", + "office-id": "SWT", "rating-spec-id": "TEST.Stage;Flow.Linear.Production", + }) + self.assertIsInstance(rating.actual_instance, ExpressionRating) + self.assertEqual(rating.to_dict()["office-id"], "SWT") + self.assertEqual(rating.to_dict()["expression"], "I1 * 2") + + def test_offices_response_query_path_and_authentication(self): + type(self).response_body = [ + {"name": "SWT", "long-name": "Tulsa District", "type": "DIS", "reports-to": "SWD"} + ] + with ApiClient(self.configuration) as client: + offices = OfficesApi(client).get_offices(has_data=True) + self.assertIsInstance(offices[0], Office) + self.assertEqual(offices[0].long_name, "Tulsa District") + self.assertEqual(offices[0].type, "DIS") + self.assertEqual(offices[0].to_dict()["reports-to"], "SWD") + path, headers = self.requests[0] + self.assertEqual(urlsplit(path).path, "/cwms-data/offices") + self.assertEqual(parse_qs(urlsplit(path).query), {"has-data": ["true"]}) + self.assertEqual(headers["Authorization"], "apikey test-key") + self.assertIn("application/json", headers["Accept"]) + + def test_http_error_preserves_status_and_body(self): + type(self).response_status = 403 + type(self).response_body = {"message": "Denied", "source": "test", "details": {}} + with ApiClient(self.configuration) as client: + with self.assertRaises(ApiException) as caught: + OfficesApi(client).get_offices() + self.assertEqual(caught.exception.status, 403) + self.assertEqual(json.loads(caught.exception.body)["message"], "Denied") + + def test_documented_examples_use_the_installed_client(self): + cases = [ + (get_time_series, {"name": "TEST", "units": "ft", "interval": "PT1H", "values": [[1, 2.0, 0]]}, "/cwms-data/timeseries"), + (get_level, {"office-id": "SWT", "location-level-id": "TEST", "constant-value": 723.0}, "/cwms-data/levels/KEYS.Elev.Inst.0.Top%20of%20Conservation"), + (get_location, {"office-id": "SWT", "name": "KEYS", "latitude": 36.15, "longitude": -96.25}, "/cwms-data/locations/KEYS"), + ] + with ApiClient(self.configuration) as client: + for example, body, path in cases: + with self.subTest(example=example.__name__): + type(self).response_body = body + result = example(client) + self.assertIsNotNone(result) + self.assertEqual(urlsplit(self.requests[-1][0]).path, path) + + +if __name__ == "__main__": + unittest.main() diff --git a/clients/python/tests/test_spec.py b/clients/python/tests/test_spec.py new file mode 100644 index 0000000000..414bab7b18 --- /dev/null +++ b/clients/python/tests/test_spec.py @@ -0,0 +1,24 @@ +import unittest + +from scripts.prepare_spec import prepare_spec + + +class SpecTest(unittest.TestCase): + def test_local_export_context_is_removed_once(self): + source = { + "paths": {"/cwms-data/offices": {"get": {"operationId": "getCwmsDataOffices"}}}, + "components": {"schemas": {}}, + } + result = prepare_spec(source) + self.assertEqual(result["paths"]["/offices"]["get"]["operationId"], "getOffices") + self.assertIn("/cwms-data/offices", source["paths"]) + self.assertEqual(prepare_spec(result), result) + + def test_wire_paths_and_names_are_preserved(self): + source = { + "paths": {"/timeseries": {"get": {"operationId": "getTimeSeries"}}}, + "components": {"schemas": {"Example": {"properties": {"office-id": {"type": "string"}}}}}, + } + result = prepare_spec(source) + self.assertEqual(result["paths"], source["paths"]) + self.assertEqual(result["components"], source["components"]) diff --git a/clients/python/tests/test_version.py b/clients/python/tests/test_version.py new file mode 100644 index 0000000000..c1f70bfaf3 --- /dev/null +++ b/clients/python/tests/test_version.py @@ -0,0 +1,25 @@ +import unittest +from packaging.version import Version +from scripts.package_version import package_version + + +class VersionTest(unittest.TestCase): + def test_untagged_ci_commit_versions_are_development_artifacts(self): + for revision in ["7098006", "abcdef1", "F9A6B660F8DE10ECCA746AAA3599DBDF802BB966"]: + with self.subTest(revision=revision): + version = Version(package_version(revision)) + self.assertEqual(str(version), f"0.dev0+g{revision.lower()}") + self.assertTrue(version.is_devrelease) + self.assertLess(version, Version("2026.9.3")) + + def test_cda_release_versions_and_order(self): + inputs = ["2026.09.03-dev", "2026.09.03-deva", "2026.09.03-test", "2026.09.03-testa", "2026.09.03", "2026.09.03-a"] + expected = ["2026.9.3.dev0", "2026.9.3.dev1", "2026.9.3rc0", "2026.9.3rc1", "2026.9.3", "2026.9.3.post1"] + self.assertEqual([package_version(value) for value in inputs], expected) + self.assertEqual(sorted(map(Version, expected)), list(map(Version, expected))) + + def test_development_and_invalid_versions(self): + self.assertEqual(package_version("2026.09.03-feature/python-sdk"), "2026.9.3.dev0+feature.python.sdk") + for value in ["2026.02.30", "0.1.0"]: + with self.subTest(value=value), self.assertRaises(ValueError): + package_version(value) diff --git a/docs/source/libraries/python.rst b/docs/source/libraries/python.rst index 12d6ccb61c..38bc74d65a 100644 --- a/docs/source/libraries/python.rst +++ b/docs/source/libraries/python.rst @@ -1,7 +1,24 @@ .. _cwms-python: -CWMS Python Client Library - CWMSpy -=================================== +Python SDK and client libraries +============================== + +Generated CDA SDK: cda-python +---------------------------- + +``cda-python`` provides generated API methods and models, imported as ``cda``. +Its version follows the CDA release, with Python package normalization (for +example, CDA ``2026.09.03`` becomes ``2026.9.3``). + +* `Generator source and local installation `_ +* `Generated Python reference and tested examples `_ + +The generated documentation link is published by the shared SDK Pages workflow +after that workflow and the Python SDK are deployed. The package has not yet been +published to PyPI; install a locally built wheel or a GitHub release artifact. + +CWMS Python client: cwms-python +------------------------------ The `cwms-python` library is a client library for interacting with the CWMS Data API. @@ -21,7 +38,7 @@ Latest Version The latest version of the `cwms-python` library can be found on GitHub: - https://github.com/USACE/cwms-python + https://github.com/HydrologicEngineeringCenter/cwms-python To install the latest version, you can use pip: diff --git a/settings.gradle b/settings.gradle index 6e2ac195cd..e1c4089bed 100644 --- a/settings.gradle +++ b/settings.gradle @@ -16,4 +16,5 @@ include ":cda-gui" include ":docs" project(":docs").projectDir = file("docs") include ":clients:typescript" +include ":clients:python" include ":cda-etl"