diff --git a/.github/workflows/nightly-schedule.yml b/.github/workflows/nightly-schedule.yml index b58a96c114..ee7f37d0d1 100644 --- a/.github/workflows/nightly-schedule.yml +++ b/.github/workflows/nightly-schedule.yml @@ -8,6 +8,8 @@ jobs: develop: permissions: packages: write + pages: write + id-token: write contents: write uses: ./.github/workflows/release.yml secrets: diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index f16620ad7f..f1eed123c7 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -49,6 +49,8 @@ jobs: outputs: api_image: ${{steps.set_image.outputs.api_image}} migration_image: ${{steps.migration-publish.outputs.image}} + source_sha: ${{steps.source.outputs.sha}} + cda_version: ${{steps.version.outputs.version}} steps: - name: Clean up disk space, so we don't run out. if: matrix.platform == 'ubuntu-latest' @@ -60,6 +62,9 @@ jobs: uses: actions/checkout@v5.0.0 with: ref: ${{inputs.branch}} + - name: Record source revision + id: source + run: echo "sha=$(git rev-parse HEAD)" >> "$GITHUB_OUTPUT" - name: setup java uses: actions/setup-java@v5.2.0 with: @@ -165,3 +170,15 @@ jobs: if: ${{ always() }} run: | docker logout ${{ steps.login-ghcr.outputs.registry }} + + sdk-docs: + needs: release + permissions: + contents: write + pages: write + id-token: write + uses: ./.github/workflows/sdk-docs.yml + with: + ref: ${{ needs.release.outputs.source_sha }} + cda-version: ${{ needs.release.outputs.cda_version }} + release: ${{ !inputs.nightly }} diff --git a/.github/workflows/sdk-docs.yml b/.github/workflows/sdk-docs.yml new file mode 100644 index 0000000000..c5140989c5 --- /dev/null +++ b/.github/workflows/sdk-docs.yml @@ -0,0 +1,148 @@ +name: SDK documentation + +on: + pull_request: + branches: [develop] + paths: + - 'clients/**' + - 'cwms-data-api/src/**' + - 'scripts/sdk-docs/**' + - '*.gradle' + - '.github/workflows/sdk-docs.yml' + - '.github/workflows/release.yml' + push: + branches: [develop] + paths: + - 'clients/**' + - 'cwms-data-api/src/**' + - 'scripts/sdk-docs/**' + - '*.gradle' + - '.github/workflows/sdk-docs.yml' + workflow_dispatch: + workflow_call: + inputs: + ref: + type: string + required: true + cda-version: + type: string + required: true + release: + type: boolean + default: false + +permissions: + contents: read + +jobs: + build: + runs-on: ubuntu-latest + outputs: + version: ${{ steps.version.outputs.value }} + steps: + - uses: actions/checkout@v6 + with: + ref: ${{ inputs.ref || github.ref }} + - uses: actions/setup-java@v5 + with: + distribution: temurin + java-version: '11' + cache: gradle + - uses: actions/setup-node@v4 + with: + node-version: '22' + - uses: actions/setup-python@v6 + with: + python-version: '3.13' + - name: Select documentation version + id: version + env: + CDA_VERSION: ${{ inputs.cda-version }} + run: | + if [ -z "$CDA_VERSION" ]; then + CDA_VERSION="$(date -u +%Y.%m.%d)-snapshot" + fi + echo "value=$CDA_VERSION" >> "$GITHUB_OUTPUT" + - name: Test site assembly + run: python -m unittest discover -s scripts/sdk-docs/tests -v + - name: Build and test JavaScript SDK documentation + env: + CDA_VERSION: ${{ steps.version.outputs.value }} + run: ./gradlew :clients:typescript:build --init-script init.gradle "-PversionOverride=$CDA_VERSION" + - name: Build and test Python SDK documentation when present + env: + CDA_VERSION: ${{ steps.version.outputs.value }} + run: | + if [ -f clients/python/build.gradle ]; then + ./gradlew :clients:python:build --init-script init.gradle "-PversionOverride=$CDA_VERSION" + fi + - name: Assemble SDK documentation + env: + CDA_VERSION: ${{ steps.version.outputs.value }} + run: python scripts/sdk-docs/site.py stage . build/sdk-docs "$CDA_VERSION" + - uses: actions/upload-artifact@v4 + with: + name: sdk-docs-${{ github.run_id }}-${{ github.run_attempt }} + path: build/sdk-docs + if-no-files-found: error + + publish: + needs: build + if: github.event_name != 'pull_request' && github.repository == 'USACE/cwms-data-api' + runs-on: ubuntu-latest + concurrency: + group: sdk-pages + cancel-in-progress: false + permissions: + contents: write + pages: write + id-token: write + environment: + name: github-pages + url: ${{ steps.deployment.outputs.page_url }} + steps: + - uses: actions/checkout@v6 + with: + ref: ${{ inputs.ref || github.ref }} + - uses: actions/download-artifact@v4 + with: + name: sdk-docs-${{ github.run_id }}-${{ github.run_attempt }} + path: build/sdk-docs + - name: Restore documentation history + run: | + if git ls-remote --exit-code --heads origin sdk-pages; then + git fetch origin sdk-pages --depth=1 + git worktree add site FETCH_HEAD + else + mkdir site + git -C site init + git -C site switch --orphan sdk-pages + git -C site config include.path "$GITHUB_WORKSPACE/.git/config" + fi + - name: Update documentation site + env: + CDA_VERSION: ${{ needs.build.outputs.version }} + IS_RELEASE: ${{ inputs.release }} + run: | + args=() + if [ "$IS_RELEASE" = true ]; then args+=(--release); fi + python scripts/sdk-docs/site.py publish build/sdk-docs site "$CDA_VERSION" "${args[@]}" + - name: Save documentation history + run: | + git -C site config user.name github-actions[bot] + git -C site config user.email 41898282+github-actions[bot]@users.noreply.github.com + git -C site add . + if ! git -C site diff --cached --quiet; then + git -C site commit -m "Update SDK documentation" + git -C site push origin HEAD:refs/heads/sdk-pages + fi + - uses: actions/configure-pages@v5 + - name: Prepare Pages artifact + run: | + mkdir build/pages + rsync -a --exclude=.git site/ build/pages/ + - uses: actions/upload-pages-artifact@v4 + with: + path: build/pages + - uses: actions/deploy-pages@v4 + id: deployment diff --git a/.github/workflows/tagged-release.yml b/.github/workflows/tagged-release.yml index 7f86ed6f1e..8e0de0e661 100644 --- a/.github/workflows/tagged-release.yml +++ b/.github/workflows/tagged-release.yml @@ -10,6 +10,11 @@ on: - '[0-9][0-9][0-9][0-9].[0-9][0-9].[0-9][0-9]' jobs: release: + permissions: + contents: write + packages: write + pages: write + id-token: write uses: ./.github/workflows/release.yml secrets: token: ${{ secrets.GITHUB_TOKEN }} diff --git a/clients/typescript/README.md b/clients/typescript/README.md index 8321cc20aa..030566ade8 100644 --- a/clients/typescript/README.md +++ b/clients/typescript/README.md @@ -76,3 +76,17 @@ The Gradle build passes the CDA project version into the client package step. Wh `./gradlew :clients:typescript:build` All generated files (source, library, and docs) will be in `[repo]/clients/typescript/cwmsjs` + +## Hosted documentation + +The [CDA SDK site](https://usace.github.io/cwms-data-api/sdk/javascript/) hosts +TypeDoc and the custom examples extracted from passing tests. CDA release builds +publish matching versioned docs, while `develop` publishes separately under +[/development/sdk/javascript/](https://usace.github.io/cwms-data-api/development/sdk/javascript/). +These URLs become available after the SDK Pages workflow is deployed. + +The Gradle build runs the documentation examples before generating their pages. +Run `:clients:typescript:testTypeScriptExamples` to test them directly. The +private-server authorization template is excluded from public examples. +See [the SDK Pages workflow guide](../../scripts/sdk-docs/README.md) for setup, +release retention, and local validation. diff --git a/clients/typescript/build.gradle b/clients/typescript/build.gradle index 245340aa66..1c447a2344 100644 --- a/clients/typescript/build.gradle +++ b/clients/typescript/build.gradle @@ -1,5 +1,4 @@ import com.github.gradle.node.npm.task.NpmTask -import com.github.gradle.node.npm.task.NpxTask import org.openapitools.generator.gradle.plugin.tasks.GenerateTask import org.openapitools.generator.gradle.plugin.tasks.ValidateTask @@ -13,19 +12,25 @@ def npmCacheDir = layout.buildDirectory.dir("npm-cache") def rawSpecFile = layout.projectDirectory.file("cwms-swagger-raw.json") def modifiedSpecFile = layout.projectDirectory.file("cwms-swagger-mod.json") def generatedClientDir = layout.projectDirectory.dir("cwmsjs") +def specOverride = providers.gradleProperty('typescriptOpenApiSpec') +def sourceSpec = specOverride.isPresent() ? file(specOverride.get()) : rootProject.file('cwms-data-api/build/openapi.json') node { nodeProjectDir = projectDir } -tasks.withType(NpmTask).configureEach { - environment.put("npm_config_cache", npmCacheDir.get().asFile.absolutePath) +tasks.named('npmInstall') { + // The generator uses the JS scripts directly; legacy node-jq install hooks + // are unnecessary here. Generated package installation still runs scripts. + args = ['--ignore-scripts', '--no-audit'] } -tasks.withType(NpxTask).configureEach { +tasks.withType(NpmTask).configureEach { environment.put("npm_config_cache", npmCacheDir.get().asFile.absolutePath) + environment.put("npm_config_audit", "false") } + tasks.register('cleanGeneration', Delete) { delete project.layout.projectDirectory.dir('build') delete project.layout.projectDirectory.dir('dist') @@ -40,23 +45,25 @@ clean.dependsOn cleanGeneration tasks.register('copyOpenApiSpec') { group 'openapi' description 'Copy the locally generated CDA OpenAPI spec into the TypeScript client workspace.' - dependsOn ':cwms-data-api:executeOpenAPIConversion' + if (!specOverride.isPresent()) { + dependsOn ':cwms-data-api:executeOpenAPIConversion' + } + inputs.file sourceSpec outputs.file rawSpecFile doLast { copy { - from "$rootDir/cwms-data-api/build/openapi.json" + from sourceSpec into projectDir rename { 'cwms-swagger-raw.json' } } } } -tasks.register('modSpec', NpxTask) { +tasks.register('modSpec', NpmTask) { group 'openapi' description 'Apply TypeScript client adjustments to the CDA OpenAPI spec.' dependsOn npmInstall dependsOn copyOpenApiSpec - command = 'npm' args = ['run', 'modSpec'] inputs.file rawSpecFile inputs.files fileTree('scripts/spec-updates') @@ -90,12 +97,11 @@ tasks.register('generateTypeScriptClient', GenerateTask) { ignoreFileOverride = layout.projectDirectory.file('.openapi-generator-ignore').asFile.absolutePath } -tasks.register('modPackage', NpxTask) { +tasks.register('modPackage', NpmTask) { group 'openapi' description 'Apply package metadata updates to the generated cwmsjs client.' dependsOn npmInstall dependsOn generateTypeScriptClient - command = 'npm' args = ['run', 'modPackage'] environment.put('CDA_CLIENT_VERSION_SUFFIX', project.version.toString()) inputs.file 'package.json' @@ -104,34 +110,31 @@ tasks.register('modPackage', NpxTask) { outputs.file generatedClientDir.file('package.json') } -tasks.register('postGenerate', NpxTask) { +tasks.register('postGenerate', NpmTask) { group 'openapi' description 'Apply source patches required after OpenAPI generation.' dependsOn modPackage - command = 'npm' args = ['run', 'postGenerate'] inputs.dir generatedClientDir inputs.files fileTree('scripts').include('postGenerate.js') outputs.dir generatedClientDir } -tasks.register('installGeneratedClient', NpxTask) { +tasks.register('installGeneratedClient', NpmTask) { group 'build' description 'Install dependencies for the generated cwmsjs client.' dependsOn postGenerate workingDir = generatedClientDir.asFile - command = 'npm' args = ['install'] inputs.file generatedClientDir.file('package.json') outputs.dir generatedClientDir.dir('node_modules') } -tasks.register('buildTypeScriptClient', NpxTask) { +tasks.register('buildTypeScriptClient', NpmTask) { group 'build' description 'Compile the generated cwmsjs TypeScript client.' dependsOn installGeneratedClient workingDir = generatedClientDir.asFile - command = 'npm' args = ['run', 'build'] inputs.dir generatedClientDir.dir('src') inputs.file generatedClientDir.file('package.json') @@ -139,18 +142,30 @@ tasks.register('buildTypeScriptClient', NpxTask) { outputs.dir generatedClientDir.dir('dist') } -tasks.register('buildTypeScriptDocs', NpxTask) { +tasks.register('buildTypeScriptDocs', NpmTask) { group 'documentation' description 'Build TypeDoc and example documentation for the generated cwmsjs client.' - dependsOn buildTypeScriptClient - command = 'npm' + dependsOn 'testTypeScriptExamples' args = ['run', 'buildDocs'] inputs.dir generatedClientDir.dir('src') inputs.dir 'tests' - inputs.files fileTree('scripts').include('buildTypedoc.js', 'tests2exampledocs.js', 'exampletemplate.html') + inputs.files fileTree('scripts').include('buildTypedoc.js', 'tests2exampledocs.js', 'exampleFiles.js', 'exampletemplate.html') outputs.dir generatedClientDir.dir('docs') } +tasks.register('installExampleDependencies', NpmTask) { + workingDir = file('tests') + args = ['ci'] +} + +tasks.register('testTypeScriptExamples', NpmTask) { + group 'verification' + description 'Run the examples before publishing their source as documentation.' + dependsOn buildTypeScriptClient, installExampleDependencies + workingDir = file('tests') + args = ['run', 'test:docs'] +} + tasks.named('build') { dependsOn buildTypeScriptClient dependsOn buildTypeScriptDocs diff --git a/clients/typescript/scripts/exampleFiles.js b/clients/typescript/scripts/exampleFiles.js new file mode 100644 index 0000000000..6ba995375b --- /dev/null +++ b/clients/typescript/scripts/exampleFiles.js @@ -0,0 +1,16 @@ +const fs = require('node:fs'); +const path = require('node:path'); +const crypto = require('node:crypto'); + +const root = path.resolve(__dirname, '..'); +function exampleFiles() { + return ['endpoints', 'generator'].flatMap(directory => + fs.readdirSync(path.join(root, 'tests', directory)) + .filter(name => name.endsWith('.test.js')) + .map(name => path.join('tests', directory, name)) + ).filter(file => !fs.readFileSync(path.join(root, file), 'utf8').includes('//!ignore')).sort(); +} +function digest(file) { + return crypto.createHash('sha256').update(fs.readFileSync(path.join(root, file))).digest('hex'); +} +module.exports = { root, exampleFiles, digest }; diff --git a/clients/typescript/scripts/spec-updates/modSpec.js b/clients/typescript/scripts/spec-updates/modSpec.js index 15e8c4a67d..bf4b166a86 100644 --- a/clients/typescript/scripts/spec-updates/modSpec.js +++ b/clients/typescript/scripts/spec-updates/modSpec.js @@ -72,10 +72,6 @@ function stripCwmsDataPrefix(spec) { paths[normalizedRoute] = routeConfig; } spec.paths = paths; - spec.servers = (spec.servers || []).map((server) => ({ - ...server, - url: server.url.replace(/\/cwms-data\/?$/, ""), - })); } function normalizeOperationIds(spec) { @@ -161,6 +157,13 @@ function main() { const cleaned = removeUniqueItems(spec); const normalized = normalizeStrings(cleaned); + // Operation names use TimeSeries, but CDA routes remain /timeseries. + normalized.paths = Object.fromEntries( + Object.entries(cleaned.paths).map(([route, item]) => [ + route.replace(/\{([^}]+)\}/g, (_, parameter) => `{${replaceInString(parameter)}}`), + normalizeStrings(item), + ]), + ); normalizeOperationIds(normalized); stripCwmsDataPrefix(normalized); diff --git a/clients/typescript/scripts/testExamples.js b/clients/typescript/scripts/testExamples.js new file mode 100644 index 0000000000..c219a4dafb --- /dev/null +++ b/clients/typescript/scripts/testExamples.js @@ -0,0 +1,16 @@ +const fs = require('node:fs'); +const path = require('node:path'); +const { spawnSync } = require('node:child_process'); +const { root, exampleFiles, digest } = require('./exampleFiles'); +const files = exampleFiles(); +const manifest = path.join(root, 'build', 'tested-examples.json'); +fs.mkdirSync(path.dirname(manifest), { recursive: true }); +fs.rmSync(manifest, { force: true }); +if (!files.length) throw new Error('No documentation examples were found'); +const result = spawnSync(process.execPath, [ + '--experimental-vm-modules', path.join(root, 'tests/node_modules/jest/bin/jest.js'), + '--runInBand', '--ci', '--runTestsByPath', ...files.map(file => path.join(root, file)), +], { cwd: path.join(root, 'tests'), stdio: 'inherit' }); +if (result.error) throw result.error; +if (result.status !== 0) process.exit(result.status || 1); +fs.writeFileSync(manifest, JSON.stringify(Object.fromEntries(files.map(file => [file.replaceAll('\\', '/'), digest(file)])), null, 2) + '\n'); diff --git a/clients/typescript/scripts/tests2exampledocs.js b/clients/typescript/scripts/tests2exampledocs.js index 557eca170b..96c26fc855 100644 --- a/clients/typescript/scripts/tests2exampledocs.js +++ b/clients/typescript/scripts/tests2exampledocs.js @@ -6,6 +6,8 @@ const fs = require("fs"); const path = require("path"); const prettier = require("prettier"); +const { exampleFiles, digest, root } = require('./exampleFiles'); +const testedExamples = JSON.parse(fs.readFileSync(path.join(root, 'build/tested-examples.json'), 'utf8')); const prettierOptions = { parser: "babel", @@ -27,16 +29,12 @@ if (!fs.existsSync(outputDirectory)) { fs.mkdirSync(outputDirectory, { recursive: true }); } function getFiles(directories) { - // Combine all the test files across directories we care about - let all_files = []; - directories.forEach((d) => { - all_files = fs - .readdirSync(d) - .filter((fileName) => fileName.endsWith(".test.js")) - .map((fileName) => path.join(d, fileName)) - .concat(all_files); + return exampleFiles().map(file => { + if (testedExamples[file.replaceAll('\\', '/')] !== digest(file)) { + throw new Error(`Example changed since testing: ${file}. Run the documentation tests first.`); + } + return file; }); - return all_files.sort(); } // Read the HTML template fs.readFile(templatePath, "utf8", (err, template) => { @@ -59,8 +57,7 @@ fs.readFile(templatePath, "utf8", (err, template) => { console.log(extractBlock[0]); // Strip out the bits we do not need from the file let block = extractBlock[1] - .replaceAll(/expect\((.*?)\)\.toBeDefined\(\)/g, "console.log($1)") // Convert expects to logs - .replaceAll("await ", "") // Let users decide when to add await + .replaceAll(/expect\((.*?)\)\.toBe(?:Defined\(\)|\(true\))/g, "console.log($1)") .trim(); // Adjust indentation (simple left trim here, more sophisticated methods might be needed) @@ -95,7 +92,7 @@ fs.readFile(templatePath, "utf8", (err, template) => { To Install: npm install cwmsjs --save
-${combinedImports}\n\n${formattedBlock}
+${escapeHtml(combinedImports + "\n\n" + formattedBlock)}

diff --git a/clients/typescript/tests/endpoints/Counties.test.js b/clients/typescript/tests/endpoints/Counties.test.js index aa44d33f2e..678631dff2 100644 --- a/clients/typescript/tests/endpoints/Counties.test.js +++ b/clients/typescript/tests/endpoints/Counties.test.js @@ -4,13 +4,12 @@ import { CountiesApi, Configuration } from "cwmsjs"; import fetch from "node-fetch"; global.fetch = fetch; -// TODO: Why does the query fail when you do not specify version 2 in the headers? -const c_config = new Configuration({ - headers: { - accept: "application/json;version=2", - }, -}); test("Test Counties", async () => { + const c_config = new Configuration({ + headers: { + accept: "application/json;version=2", + }, + }); const c_api = new CountiesApi(c_config); await c_api .getCounties() diff --git a/clients/typescript/tests/endpoints/Projects.test.js b/clients/typescript/tests/endpoints/Projects.test.js index a37f82fefc..35ec0083e6 100644 --- a/clients/typescript/tests/endpoints/Projects.test.js +++ b/clients/typescript/tests/endpoints/Projects.test.js @@ -44,13 +44,4 @@ test("Test Projects", async () => { } }); - await projects_api - .getProjectsLocations({ - office: "SWT", - projectLike: "KEYS*", - }) - .then((data) => { - expect(Array.isArray(data)).toBe(true); - console.log(`Returned ${data.length} project child-location groups`); - }); }, 30000); diff --git a/clients/typescript/tests/endpoints/Stream-Locations.test.js b/clients/typescript/tests/endpoints/Stream-Locations.test.js index 966994ebea..6e0d8f480a 100644 --- a/clients/typescript/tests/endpoints/Stream-Locations.test.js +++ b/clients/typescript/tests/endpoints/Stream-Locations.test.js @@ -21,12 +21,15 @@ test("Test Stream Locations", async () => { firstLocation?.streamLocationNode?.id?.name && firstLocation?.streamLocationNode?.streamNode?.streamId?.name ) { - const detail = await stream_locations_api.getStreamLocationsWithName({ + // The server returns one object here, although the schema declares an + // array. Read the raw JSON until that response schema is corrected. + const response = await stream_locations_api.getStreamLocationsWithNameRaw({ office: firstLocation.streamLocationNode.id.officeId, name: firstLocation.streamLocationNode.id.name, streamId: firstLocation.streamLocationNode.streamNode.streamId.name, }); - expect(Array.isArray(detail)).toBe(true); + const detail = await response.raw.json(); + expect(detail["stream-location-node"]).toBeDefined(); } console.log(`Returned ${data.length} stream locations`); diff --git a/clients/typescript/tests/generator/Configuration.test.js b/clients/typescript/tests/generator/Configuration.test.js index f8ac8dd693..3804cbbcbb 100644 --- a/clients/typescript/tests/generator/Configuration.test.js +++ b/clients/typescript/tests/generator/Configuration.test.js @@ -8,17 +8,18 @@ test("Test Timeseries V2", async () => { // If you want to change the server or any other base level configuration default items you can use // Configuration const v2_config = new Configuration({ - basePath: "https://water.usace.army.mil/cwms-data", + basePath: "https://cwms-data.usace.army.mil/cwms-data", headers: { accept: "application/json;version=2", }, }); const ts_api = new TimeSeriesApi(v2_config); await ts_api - .getDataTimeseries({ + .getTimeSeriesRaw({ office: "SWT", name: "KEYS.Elev.Inst.1Hour.0.Ccp-Rev", }) + .then(response => response.raw.json()) .then((data) => { expect(data?.values).toBeDefined(); }) diff --git a/clients/typescript/tests/generator/Using-Raw-method-for-Authorization.test.js b/clients/typescript/tests/generator/Using-Raw-method-for-Authorization.test.js index b1abc051e8..55d0659ed5 100644 --- a/clients/typescript/tests/generator/Using-Raw-method-for-Authorization.test.js +++ b/clients/typescript/tests/generator/Using-Raw-method-for-Authorization.test.js @@ -1,3 +1,4 @@ +//!ignore Requires a private authenticated district deployment; not a runnable public example. import { Configuration, AuthorizationApi } from "cwmsjs"; import fetch from "node-fetch"; global.fetch = fetch; diff --git a/clients/typescript/tests/package.json b/clients/typescript/tests/package.json index 0de3515f19..c53a4554fc 100644 --- a/clients/typescript/tests/package.json +++ b/clients/typescript/tests/package.json @@ -20,12 +20,14 @@ "docs": "node ../scripts/tests2exampledocs.js", "smoke": "node smoke.js", "test:ts": "npm run jest-modules Timeseries.v2.test.js", - "link": "cd ../cwmsjs && npm link && cd ../tests && npm link cwmsjs" + "link": "cd ../cwmsjs && npm link && cd ../tests && npm link cwmsjs", + "test:docs": "node ../scripts/testExamples.js" }, "jest": { "clearMocks": true, "testMatch": [ - "**/endpoints/**/*.test.js" + "**/endpoints/**/*.test.js", + "**/generator/**/*.test.js" ], "coverageDirectory": "coverage", "testEnvironment": "node", @@ -35,7 +37,10 @@ "transformIgnorePatterns": [ "node_modules/(?!(chalk)/)", "node_modules/(?!(node-fetch)/)" - ] + ], + "moduleNameMapper": { + "^cwmsjs$": "/../cwmsjs/dist/index.js" + } }, "dependencies": { "chalk": "^5.3.0", diff --git a/docs/source/index.rst b/docs/source/index.rst index 9b84789941..1b4d584514 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -33,7 +33,7 @@ Welcome to CWMS Data API documentation! :caption: Data and References Data <./data/index.rst> - Client Libraries + SDKs and Client Repositories RFCs <./rfc/index.rst> diff --git a/docs/source/libraries/index.rst b/docs/source/libraries/index.rst index 5e6fa9014f..44eb102d13 100644 --- a/docs/source/libraries/index.rst +++ b/docs/source/libraries/index.rst @@ -1,5 +1,17 @@ -Client Libraries -================ +SDKs and client repositories +=========================== + +Generated SDK reference and examples are hosted on +`CDA GitHub Pages `_. The site identifies +the CDA and package versions and retains release-specific documentation. + +Generated SDK source lives in +`CDA clients `_. +Higher-level clients can add data analysis, conversions, and application workflows. + +* `cwmsjs generator `_ +* `CWMS Python `_ +* `CWMS CLI `_ .. toctree:: @@ -8,4 +20,4 @@ Client Libraries Java JavaScript Jython - Python \ No newline at end of file + Python diff --git a/docs/source/libraries/javascript.rst b/docs/source/libraries/javascript.rst index 8eec8f5665..d11fb64e5f 100644 --- a/docs/source/libraries/javascript.rst +++ b/docs/source/libraries/javascript.rst @@ -3,4 +3,22 @@ CWMS JavaScript Client Library - cwmsjs ======================================= -This page is coming soon. Please check back later for updates and new content. +``cwmsjs`` is the generated JavaScript/TypeScript SDK for CDA. Its generator now +lives alongside the CDA source. + +* `API reference and tested examples `_ +* `Development documentation `_ +* `Generator source `_ +* `Previous standalone documentation `_ + +The CDA Pages links become available after the SDK documentation workflow is +merged and its first deployment succeeds. The previous site remains available +during the transition. + +Install with ``npm install cwmsjs``. The TypeDoc reference and custom examples are +built from the same generated client. Example pages are extracted from test +source only after those tests pass; changed source requires another test run. + +CDA release builds publish versioned documentation under +``https://usace.github.io/cwms-data-api/releases//sdk/javascript/``. +Development builds have a separate URL and do not overwrite released docs. diff --git a/scripts/sdk-docs/.gitignore b/scripts/sdk-docs/.gitignore new file mode 100644 index 0000000000..c18dd8d83c --- /dev/null +++ b/scripts/sdk-docs/.gitignore @@ -0,0 +1 @@ +__pycache__/ diff --git a/scripts/sdk-docs/README.md b/scripts/sdk-docs/README.md new file mode 100644 index 0000000000..fc5d9060ee --- /dev/null +++ b/scripts/sdk-docs/README.md @@ -0,0 +1,57 @@ +# SDK documentation on CDA GitHub Pages + +The `SDK documentation` workflow builds and tests cwmsjs, generates TypeDoc and +test-derived examples, and assembles the site. If `clients/python` is present it +also builds and tests that SDK's HTML documentation. There is one combined Pages +deployment so SDKs cannot overwrite one another's sites. + +- Pull requests build downloadable HTML artifacts without deploying. +- Changes on `develop` publish development documentation. +- Successful CDA release/nightly workflows call the documentation workflow with + the exact source commit and CDA version used by the release. +- Manual runs rebuild development docs from the selected ref. +- SDK packages are not published to npm or PyPI by this workflow. + +## Addresses + +| Documentation | URL | +| --- | --- | +| SDK index | https://usace.github.io/cwms-data-api/ | +| cwmsjs | https://usace.github.io/cwms-data-api/sdk/javascript/ | +| Development cwmsjs | https://usace.github.io/cwms-data-api/development/sdk/javascript/ | +| CDA release | `https://usace.github.io/cwms-data-api/releases//sdk/javascript/` | + +Python uses `sdk/python/` when its generator is available. `/sdk/` initially +points to development output until the first stable release. Stable releases +then own that path; development and prereleases do not replace it. Rebuilding an +older release also does not replace the current stable version. + +The `sdk-pages` branch stores generated documentation history. The workflow +copies that history into an Actions Pages artifact, excluding Git metadata. +Repository **Settings > Pages > Source** must be **GitHub Actions**, and the +`github-pages` environment must allow the intended development and release refs. +The existing standalone cwmsjs site is not modified; redirecting it is a separate +follow-up after the new site is deployed. + +## Local verification + +```sh +./gradlew :clients:typescript:build --init-script init.gradle +python -m unittest discover -s scripts/sdk-docs/tests -v +python scripts/sdk-docs/site.py stage . build/sdk-docs 2026.09.03 +python scripts/sdk-docs/site.py publish build/sdk-docs build/pages 2026.09.03 --release +``` + +An exported spec can be supplied with +`-PtypescriptOpenApiSpec=/absolute/path/to/openapi.json` instead of running the +Docker-dependent CDA export. It is never downloaded implicitly. + +The documentation tests use the generated package and public CDA. They require +network access but no credentials. Files marked `//!ignore` are not published as +examples. A source hash recorded after successful tests prevents generating docs +from changed, untested example source. Examples retain their `await` expressions. + +The Gradle generator tasks invoke npm directly instead of invoking it through +npx. Legacy node-jq installation hooks are skipped for generator tooling; package +build hooks still execute for the generated client. Dependency vulnerability +auditing remains separate from the documentation build. diff --git a/scripts/sdk-docs/site.py b/scripts/sdk-docs/site.py new file mode 100644 index 0000000000..00ba225adc --- /dev/null +++ b/scripts/sdk-docs/site.py @@ -0,0 +1,84 @@ +"""Assemble SDK documentation and retain released versions on GitHub Pages.""" + +import argparse +import html +import json +from pathlib import Path +import re +import shutil + + +def copy_sdk(source, destination): + if not (source / "index.html").is_file(): + raise ValueError(f"Missing SDK documentation index: {source}") + if any(path.is_symlink() for path in source.rglob('*')): + raise ValueError("Pages documentation must not contain symbolic links") + if destination.exists(): + shutil.rmtree(destination) + shutil.copytree(source, destination) + + +def landing(directory, title, links): + directory.mkdir(parents=True, exist_ok=True) + items = ''.join(f'
  • {html.escape(label)}
    • ' for label, url in links) + (directory / 'index.html').write_text( + '' + f'{html.escape(title)}' + f'

    {html.escape(title)}

      {items}
    ' + '

    CDA guides and client repositories

    \n', encoding='utf-8') + + +def stage(root, output, version): + sdks = [("javascript", "cwmsjs", root / 'clients/typescript/cwmsjs/docs')] + if (root / 'clients/python/build.gradle').exists(): + sdks.append(("python", "cda-python", root / 'clients/python/build/docs/html')) + links = [] + for slug, name, source in sdks: + copy_sdk(source, output / 'sdk' / slug) + if slug == 'javascript': + package_version = json.loads((root / 'clients/typescript/cwmsjs/package.json').read_text())['version'] + else: + package_version = json.loads((source / 'sdk.json').read_text())['version'] + links.append((f'{name} {package_version}', f'{slug}/')) + landing(output / 'sdk', f'CDA {version} SDK documentation', links) + (output / 'version.json').write_text(json.dumps({'cda_version': version}) + '\n', encoding='utf-8') + + +def publish(source, site, version, release): + if version in {'.', '..'} or not re.fullmatch(r'[A-Za-z0-9._-]+', version): + raise ValueError('Invalid documentation version path') + target = site / ('releases/' + version if release else 'development') + copy_sdk(source / 'sdk', target / 'sdk') + # Stable releases own /sdk. Development is the initial default until one exists. + if release and re.fullmatch(r'\d{4}\.\d{2}\.\d{2}(?:-[a-z])?', version): + current_file = site / 'stable.json' + previous = json.loads(current_file.read_text())['version'] if current_file.exists() else '' + if version >= previous: + copy_sdk(source / 'sdk', site / 'sdk') + current_file.write_text(json.dumps({'version': version}), encoding='utf-8') + elif not (site / 'stable.json').exists() and not release: + copy_sdk(source / 'sdk', site / 'sdk') + links = [] + if (site / 'sdk/index.html').exists(): + links.append(('Current SDK documentation', 'sdk/')) + if (site / 'development/sdk/index.html').exists(): + links.append(('Development SDK documentation', 'development/sdk/')) + for entry in sorted((site / 'releases').glob('*'), reverse=True): + if (entry / 'sdk/index.html').exists(): + links.append((f'CDA {entry.name}', f'releases/{entry.name}/sdk/')) + landing(site, 'CWMS Data API SDK documentation', links) + (site / '.nojekyll').touch() + + +if __name__ == '__main__': + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('mode', choices=['stage', 'publish']) + parser.add_argument('source', type=Path) + parser.add_argument('output', type=Path) + parser.add_argument('version') + parser.add_argument('--release', action='store_true') + args = parser.parse_args() + if args.mode == 'stage': + stage(args.source.resolve(), args.output.resolve(), args.version) + else: + publish(args.source.resolve(), args.output.resolve(), args.version, args.release) diff --git a/scripts/sdk-docs/tests/test_site.py b/scripts/sdk-docs/tests/test_site.py new file mode 100644 index 0000000000..8eb7579038 --- /dev/null +++ b/scripts/sdk-docs/tests/test_site.py @@ -0,0 +1,60 @@ +import importlib.util +from pathlib import Path +import tempfile +import unittest + +spec = importlib.util.spec_from_file_location('sdk_site', Path(__file__).resolve().parents[1] / 'site.py') +site = importlib.util.module_from_spec(spec) +spec.loader.exec_module(site) + + +class SiteTest(unittest.TestCase): + def test_stage_includes_python_when_its_generator_is_present(self): + with tempfile.TemporaryDirectory() as temp: + root = Path(temp) + javascript = root / 'clients/typescript/cwmsjs' + (javascript / 'docs').mkdir(parents=True) + (javascript / 'docs/index.html').write_text('javascript') + (javascript / 'package.json').write_text('{"version": "1.0.0"}') + site.stage(root, root / 'js-only', '2026.09.03') + self.assertTrue((root / 'js-only/sdk/javascript/index.html').exists()) + self.assertFalse((root / 'js-only/sdk/python').exists()) + python = root / 'clients/python' + (python / 'build/docs/html').mkdir(parents=True) + (python / 'build.gradle').touch() + with self.assertRaises(ValueError): + site.stage(root, root / 'missing-python-docs', '2026.09.03') + (python / 'build/docs/html/index.html').write_text('python') + (python / 'build/docs/html/sdk.json').write_text('{"version": "2026.9.3"}') + site.stage(root, root / 'combined', '2026.09.03') + self.assertTrue((root / 'combined/sdk/python/index.html').exists()) + self.assertIn('cda-python 2026.9.3', (root / 'combined/sdk/index.html').read_text()) + + def test_versions_are_retained_and_development_does_not_replace_stable(self): + with tempfile.TemporaryDirectory() as temp: + root = Path(temp) + source = root / 'source' + (source / 'sdk/javascript').mkdir(parents=True) + (source / 'sdk/index.html').write_text('sdk') + page = source / 'sdk/javascript/index.html' + page.write_text('release one') + site.publish(source, root / 'site', '2026.09.03', True) + page.write_text('development') + site.publish(source, root / 'site', '2026.09.04-snapshot', False) + self.assertEqual((root / 'site/sdk/javascript/index.html').read_text(), 'release one') + self.assertEqual((root / 'site/development/sdk/javascript/index.html').read_text(), 'development') + page.write_text('release two') + site.publish(source, root / 'site', '2026.09.04', True) + self.assertEqual((root / 'site/releases/2026.09.03/sdk/javascript/index.html').read_text(), 'release one') + page.write_text('older release rebuilt') + site.publish(source, root / 'site', '2026.09.03', True) + self.assertEqual((root / 'site/sdk/javascript/index.html').read_text(), 'release two') + + def test_missing_docs_and_unsafe_version_fail(self): + with tempfile.TemporaryDirectory() as temp: + root = Path(temp) + for version in ('../escape', '.', '..'): + with self.assertRaises(ValueError): + site.publish(root, root / 'site', version, True) + with self.assertRaises(ValueError): + site.copy_sdk(root / 'absent', root / 'target')