ci: publish tested SDK documentation on CDA GitHub Pages

.github/workflows/nightly-schedule.yml

@@ -8,6 +8,8 @@ jobs:
   develop:
     permissions:
       packages: write
+      pages: write
+      id-token: write
       contents: write
     uses: ./.github/workflows/release.yml
     secrets:

.github/workflows/release.yml

@@ -49,6 +49,8 @@ jobs:
     outputs:
       api_image: ${{steps.set_image.outputs.api_image}}
       migration_image: ${{steps.migration-publish.outputs.image}}
+      source_sha: ${{steps.source.outputs.sha}}
+      cda_version: ${{steps.version.outputs.version}}
     steps:
       - name: Clean up disk space, so we don't run out.
         if: matrix.platform == 'ubuntu-latest'
@@ -60,6 +62,9 @@ jobs:
         uses: actions/checkout@v5.0.0
         with:
           ref: ${{inputs.branch}}
+      - name: Record source revision
+        id: source
+        run: echo "sha=$(git rev-parse HEAD)" >> "$GITHUB_OUTPUT"
       - name: setup java
         uses: actions/setup-java@v5.2.0
         with:
@@ -165,3 +170,15 @@ jobs:
         if: ${{ always() }}
         run: |
           docker logout ${{ steps.login-ghcr.outputs.registry }}
+
+  sdk-docs:
+    needs: release
+    permissions:
+      contents: write
+      pages: write
+      id-token: write
+    uses: ./.github/workflows/sdk-docs.yml
+    with:
+      ref: ${{ needs.release.outputs.source_sha }}
+      cda-version: ${{ needs.release.outputs.cda_version }}
+      release: ${{ !inputs.nightly }}

.github/workflows/sdk-docs.yml

@@ -0,0 +1,148 @@
+name: SDK documentation
+
+on:
+  pull_request:
+    branches: [develop]
+    paths:
+      - 'clients/**'
+      - 'cwms-data-api/src/**'
+      - 'scripts/sdk-docs/**'
+      - '*.gradle'
+      - '.github/workflows/sdk-docs.yml'
+      - '.github/workflows/release.yml'
+  push:
+    branches: [develop]
+    paths:
+      - 'clients/**'
+      - 'cwms-data-api/src/**'
+      - 'scripts/sdk-docs/**'
+      - '*.gradle'
+      - '.github/workflows/sdk-docs.yml'
+  workflow_dispatch:
+  workflow_call:
+    inputs:
+      ref:
+        type: string
+        required: true
+      cda-version:
+        type: string
+        required: true
+      release:
+        type: boolean
+        default: false
+
+permissions:
+  contents: read
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    outputs:
+      version: ${{ steps.version.outputs.value }}
+    steps:
+      - uses: actions/checkout@v6
+        with:
+          ref: ${{ inputs.ref || github.ref }}
+      - uses: actions/setup-java@v5
+        with:
+          distribution: temurin
+          java-version: '11'
+          cache: gradle
+      - uses: actions/setup-node@v4
+        with:
+          node-version: '22'
+      - uses: actions/setup-python@v6
+        with:
+          python-version: '3.13'
+      - name: Select documentation version
+        id: version
+        env:
+          CDA_VERSION: ${{ inputs.cda-version }}
+        run: |
+          if [ -z "$CDA_VERSION" ]; then
+            CDA_VERSION="$(date -u +%Y.%m.%d)-snapshot"
+          fi
+          echo "value=$CDA_VERSION" >> "$GITHUB_OUTPUT"
+      - name: Test site assembly
+        run: python -m unittest discover -s scripts/sdk-docs/tests -v
+      - name: Build and test JavaScript SDK documentation
+        env:
+          CDA_VERSION: ${{ steps.version.outputs.value }}
+        run: ./gradlew :clients:typescript:build --init-script init.gradle "-PversionOverride=$CDA_VERSION"
+      - name: Build and test Python SDK documentation when present
+        env:
+          CDA_VERSION: ${{ steps.version.outputs.value }}
+        run: |
+          if [ -f clients/python/build.gradle ]; then
+            ./gradlew :clients:python:build --init-script init.gradle "-PversionOverride=$CDA_VERSION"
+          fi
+      - name: Assemble SDK documentation
+        env:
+          CDA_VERSION: ${{ steps.version.outputs.value }}
+        run: python scripts/sdk-docs/site.py stage . build/sdk-docs "$CDA_VERSION"
+      - uses: actions/upload-artifact@v4
+        with:
+          name: sdk-docs-${{ github.run_id }}-${{ github.run_attempt }}
+          path: build/sdk-docs
+          if-no-files-found: error
+
+  publish:
+    needs: build
+    if: github.event_name != 'pull_request' && github.repository == 'USACE/cwms-data-api'
+    runs-on: ubuntu-latest
+    concurrency:
+      group: sdk-pages
+      cancel-in-progress: false
+    permissions:
+      contents: write
+      pages: write
+      id-token: write
+    environment:
+      name: github-pages
+      url: ${{ steps.deployment.outputs.page_url }}
+    steps:
+      - uses: actions/checkout@v6
+        with:
+          ref: ${{ inputs.ref || github.ref }}
+      - uses: actions/download-artifact@v4
+        with:
+          name: sdk-docs-${{ github.run_id }}-${{ github.run_attempt }}
+          path: build/sdk-docs
+      - name: Restore documentation history
+        run: |
+          if git ls-remote --exit-code --heads origin sdk-pages; then
+            git fetch origin sdk-pages --depth=1
+            git worktree add site FETCH_HEAD
+          else
+            mkdir site
+            git -C site init
+            git -C site switch --orphan sdk-pages
+            git -C site config include.path "$GITHUB_WORKSPACE/.git/config"
+          fi
+      - name: Update documentation site
+        env:
+          CDA_VERSION: ${{ needs.build.outputs.version }}
+          IS_RELEASE: ${{ inputs.release }}
+        run: |
+          args=()
+          if [ "$IS_RELEASE" = true ]; then args+=(--release); fi
+          python scripts/sdk-docs/site.py publish build/sdk-docs site "$CDA_VERSION" "${args[@]}"
+      - name: Save documentation history
+        run: |
+          git -C site config user.name github-actions[bot]
+          git -C site config user.email 41898282+github-actions[bot]@users.noreply.github.com
+          git -C site add .
+          if ! git -C site diff --cached --quiet; then
+            git -C site commit -m "Update SDK documentation"
+            git -C site push origin HEAD:refs/heads/sdk-pages
+          fi
+      - uses: actions/configure-pages@v5
+      - name: Prepare Pages artifact
+        run: |
+          mkdir build/pages
+          rsync -a --exclude=.git site/ build/pages/
+      - uses: actions/upload-pages-artifact@v4
+        with:
+          path: build/pages
+      - uses: actions/deploy-pages@v4
+        id: deployment

.github/workflows/tagged-release.yml

@@ -10,6 +10,11 @@ on:
       - '[0-9][0-9][0-9][0-9].[0-9][0-9].[0-9][0-9]'
 jobs:
   release:
+    permissions:
+      contents: write
+      packages: write
+      pages: write
+      id-token: write
     uses: ./.github/workflows/release.yml
     secrets:
       token: ${{ secrets.GITHUB_TOKEN }}

clients/typescript/README.md

@@ -76,3 +76,17 @@ The Gradle build passes the CDA project version into the client package step. Wh
   `./gradlew :clients:typescript:build`
 
 All generated files (source, library, and docs) will be in `[repo]/clients/typescript/cwmsjs`
+
+## Hosted documentation
+
+The [CDA SDK site](https://usace.github.io/cwms-data-api/sdk/javascript/) hosts
+TypeDoc and the custom examples extracted from passing tests. CDA release builds
+publish matching versioned docs, while `develop` publishes separately under
+[/development/sdk/javascript/](https://usace.github.io/cwms-data-api/development/sdk/javascript/).
+These URLs become available after the SDK Pages workflow is deployed.
+
+The Gradle build runs the documentation examples before generating their pages.
+Run `:clients:typescript:testTypeScriptExamples` to test them directly. The
+private-server authorization template is excluded from public examples.
+See [the SDK Pages workflow guide](../../scripts/sdk-docs/README.md) for setup,
+release retention, and local validation.

clients/typescript/build.gradle

@@ -1,5 +1,4 @@
 import com.github.gradle.node.npm.task.NpmTask
-import com.github.gradle.node.npm.task.NpxTask
 import org.openapitools.generator.gradle.plugin.tasks.GenerateTask
 import org.openapitools.generator.gradle.plugin.tasks.ValidateTask
 
@@ -13,19 +12,25 @@ def npmCacheDir = layout.buildDirectory.dir("npm-cache")
 def rawSpecFile = layout.projectDirectory.file("cwms-swagger-raw.json")
 def modifiedSpecFile = layout.projectDirectory.file("cwms-swagger-mod.json")
 def generatedClientDir = layout.projectDirectory.dir("cwmsjs")
+def specOverride = providers.gradleProperty('typescriptOpenApiSpec')
+def sourceSpec = specOverride.isPresent() ? file(specOverride.get()) : rootProject.file('cwms-data-api/build/openapi.json')
 
 node {
     nodeProjectDir = projectDir
 }
 
-tasks.withType(NpmTask).configureEach {
-    environment.put("npm_config_cache", npmCacheDir.get().asFile.absolutePath)
+tasks.named('npmInstall') {
+    // The generator uses the JS scripts directly; legacy node-jq install hooks
+    // are unnecessary here. Generated package installation still runs scripts.
+    args = ['--ignore-scripts', '--no-audit']
 }
 
-tasks.withType(NpxTask).configureEach {
+tasks.withType(NpmTask).configureEach {
     environment.put("npm_config_cache", npmCacheDir.get().asFile.absolutePath)
+    environment.put("npm_config_audit", "false")
 }
 
+
 tasks.register('cleanGeneration', Delete) {
     delete project.layout.projectDirectory.dir('build')
     delete project.layout.projectDirectory.dir('dist')
@@ -40,23 +45,25 @@ clean.dependsOn cleanGeneration
 tasks.register('copyOpenApiSpec') {
     group 'openapi'
     description 'Copy the locally generated CDA OpenAPI spec into the TypeScript client workspace.'
-    dependsOn ':cwms-data-api:executeOpenAPIConversion'
+    if (!specOverride.isPresent()) {
+        dependsOn ':cwms-data-api:executeOpenAPIConversion'
+    }
+    inputs.file sourceSpec
     outputs.file rawSpecFile
     doLast {
         copy {
-            from "$rootDir/cwms-data-api/build/openapi.json"
+            from sourceSpec
             into projectDir
             rename { 'cwms-swagger-raw.json' }
         }
     }
 }
 
-tasks.register('modSpec', NpxTask) {
+tasks.register('modSpec', NpmTask) {
     group 'openapi'
     description 'Apply TypeScript client adjustments to the CDA OpenAPI spec.'
     dependsOn npmInstall
     dependsOn copyOpenApiSpec
-    command = 'npm'
     args = ['run', 'modSpec']
     inputs.file rawSpecFile
     inputs.files fileTree('scripts/spec-updates')
@@ -90,12 +97,11 @@ tasks.register('generateTypeScriptClient', GenerateTask) {
     ignoreFileOverride = layout.projectDirectory.file('.openapi-generator-ignore').asFile.absolutePath
 }
 
-tasks.register('modPackage', NpxTask) {
+tasks.register('modPackage', NpmTask) {
     group 'openapi'
     description 'Apply package metadata updates to the generated cwmsjs client.'
     dependsOn npmInstall
     dependsOn generateTypeScriptClient
-    command = 'npm'
     args = ['run', 'modPackage']
     environment.put('CDA_CLIENT_VERSION_SUFFIX', project.version.toString())
     inputs.file 'package.json'
@@ -104,53 +110,62 @@ tasks.register('modPackage', NpxTask) {
     outputs.file generatedClientDir.file('package.json')
 }
 
-tasks.register('postGenerate', NpxTask) {
+tasks.register('postGenerate', NpmTask) {
     group 'openapi'
     description 'Apply source patches required after OpenAPI generation.'
     dependsOn modPackage
-    command = 'npm'
     args = ['run', 'postGenerate']
     inputs.dir generatedClientDir
     inputs.files fileTree('scripts').include('postGenerate.js')
     outputs.dir generatedClientDir
 }
 
-tasks.register('installGeneratedClient', NpxTask) {
+tasks.register('installGeneratedClient', NpmTask) {
     group 'build'
     description 'Install dependencies for the generated cwmsjs client.'
     dependsOn postGenerate
     workingDir = generatedClientDir.asFile
-    command = 'npm'
     args = ['install']
     inputs.file generatedClientDir.file('package.json')
     outputs.dir generatedClientDir.dir('node_modules')
 }
 
-tasks.register('buildTypeScriptClient', NpxTask) {
+tasks.register('buildTypeScriptClient', NpmTask) {
     group 'build'
     description 'Compile the generated cwmsjs TypeScript client.'
     dependsOn installGeneratedClient
     workingDir = generatedClientDir.asFile
-    command = 'npm'
     args = ['run', 'build']
     inputs.dir generatedClientDir.dir('src')
     inputs.file generatedClientDir.file('package.json')
     inputs.file generatedClientDir.file('tsconfig.json')
     outputs.dir generatedClientDir.dir('dist')
 }
 
-tasks.register('buildTypeScriptDocs', NpxTask) {
+tasks.register('buildTypeScriptDocs', NpmTask) {
     group 'documentation'
     description 'Build TypeDoc and example documentation for the generated cwmsjs client.'
-    dependsOn buildTypeScriptClient
-    command = 'npm'
+    dependsOn 'testTypeScriptExamples'
     args = ['run', 'buildDocs']
     inputs.dir generatedClientDir.dir('src')
     inputs.dir 'tests'
-    inputs.files fileTree('scripts').include('buildTypedoc.js', 'tests2exampledocs.js', 'exampletemplate.html')
+    inputs.files fileTree('scripts').include('buildTypedoc.js', 'tests2exampledocs.js', 'exampleFiles.js', 'exampletemplate.html')
     outputs.dir generatedClientDir.dir('docs')
 }
 
+tasks.register('installExampleDependencies', NpmTask) {
+    workingDir = file('tests')
+    args = ['ci']
+}
+
+tasks.register('testTypeScriptExamples', NpmTask) {
+    group 'verification'
+    description 'Run the examples before publishing their source as documentation.'
+    dependsOn buildTypeScriptClient, installExampleDependencies
+    workingDir = file('tests')
+    args = ['run', 'test:docs']
+}
+
 tasks.named('build') {
     dependsOn buildTypeScriptClient
     dependsOn buildTypeScriptDocs

clients/typescript/scripts/exampleFiles.js

@@ -0,0 +1,16 @@
+const fs = require('node:fs');
+const path = require('node:path');
+const crypto = require('node:crypto');
+
+const root = path.resolve(__dirname, '..');
+function exampleFiles() {
+  return ['endpoints', 'generator'].flatMap(directory =>
+    fs.readdirSync(path.join(root, 'tests', directory))
+      .filter(name => name.endsWith('.test.js'))
+      .map(name => path.join('tests', directory, name))
+  ).filter(file => !fs.readFileSync(path.join(root, file), 'utf8').includes('//!ignore')).sort();
+}
+function digest(file) {
+  return crypto.createHash('sha256').update(fs.readFileSync(path.join(root, file))).digest('hex');
+}
+module.exports = { root, exampleFiles, digest };