diff --git a/.github/actions/save-build-caches/action.yml b/.github/actions/save-build-caches/action.yml new file mode 100644 index 000000000..c1d4d3e60 --- /dev/null +++ b/.github/actions/save-build-caches/action.yml @@ -0,0 +1,47 @@ +name: Save completed build work +description: Preserve compiler objects and uv wheels when later tests fail, including PR runs. +inputs: + rust-key: + description: Boxington primary key, empty when Rust caching is disabled. + default: '' + uv-key: + description: setup-uv primary key, empty when uv caching is disabled. + default: '' +runs: + using: composite + steps: + # The upstream action saves successful default-branch pushes. Preserve + # completed objects for failed jobs and PRs as well; GitHub scopes PR + # caches to the PR merge ref, so they cannot populate the trunk cache. + - name: Export completed Rust builds + id: rust + if: inputs.rust-key != '' && (github.event_name != 'push' || failure()) + shell: bash + run: | + archive="$(mbx cache dir)/github-actions-cache-v1.tar" + if mbx cache export --group "$MBX_CACHE_EXPORT_GROUP" "$archive" > "$RUNNER_TEMP/mbx-export.log" 2>&1; then + echo "archive=$archive" >> "$GITHUB_OUTPUT" + elif grep -q 'no completed mbx builds are recorded for export group' "$RUNNER_TEMP/mbx-export.log"; then + cat "$RUNNER_TEMP/mbx-export.log" + else + cat "$RUNNER_TEMP/mbx-export.log" >&2 + exit 1 + fi + + - uses: actions/cache/save@v4 + if: ${{ !cancelled() && steps.rust.outputs.archive != '' }} + with: + path: ${{ steps.rust.outputs.archive }} + key: ${{ inputs.rust-key }}-${{ github.sha }} + + # setup-uv saves successful jobs itself, but its post step skips failures. + - name: Prune uv cache after failure + if: inputs.uv-key != '' && failure() + shell: bash + run: uv cache prune --ci + + - uses: actions/cache/save@v4 + if: inputs.uv-key != '' && failure() + with: + path: ${{ runner.temp }}/uv-cache + key: ${{ inputs.uv-key }} diff --git a/.github/actions/setup-rust-cache/action.yml b/.github/actions/setup-rust-cache/action.yml new file mode 100644 index 000000000..800e24a8b --- /dev/null +++ b/.github/actions/setup-rust-cache/action.yml @@ -0,0 +1,76 @@ +name: Cache Rust compilation +description: Share completed compiler objects across Cargo, uv, maturin, and Make builds. +inputs: + suffix: + description: Job and matrix partition for the compiler cache. + required: true +outputs: + key: + description: Key used by the Boxington action. + value: ${{ steps.cache.outputs.cache-primary-key }} +runs: + using: composite + steps: + # Upstream 1.11.0 publishes ARM macOS binaries, but no Intel macOS binary. + # Bootstrap that host once, then reuse the pinned executable across jobs. + - uses: actions/cache/restore@v4 + if: runner.os == 'macOS' && runner.arch == 'X64' + id: intel + with: + path: ${{ runner.temp }}/mbx-intel/bin/mbx + key: mbx-1.11.0-macos-x64-v1 + + - name: Build Boxington for Intel macOS + if: runner.os == 'macOS' && runner.arch == 'X64' && steps.intel.outputs.cache-hit != 'true' + shell: bash + run: cargo install --git https://github.com/jdx/mr-boxington --rev 4ebd2af1eb7e70fee37ec8678c30c0619f5ed5af --locked --root "$RUNNER_TEMP/mbx-intel" mbx + + - uses: actions/cache/save@v4 + if: runner.os == 'macOS' && runner.arch == 'X64' && steps.intel.outputs.cache-hit != 'true' + with: + path: ${{ runner.temp }}/mbx-intel/bin/mbx + key: mbx-1.11.0-macos-x64-v1 + + - name: Enable pinned Intel Boxington + if: runner.os == 'macOS' && runner.arch == 'X64' + shell: bash + run: | + test "$("$RUNNER_TEMP/mbx-intel/bin/mbx" --version)" = 'mbx 1.11.0' + echo "$RUNNER_TEMP/mbx-intel/bin" >> "$GITHUB_PATH" + + - uses: jdx/mr-boxington-action@a20e1ffcd962370fb2b6045c13b7b349f7b03386 + id: cache + env: + MBX_CACHE_DIR: ${{ github.workspace }}/.ci-mbx + MBX_TARGET_VIEWS: '0' + with: + version: ${{ (runner.os != 'macOS' || runner.arch != 'X64') && '1.11.0' || '' }} + github-cache-mode: objects + cache-generation: sidemantic-v1-${{ inputs.suffix }} + + - name: Remove imported Rust cache archive + env: + MBX_CACHE_DIR: ${{ github.workspace }}/.ci-mbx + shell: bash + run: | + # The pinned action imports this transport archive but leaves it on disk. + # Export writes a new full-size temporary tar before replacing this path; + # keep the imported objects and targets, not a second copy of their bytes. + rm -f -- "$(mbx cache dir)/github-actions-cache-v1.tar" + + - name: Enable Cargo subprocess caching + shell: bash + run: | + # mbx dispatches as a Cargo shim when invoked under Cargo's filename. + # Keep the shim in the workspace so maturin's workspace mount can use it too. + shim_dir="$GITHUB_WORKSPACE/.ci-cache-bin" + mkdir -p "$shim_dir" + if [ "$RUNNER_OS" = Windows ]; then + cp "$(command -v mbx)" "$shim_dir/cargo.exe" + else + cp "$(command -v mbx)" "$shim_dir/cargo" + chmod +x "$shim_dir/cargo" + fi + echo "$shim_dir" >> "$GITHUB_PATH" + echo "MBX_CACHE_DIR=$GITHUB_WORKSPACE/.ci-mbx" >> "$GITHUB_ENV" + echo "MBX_TARGET_VIEWS=0" >> "$GITHUB_ENV" diff --git a/.github/actions/setup-uv/action.yml b/.github/actions/setup-uv/action.yml new file mode 100644 index 000000000..434a2c7a0 --- /dev/null +++ b/.github/actions/setup-uv/action.yml @@ -0,0 +1,41 @@ +name: Set up uv with native source caching +description: Cache downloaded dependencies and built wheels, including their Rust build inputs. +inputs: + enable-cache: + description: Disable in production release jobs. + default: 'true' + suffix: + description: Job and matrix partition for the dependency cache. + required: true +outputs: + cache-key: + description: Key used by setup-uv. + value: ${{ steps.uv.outputs.cache-key }} +runs: + using: composite + steps: + - uses: astral-sh/setup-uv@v7 + id: uv + with: + version: '0.12.14' + enable-cache: ${{ inputs.enable-cache }} + cache-local-path: ${{ runner.temp }}/uv-cache + cache-suffix: ${{ inputs.suffix }} + cache-dependency-glob: | + uv.lock + pyproject.toml + sidemantic-rs/pyproject.toml + crates/dax-pyo3/pyproject.toml + crates/dax-pyo3/README.md + .cargo/config.toml + rust-toolchain* + Cargo.lock + Cargo.toml + sidemantic-rs/Cargo.toml + sidemantic-rs/build.rs + sidemantic-rs/src/** + crates/*/Cargo.toml + crates/*/src/** + crates/dax-pyo3/python/**/*.py + crates/dax-pyo3/python/**/*.pyi + crates/dax-pyo3/python/**/py.typed diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0f570ce83..a37423399 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -12,14 +12,19 @@ concurrency: jobs: python: - name: Python ${{ matrix.python-version }} + name: Python ${{ matrix.python-version }} (${{ matrix.engine }} engine) runs-on: ubuntu-latest env: UV_PYTHON: ${{ matrix.python-version }} + SIDEMANTIC_ENGINE: ${{ matrix.engine }} strategy: fail-fast: false matrix: python-version: ["3.11", "3.12", "3.13", "3.14"] + engine: [python] + include: + - python-version: "3.12" + engine: rust steps: - uses: actions/checkout@v4 @@ -27,10 +32,18 @@ jobs: - name: Install Rust uses: dtolnay/rust-toolchain@stable + - name: Cache Rust compilation + id: rust-cache + uses: ./.github/actions/setup-rust-cache + with: + suffix: ${{ github.workflow }}-${{ github.job }}-${{ strategy.job-index || 0 }} + - name: Install uv - uses: astral-sh/setup-uv@v5 + id: uv + uses: ./.github/actions/setup-uv with: enable-cache: true + suffix: ${{ github.workflow }}-${{ github.job }}-${{ strategy.job-index || 0 }} - name: Set up Python ${{ matrix.python-version }} run: uv python install ${{ matrix.python-version }} @@ -83,11 +96,24 @@ jobs: uv build --wheel unzip -l dist/*.whl | grep -q 'sidemantic/py.typed' - - name: Run tests - run: uv run pytest -v --cov-fail-under=76 + - name: Run Python reference suite + if: matrix.engine == 'python' + run: uv run --with adbc-driver-sqlite pytest -v --test-engine python --cov-fail-under=76 --junitxml=test-results.xml + + - name: Run full shared suite against Rust + if: matrix.engine == 'rust' + run: uv run --with adbc-driver-sqlite pytest -v --test-engine rust --no-cov --junitxml=test-results.xml + + - name: Upload test results + if: always() + uses: actions/upload-artifact@v4 + with: + name: tests-${{ matrix.python-version }}-${{ matrix.engine }} + path: test-results.xml - name: Run installed-wheel DAX smoke run: | + uv build sidemantic-rs --out-dir /tmp/sidemantic-dax-dist uv build crates/dax-pyo3 --out-dir /tmp/sidemantic-dax-dist uv build --out-dir /tmp/sidemantic-dist SIDEMANTIC_WHEEL=$(realpath "$(find /tmp/sidemantic-dist -name 'sidemantic-[0-9]*.whl' -print -quit)") @@ -98,6 +124,13 @@ jobs: sidemantic_dax.parse_expression("1") PY + - name: Save completed build caches + if: ${{ !cancelled() }} + uses: ./.github/actions/save-build-caches + with: + rust-key: ${{ steps.rust-cache.outputs.key }} + uv-key: ${{ steps.uv.outputs.cache-key }} + dependency-compatibility: name: Dependencies (${{ matrix.name }}) runs-on: ubuntu-latest @@ -119,10 +152,18 @@ jobs: - name: Install Rust uses: dtolnay/rust-toolchain@stable + - name: Cache Rust compilation + id: rust-cache + uses: ./.github/actions/setup-rust-cache + with: + suffix: ${{ github.workflow }}-${{ github.job }}-${{ strategy.job-index || 0 }} + - name: Install uv - uses: astral-sh/setup-uv@v5 + id: uv + uses: ./.github/actions/setup-uv with: enable-cache: true + suffix: ${{ github.workflow }}-${{ github.job }}-${{ strategy.job-index || 0 }} - name: Set up Python ${{ matrix.python-version }} run: uv python install ${{ matrix.python-version }} @@ -151,20 +192,41 @@ jobs: -q -o addopts="-m 'not integration'" + - name: Save completed build caches + if: ${{ !cancelled() }} + uses: ./.github/actions/save-build-caches + with: + rust-key: ${{ steps.rust-cache.outputs.key }} + uv-key: ${{ steps.uv.outputs.cache-key }} + base-install-cli: name: Base Install CLI runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 + - name: Install Rust + uses: dtolnay/rust-toolchain@stable + + - name: Cache Rust compilation + id: rust-cache + uses: ./.github/actions/setup-rust-cache + with: + suffix: ${{ github.workflow }}-${{ github.job }}-${{ strategy.job-index || 0 }} + - name: Install uv - uses: astral-sh/setup-uv@v5 + id: uv + uses: ./.github/actions/setup-uv with: enable-cache: true + suffix: ${{ github.workflow }}-${{ github.job }}-${{ strategy.job-index || 0 }} - name: Set up Python run: uv python install 3.12 + - name: Build runtime distribution + run: uv build sidemantic-rs --out-dir /tmp/sidemantic-runtime-dist + - name: Smoke base CLI install run: | set -euo pipefail @@ -183,12 +245,20 @@ jobs: agg: count YAML - uv run --no-project --with . sidemantic --version - uv run --no-project --with . sidemantic validate "$tmpdir/models" - uv run --no-project --with . sidemantic query "SELECT order_count, status FROM orders" --models "$tmpdir/models" --dry-run + uv run --no-project --find-links /tmp/sidemantic-runtime-dist --with . sidemantic --version + uv run --no-project --find-links /tmp/sidemantic-runtime-dist --with . sidemantic validate "$tmpdir/models" + uv run --no-project --find-links /tmp/sidemantic-runtime-dist --with . sidemantic query "SELECT order_count, status FROM orders" --models "$tmpdir/models" --dry-run + + uv run --no-project --find-links /tmp/sidemantic-runtime-dist --with . python - <<'PYTHON' + from sidemantic import SemanticLayer, Model, Metric + layer = SemanticLayer() + layer.add_model(Model(name="orders", table="orders", metrics=[Metric(name="count", agg="count")])) + layer.compile(metrics=["orders.count"]) + assert layer.last_engine_selection == {"engine": "rust", "reason": None} + PYTHON set +e - timeout 10s uv run --no-project --with . sidemantic serve "$tmpdir/models" >"$tmpdir/serve.out" 2>"$tmpdir/serve.err" + timeout 10s uv run --no-project --find-links /tmp/sidemantic-runtime-dist --with . sidemantic serve "$tmpdir/models" >"$tmpdir/serve.out" 2>"$tmpdir/serve.err" serve_status=$? set -e if [ "$serve_status" -eq 0 ]; then @@ -203,6 +273,13 @@ jobs: fi grep -q "sidemantic\\[api\\]" "$tmpdir/serve.err" + - name: Save completed build caches + if: ${{ !cancelled() }} + uses: ./.github/actions/save-build-caches + with: + rust-key: ${{ steps.rust-cache.outputs.key }} + uv-key: ${{ steps.uv.outputs.cache-key }} + update-schema: name: Update JSON Schema needs: python @@ -215,17 +292,25 @@ jobs: with: ref: ${{ github.head_ref }} + - name: Install Rust + uses: dtolnay/rust-toolchain@stable + + - name: Cache Rust compilation + id: rust-cache + uses: ./.github/actions/setup-rust-cache + with: + suffix: ${{ github.workflow }}-${{ github.job }}-${{ strategy.job-index || 0 }} + - name: Install uv - uses: astral-sh/setup-uv@v5 + id: uv + uses: ./.github/actions/setup-uv with: enable-cache: true + suffix: ${{ github.workflow }}-${{ github.job }}-${{ strategy.job-index || 0 }} - name: Set up Python run: uv python install 3.12 - - name: Install Rust - uses: dtolnay/rust-toolchain@stable - - name: Install dependencies run: uv sync --extra dev --extra dax @@ -240,16 +325,34 @@ jobs: git push fi + - name: Save completed build caches + if: ${{ !cancelled() }} + uses: ./.github/actions/save-build-caches + with: + rust-key: ${{ steps.rust-cache.outputs.key }} + uv-key: ${{ steps.uv.outputs.cache-key }} + native-compat: name: Native Compatibility runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 + - name: Install Rust + uses: dtolnay/rust-toolchain@stable + + - name: Cache Rust compilation + id: rust-cache + uses: ./.github/actions/setup-rust-cache + with: + suffix: ${{ github.workflow }}-${{ github.job }}-${{ strategy.job-index || 0 }} + - name: Install uv - uses: astral-sh/setup-uv@v5 + id: uv + uses: ./.github/actions/setup-uv with: enable-cache: true + suffix: ${{ github.workflow }}-${{ github.job }}-${{ strategy.job-index || 0 }} - name: Set up Python run: uv python install 3.12 @@ -257,14 +360,6 @@ jobs: - name: Install dependencies run: uv sync --extra dev - - name: Install Rust - uses: dtolnay/rust-toolchain@stable - - - name: Cache cargo - uses: Swatinem/rust-cache@v2 - with: - workspaces: sidemantic-rs - - name: Run Python native fixtures run: uv run pytest tests/native_compat -v @@ -326,6 +421,13 @@ jobs: if-no-files-found: warn retention-days: 7 + - name: Save completed build caches + if: ${{ !cancelled() }} + uses: ./.github/actions/save-build-caches + with: + rust-key: ${{ steps.rust-cache.outputs.key }} + uv-key: ${{ steps.uv.outputs.cache-key }} + check-rust-changes: name: Check Rust/DuckDB changes runs-on: ubuntu-latest @@ -348,6 +450,7 @@ jobs: - '.cargo/**' - 'rust-toolchain*' - '.github/workflows/ci.yml' + - '.github/actions/**' # The installed-extension conformance job exercises Python callers, # not just the crate. Keep its contract and fixture inputs gated. - 'pyproject.toml' @@ -395,9 +498,12 @@ jobs: - '.cargo/**' - 'rust-toolchain*' - '.github/workflows/ci.yml' + - '.github/actions/**' - 'pyproject.toml' - 'uv.lock' dax: + - '.github/actions/**' + - '.github/workflows/ci.yml' - 'crates/**' - 'Cargo.toml' - 'Cargo.lock' @@ -428,10 +534,11 @@ jobs: - name: Install Rust uses: dtolnay/rust-toolchain@stable - - name: Cache cargo - uses: Swatinem/rust-cache@v2 + - name: Cache Rust compilation + id: rust-cache + uses: ./.github/actions/setup-rust-cache with: - workspaces: . + suffix: ${{ github.workflow }}-${{ github.job }}-${{ strategy.job-index || 0 }} - name: Run cargo fmt check run: cargo fmt --check @@ -442,11 +549,23 @@ jobs: - name: Run cargo test run: cargo test -p dax-parser -p dax-pyo3 + - name: Save completed build caches + if: ${{ !cancelled() }} + uses: ./.github/actions/save-build-caches + with: + rust-key: ${{ steps.rust-cache.outputs.key }} + rust: name: Rust (sidemantic-rs) needs: check-rust-changes if: needs.check-rust-changes.outputs.rust_changed == 'true' runs-on: ubuntu-latest + # This job builds many feature/host combinations. Full debug symbols make + # the completed-build cache too large to duplicate into its export tar. + # Keep debug assertions and unoptimized tests, but omit symbol data. + env: + CARGO_PROFILE_DEV_DEBUG: '0' + CARGO_PROFILE_TEST_DEBUG: '0' services: postgres: image: postgres:16 @@ -473,15 +592,18 @@ jobs: with: targets: wasm32-unknown-unknown - - name: Install uv - uses: astral-sh/setup-uv@v5 + - name: Cache Rust compilation + id: rust-cache + uses: ./.github/actions/setup-rust-cache with: - enable-cache: true + suffix: ${{ github.workflow }}-${{ github.job }}-${{ strategy.job-index || 0 }}-no-debuginfo - - name: Cache cargo - uses: Swatinem/rust-cache@v2 + - name: Install uv + id: uv + uses: ./.github/actions/setup-uv with: - workspaces: sidemantic-rs + enable-cache: true + suffix: ${{ github.workflow }}-${{ github.job }}-${{ strategy.job-index || 0 }} - name: Run cargo fmt check run: cargo fmt --check @@ -572,6 +694,8 @@ jobs: run: uv run --no-project --with dist/*.whl tests/python_wheel_smoke.py - name: Check lightweight Python extension build + env: + MBX_VERIFY: "1" run: uvx maturin build --no-default-features --features python --out dist-python - name: Smoke lightweight Python extension wheel @@ -598,6 +722,13 @@ jobs: - name: Smoke Python extension ADBC wheel run: uv run --no-project --with dist-adbc/*.whl tests/python_wheel_adbc_smoke.py + - name: Save completed build caches + if: ${{ !cancelled() }} + uses: ./.github/actions/save-build-caches + with: + rust-key: ${{ steps.rust-cache.outputs.key }} + uv-key: ${{ steps.uv.outputs.cache-key }} + duckdb-extension: name: DuckDB Extension needs: check-rust-changes @@ -609,17 +740,18 @@ jobs: steps: - uses: actions/checkout@v4 - - name: Fetch DuckDB dependencies - working-directory: sidemantic-duckdb - run: make deps DUCKDB_VERSION=v1.5.5 - - name: Install Rust uses: dtolnay/rust-toolchain@stable - - name: Cache cargo - uses: Swatinem/rust-cache@v2 + - name: Cache Rust compilation + id: rust-cache + uses: ./.github/actions/setup-rust-cache with: - workspaces: sidemantic-rs + suffix: ${{ github.workflow }}-${{ github.job }}-${{ strategy.job-index || 0 }} + + - name: Fetch DuckDB dependencies + working-directory: sidemantic-duckdb + run: make deps DUCKDB_VERSION=v1.5.5 - name: Install build dependencies run: sudo apt-get update && sudo apt-get install -y ninja-build @@ -633,7 +765,11 @@ jobs: run: make test - name: Install uv for SQL host acceptance - uses: astral-sh/setup-uv@v5 + id: uv + uses: ./.github/actions/setup-uv + with: + enable-cache: true + suffix: ${{ github.workflow }}-${{ github.job }}-${{ strategy.job-index || 0 }} - name: Execute SemanticInput in the loaded DuckDB extension env: @@ -641,6 +777,13 @@ jobs: SIDEMANTIC_DUCKDB_EXTENSION: sidemantic-duckdb/build/release/extension/sidemantic/sidemantic.duckdb_extension run: uv run sidemantic-duckdb/test/test_semantic_input_host.py + - name: Save completed build caches + if: ${{ !cancelled() }} + uses: ./.github/actions/save-build-caches + with: + rust-key: ${{ steps.rust-cache.outputs.key }} + uv-key: ${{ steps.uv.outputs.cache-key }} + webapp: name: Webapp (types + unit) needs: check-rust-changes diff --git a/.github/workflows/duckdb-extension-release.yml b/.github/workflows/duckdb-extension-release.yml index 695837f79..a59741b72 100644 --- a/.github/workflows/duckdb-extension-release.yml +++ b/.github/workflows/duckdb-extension-release.yml @@ -19,6 +19,7 @@ on: pull_request: paths: + - ".github/actions/**" - ".github/workflows/duckdb-extension-release.yml" permissions: @@ -35,6 +36,16 @@ jobs: steps: - uses: actions/checkout@v4 + - name: Install Rust + uses: dtolnay/rust-toolchain@stable + + - name: Cache Rust compilation + if: github.event_name == 'pull_request' + id: rust-cache + uses: ./.github/actions/setup-rust-cache + with: + suffix: ${{ github.workflow }}-${{ github.job }}-${{ strategy.job-index || 0 }} + - name: Resolve DuckDB version id: duckdb env: @@ -61,9 +72,6 @@ jobs: DUCKDB_VERSION: ${{ steps.duckdb.outputs.version }} run: make deps DUCKDB_VERSION="$DUCKDB_VERSION" - - name: Install Rust - uses: dtolnay/rust-toolchain@stable - - name: Install build dependencies run: sudo apt-get update && sudo apt-get install -y ninja-build @@ -91,6 +99,12 @@ jobs: path: sidemantic-duckdb/dist/*.duckdb_extension if-no-files-found: error + - name: Save completed build caches + if: ${{ !cancelled() && github.event_name == 'pull_request' }} + uses: ./.github/actions/save-build-caches + with: + rust-key: ${{ steps.rust-cache.outputs.key }} + github-release: name: Attach extension artifact to GitHub release needs: build diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index 9ab015ab5..f610aa66b 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -33,10 +33,21 @@ jobs: steps: - uses: actions/checkout@v4 + - name: Install Rust + uses: dtolnay/rust-toolchain@stable + + - name: Cache Rust compilation + id: rust-cache + uses: ./.github/actions/setup-rust-cache + with: + suffix: ${{ github.workflow }}-${{ github.job }}-${{ strategy.job-index || 0 }} + - name: Install uv - uses: astral-sh/setup-uv@v5 + id: uv + uses: ./.github/actions/setup-uv with: enable-cache: true + suffix: ${{ github.workflow }}-${{ github.job }}-${{ strategy.job-index || 0 }} - name: Set up Python run: uv python install 3.12 @@ -53,7 +64,9 @@ jobs: POSTGRES_DB: "sidemantic_test" POSTGRES_USER: "test" POSTGRES_PASSWORD: "test" - run: uv run pytest -m integration tests/db/test_postgres_integration.py -v + run: | + uv run pytest --test-engine python -m integration tests/db/test_postgres_integration.py -v + uv run pytest --test-engine rust -m integration tests/db/test_postgres_integration.py -v - name: Run PostgreSQL CLI e2e test env: @@ -63,7 +76,16 @@ jobs: POSTGRES_DB: "sidemantic_test" POSTGRES_USER: "test" POSTGRES_PASSWORD: "test" - run: uv run pytest -m integration tests/db/test_postgres_cli_e2e.py -v + run: | + uv run pytest --test-engine python -m integration tests/db/test_postgres_cli_e2e.py -v + uv run pytest --test-engine rust -m integration tests/db/test_postgres_cli_e2e.py -v + + - name: Save completed build caches + if: ${{ !cancelled() }} + uses: ./.github/actions/save-build-caches + with: + rust-key: ${{ steps.rust-cache.outputs.key }} + uv-key: ${{ steps.uv.outputs.cache-key }} bigquery-integration: runs-on: ubuntu-latest @@ -71,6 +93,15 @@ jobs: steps: - uses: actions/checkout@v4 + - name: Install Rust + uses: dtolnay/rust-toolchain@stable + + - name: Cache Rust compilation + id: rust-cache + uses: ./.github/actions/setup-rust-cache + with: + suffix: ${{ github.workflow }}-${{ github.job }}-${{ strategy.job-index || 0 }} + - name: Start BigQuery emulator run: | docker run -d --name bigquery-emulator \ @@ -82,9 +113,11 @@ jobs: sleep 5 - name: Install uv - uses: astral-sh/setup-uv@v5 + id: uv + uses: ./.github/actions/setup-uv with: enable-cache: true + suffix: ${{ github.workflow }}-${{ github.job }}-${{ strategy.job-index || 0 }} - name: Set up Python run: uv python install 3.12 @@ -98,7 +131,16 @@ jobs: BIGQUERY_EMULATOR_HOST: "localhost:9050" BIGQUERY_PROJECT: "test-project" BIGQUERY_DATASET: "test_dataset" - run: uv run pytest -m integration tests/db/test_bigquery_integration.py -v + run: | + uv run pytest --test-engine python -m integration tests/db/test_bigquery_integration.py -v + uv run pytest --test-engine rust -m integration tests/db/test_bigquery_integration.py -v + + - name: Save completed build caches + if: ${{ !cancelled() }} + uses: ./.github/actions/save-build-caches + with: + rust-key: ${{ steps.rust-cache.outputs.key }} + uv-key: ${{ steps.uv.outputs.cache-key }} snowflake-integration: runs-on: ubuntu-latest @@ -106,10 +148,21 @@ jobs: steps: - uses: actions/checkout@v4 + - name: Install Rust + uses: dtolnay/rust-toolchain@stable + + - name: Cache Rust compilation + id: rust-cache + uses: ./.github/actions/setup-rust-cache + with: + suffix: ${{ github.workflow }}-${{ github.job }}-${{ strategy.job-index || 0 }} + - name: Install uv - uses: astral-sh/setup-uv@v5 + id: uv + uses: ./.github/actions/setup-uv with: enable-cache: true + suffix: ${{ github.workflow }}-${{ github.job }}-${{ strategy.job-index || 0 }} - name: Set up Python run: uv python install 3.12 @@ -123,7 +176,16 @@ jobs: - name: Run Snowflake integration tests env: SNOWFLAKE_TEST: "1" - run: uv run pytest -m integration tests/db/test_snowflake_integration.py -v + run: | + uv run pytest --test-engine python -m integration tests/db/test_snowflake_integration.py -v + uv run pytest --test-engine rust -m integration tests/db/test_snowflake_integration.py -v + + - name: Save completed build caches + if: ${{ !cancelled() }} + uses: ./.github/actions/save-build-caches + with: + rust-key: ${{ steps.rust-cache.outputs.key }} + uv-key: ${{ steps.uv.outputs.cache-key }} clickhouse-integration: runs-on: ubuntu-latest @@ -147,10 +209,21 @@ jobs: steps: - uses: actions/checkout@v4 + - name: Install Rust + uses: dtolnay/rust-toolchain@stable + + - name: Cache Rust compilation + id: rust-cache + uses: ./.github/actions/setup-rust-cache + with: + suffix: ${{ github.workflow }}-${{ github.job }}-${{ strategy.job-index || 0 }} + - name: Install uv - uses: astral-sh/setup-uv@v5 + id: uv + uses: ./.github/actions/setup-uv with: enable-cache: true + suffix: ${{ github.workflow }}-${{ github.job }}-${{ strategy.job-index || 0 }} - name: Set up Python run: uv python install 3.12 @@ -164,7 +237,16 @@ jobs: CLICKHOUSE_HOST: "localhost" CLICKHOUSE_PORT: "8123" CLICKHOUSE_PASSWORD: "clickhouse" - run: uv run pytest -m integration tests/db/test_clickhouse_integration.py -v + run: | + uv run pytest --test-engine python -m integration tests/db/test_clickhouse_integration.py -v + uv run pytest --test-engine rust -m integration tests/db/test_clickhouse_integration.py -v + + - name: Save completed build caches + if: ${{ !cancelled() }} + uses: ./.github/actions/save-build-caches + with: + rust-key: ${{ steps.rust-cache.outputs.key }} + uv-key: ${{ steps.uv.outputs.cache-key }} adbc-integration: name: ADBC integration (${{ matrix.db }}) @@ -205,6 +287,15 @@ jobs: steps: - uses: actions/checkout@v4 + - name: Install Rust for Rust ADBC probe + uses: dtolnay/rust-toolchain@stable + + - name: Cache Rust compilation + id: rust-cache + uses: ./.github/actions/setup-rust-cache + with: + suffix: ${{ github.workflow }}-${{ github.job }}-${{ strategy.job-index || 0 }} + - name: Start BigQuery emulator if: matrix.db == 'bigquery' run: | @@ -215,9 +306,11 @@ jobs: sleep 5 - name: Install uv - uses: astral-sh/setup-uv@v5 + id: uv + uses: ./.github/actions/setup-uv with: enable-cache: true + suffix: ${{ github.workflow }}-${{ github.job }}-${{ strategy.job-index || 0 }} - name: Set up Python run: uv python install 3.12 @@ -255,10 +348,9 @@ jobs: CLICKHOUSE_HOST: "localhost" CLICKHOUSE_PORT: "8123" SNOWFLAKE_TEST: "1" - run: uv run pytest -m integration tests/db/test_adbc_ci_smoke.py -v - - - name: Install Rust for Rust ADBC probe - uses: dtolnay/rust-toolchain@stable + run: | + uv run pytest --test-engine python -m integration tests/db/test_adbc_ci_smoke.py -v + uv run pytest --test-engine rust -m integration tests/db/test_adbc_ci_smoke.py -v - name: Export SQLite ADBC driver for Rust if: matrix.db == 'sqlite' @@ -335,3 +427,10 @@ jobs: - name: Run Rust ADBC probe run: cargo test --manifest-path sidemantic-rs/Cargo.toml --features adbc-exec --test adbc_driver_matrix + + - name: Save completed build caches + if: ${{ !cancelled() }} + uses: ./.github/actions/save-build-caches + with: + rust-key: ${{ steps.rust-cache.outputs.key }} + uv-key: ${{ steps.uv.outputs.cache-key }} diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index c4afe4307..70593f856 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -39,6 +39,8 @@ jobs: - name: Install uv uses: astral-sh/setup-uv@v5 + with: + enable-cache: false - name: Set up Python run: uv python install 3.12 @@ -125,6 +127,7 @@ jobs: version_files = ( Path("pyproject.toml"), Path("crates/dax-pyo3/pyproject.toml"), + Path("sidemantic-rs/pyproject.toml"), Path("crates/dax-pyo3/Cargo.toml"), ) for path in version_files: @@ -152,6 +155,7 @@ jobs: pyproject_path.write_text( "\n".join( f' "sidemantic-dax>={version}",' if line.strip().startswith('"sidemantic-dax>=') + else f' "sidemantic-rs=={version}; sys_platform != \'emscripten\'",' if line.strip().startswith('"sidemantic-rs==') else line for line in pyproject_lines ) @@ -180,6 +184,7 @@ jobs: - name: Build packages run: | + uv build sidemantic-rs --sdist --out-dir dist uv build crates/dax-pyo3 --sdist --out-dir dist uv build --out-dir dist unzip -l "$(find dist -name 'sidemantic-[0-9]*.whl' -print -quit)" | grep -q 'sidemantic/py.typed' @@ -200,7 +205,7 @@ jobs: run: | git config user.name "github-actions[bot]" git config user.email "github-actions[bot]@users.noreply.github.com" - git add pyproject.toml sidemantic/__init__.py sidemantic/man/sidemantic.1 crates/dax-pyo3/pyproject.toml crates/dax-pyo3/Cargo.toml Cargo.lock uv.lock + git add pyproject.toml sidemantic/__init__.py sidemantic/man/sidemantic.1 sidemantic-rs/pyproject.toml crates/dax-pyo3/pyproject.toml crates/dax-pyo3/Cargo.toml Cargo.lock uv.lock git commit -m "Bump version to ${{ steps.version.outputs.new_version }}" git tag -a "v${{ steps.version.outputs.new_version }}" -m "Release v${{ steps.version.outputs.new_version }}" git push origin main @@ -257,11 +262,65 @@ jobs: path: crates/dax-pyo3/dist/*.whl if-no-files-found: error + build-runtime-wheels: + name: Build sidemantic-rs wheel (${{ matrix.target }}) + needs: publish + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + include: + - os: ubuntu-latest + target: x86_64 + - os: ubuntu-24.04-arm + target: aarch64 + - os: macos-15-intel + target: x86_64 + - os: macos-14 + target: aarch64 + - os: windows-latest + target: x86_64-pc-windows-msvc + + steps: + - uses: actions/checkout@v4 + with: + ref: ${{ needs.publish.outputs.tag }} + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3.12" + + - name: Build abi3 wheel + uses: PyO3/maturin-action@v1 + with: + command: build + args: --release --manifest-path sidemantic-rs/Cargo.toml --out sidemantic-rs/dist + manylinux: auto + target: ${{ matrix.target }} + + - name: Install uv + uses: astral-sh/setup-uv@v5 + with: + enable-cache: false + + - name: Smoke runtime wheel + shell: bash + run: uv run --no-project --with sidemantic-rs/dist/*.whl sidemantic-rs/tests/python_wheel_python_smoke.py + + - name: Upload abi3 wheel + uses: actions/upload-artifact@v4 + with: + name: dist-runtime-${{ matrix.os }}-${{ matrix.target }} + path: sidemantic-rs/dist/*.whl + if-no-files-found: error + release: name: Publish to PyPI and create release needs: - publish - build-dax-wheels + - build-runtime-wheels runs-on: ubuntu-latest permissions: id-token: write @@ -270,6 +329,8 @@ jobs: steps: - name: Install uv uses: astral-sh/setup-uv@v5 + with: + enable-cache: false - name: Download distributions uses: actions/download-artifact@v4 @@ -281,7 +342,10 @@ jobs: - name: Publish to PyPI env: UV_PUBLISH_TOKEN: ${{ secrets.PYPI_TOKEN }} - run: uv publish dist/* --token $UV_PUBLISH_TOKEN + run: | + uv publish dist/sidemantic_rs-* --token "$UV_PUBLISH_TOKEN" + uv publish dist/sidemantic_dax-* --token "$UV_PUBLISH_TOKEN" + uv publish dist/sidemantic-[0-9]* --token "$UV_PUBLISH_TOKEN" - name: Create GitHub Release env: diff --git a/.github/workflows/pyodide-test.yml b/.github/workflows/pyodide-test.yml index aff048ee2..725753a5e 100644 --- a/.github/workflows/pyodide-test.yml +++ b/.github/workflows/pyodide-test.yml @@ -17,7 +17,11 @@ jobs: node-version: '20' - name: Install uv - uses: astral-sh/setup-uv@v5 + id: uv + uses: ./.github/actions/setup-uv + with: + enable-cache: true + suffix: ${{ github.workflow }}-${{ github.job }}-${{ strategy.job-index || 0 }} - name: Build sidemantic wheel run: uv build @@ -129,3 +133,9 @@ jobs: }); EOF node test_pyodide.mjs + + - name: Save completed build caches + if: ${{ !cancelled() }} + uses: ./.github/actions/save-build-caches + with: + uv-key: ${{ steps.uv.outputs.cache-key }} diff --git a/.github/workflows/rust-runtime-release.yml b/.github/workflows/rust-runtime-release.yml index 731ee65ca..227204e92 100644 --- a/.github/workflows/rust-runtime-release.yml +++ b/.github/workflows/rust-runtime-release.yml @@ -20,6 +20,7 @@ on: pull_request: paths: + - ".github/actions/**" - ".github/workflows/rust-runtime-release.yml" - "sidemantic-rs/tests/packaged_cli_smoke.py" @@ -43,6 +44,13 @@ jobs: - name: Install Rust uses: dtolnay/rust-toolchain@stable + - name: Cache Rust compilation + if: github.event_name == 'pull_request' + id: rust-cache + uses: ./.github/actions/setup-rust-cache + with: + suffix: ${{ github.workflow }}-${{ github.job }}-${{ strategy.job-index || 0 }} + - name: Run package metadata tests run: cargo test --locked --test package_metadata @@ -64,6 +72,12 @@ jobs: cargo publish --locked + - name: Save completed build caches + if: ${{ !cancelled() && github.event_name == 'pull_request' }} + uses: ./.github/actions/save-build-caches + with: + rust-key: ${{ steps.rust-cache.outputs.key }} + cli-artifacts: name: Build CLI artifact (${{ matrix.artifact }}) runs-on: ${{ matrix.os }} @@ -90,6 +104,13 @@ jobs: - name: Install Rust uses: dtolnay/rust-toolchain@stable + - name: Cache Rust compilation + if: github.event_name == 'pull_request' + id: rust-cache + uses: ./.github/actions/setup-rust-cache + with: + suffix: ${{ github.workflow }}-${{ github.job }}-${{ strategy.job-index || 0 }} + - name: Build CLI run: cargo build --release --locked --bin sidemantic @@ -106,7 +127,11 @@ jobs: fi - name: Install uv - uses: astral-sh/setup-uv@v5 + id: uv + uses: ./.github/actions/setup-uv + with: + enable-cache: ${{ github.event_name == 'pull_request' }} + suffix: ${{ github.workflow }}-${{ github.job }}-${{ strategy.job-index || 0 }} - name: Smoke packaged CLI shell: bash @@ -127,6 +152,13 @@ jobs: path: sidemantic-rs/dist/* if-no-files-found: error + - name: Save completed build caches + if: ${{ !cancelled() && github.event_name == 'pull_request' }} + uses: ./.github/actions/save-build-caches + with: + rust-key: ${{ steps.rust-cache.outputs.key }} + uv-key: ${{ steps.uv.outputs.cache-key }} + github-release: name: Attach CLI artifacts to GitHub release needs: diff --git a/.github/workflows/sidemantic-rs-wheels.yml b/.github/workflows/sidemantic-rs-wheels.yml index 24fff09bd..4b2886e53 100644 --- a/.github/workflows/sidemantic-rs-wheels.yml +++ b/.github/workflows/sidemantic-rs-wheels.yml @@ -19,6 +19,7 @@ on: pull_request: paths: + - ".github/actions/**" - ".github/workflows/sidemantic-rs-wheels.yml" - "sidemantic-rs/pyproject.toml" - "sidemantic-rs/tests/python_wheel_smoke.py" @@ -51,6 +52,16 @@ jobs: steps: - uses: actions/checkout@v4 + - name: Install Rust + uses: dtolnay/rust-toolchain@stable + + - name: Cache Rust compilation + if: github.event_name == 'pull_request' + id: rust-cache + uses: ./.github/actions/setup-rust-cache + with: + suffix: ${{ github.workflow }}-${{ github.job }}-${{ strategy.job-index || 0 }} + - name: Set up Python uses: actions/setup-python@v5 with: @@ -62,14 +73,33 @@ jobs: command: build args: --release --manifest-path sidemantic-rs/Cargo.toml --out sidemantic-rs/dist manylinux: auto + docker-options: ${{ github.event_name == 'pull_request' && '--env MBX_CACHE_DIR --env MBX_CACHE_EXPORT_GROUP --env MBX_TARGET_VIEWS=0 --env MBX_CACHE_LINKS --env MBX_GC_AUTO' || '' }} + before-script-linux: | + if [ -x "$GITHUB_WORKSPACE/.ci-cache-bin/cargo" ]; then + export PATH="$GITHUB_WORKSPACE/.ci-cache-bin:$PATH" + fi target: ${{ matrix.target }} + - name: Restore ownership of the container compiler cache + if: ${{ !cancelled() && runner.os == 'Linux' && github.event_name == 'pull_request' }} + shell: bash + run: | + if [ -d "$MBX_CACHE_DIR" ]; then + sudo chown -R "$(id -u):$(id -g)" "$MBX_CACHE_DIR" + fi + - name: Install uv - uses: astral-sh/setup-uv@v5 + id: uv + uses: ./.github/actions/setup-uv + with: + enable-cache: ${{ github.event_name == 'pull_request' }} + suffix: ${{ github.workflow }}-${{ github.job }}-${{ strategy.job-index || 0 }} - name: Smoke built wheel on its target host shell: bash - run: uv run --no-project --with sidemantic-rs/dist/*.whl sidemantic-rs/tests/python_wheel_smoke.py + run: | + uv run --no-project --with sidemantic-rs/dist/*.whl sidemantic-rs/tests/python_wheel_smoke.py + uv run --no-project --with sidemantic-rs/dist/*.whl sidemantic-rs/tests/python_wheel_python_smoke.py - name: Upload wheel uses: actions/upload-artifact@v4 @@ -78,6 +108,13 @@ jobs: path: sidemantic-rs/dist/*.whl if-no-files-found: error + - name: Save completed build caches + if: ${{ !cancelled() && github.event_name == 'pull_request' }} + uses: ./.github/actions/save-build-caches + with: + rust-key: ${{ steps.rust-cache.outputs.key }} + uv-key: ${{ steps.uv.outputs.cache-key }} + build-sdist: name: Build sdist runs-on: ubuntu-latest @@ -89,9 +126,11 @@ jobs: - uses: actions/checkout@v4 - name: Install uv - uses: astral-sh/setup-uv@v5 + id: uv + uses: ./.github/actions/setup-uv with: - enable-cache: true + enable-cache: ${{ github.event_name == 'pull_request' }} + suffix: ${{ github.workflow }}-${{ github.job }}-${{ strategy.job-index || 0 }} - name: Set up Python run: uv python install 3.12 @@ -106,6 +145,12 @@ jobs: path: sidemantic-rs/dist/*.tar.gz if-no-files-found: error + - name: Save completed build caches + if: ${{ !cancelled() && github.event_name == 'pull_request' }} + uses: ./.github/actions/save-build-caches + with: + uv-key: ${{ steps.uv.outputs.cache-key }} + release: name: Publish or attach wheel artifacts needs: @@ -123,7 +168,7 @@ jobs: if: ${{ github.event_name == 'workflow_dispatch' && inputs.publish_pypi == true }} uses: astral-sh/setup-uv@v5 with: - enable-cache: true + enable-cache: false - name: Download distributions uses: actions/download-artifact@v4 diff --git a/.github/workflows/wasm-package-release.yml b/.github/workflows/wasm-package-release.yml index 0b6c2e9b1..43b5628f5 100644 --- a/.github/workflows/wasm-package-release.yml +++ b/.github/workflows/wasm-package-release.yml @@ -13,6 +13,7 @@ on: - "sidemantic-wasm-v*" pull_request: paths: + - ".github/actions/**" - "sidemantic-wasm/**" - "sidemantic-rs/**" - "Cargo.toml" @@ -41,6 +42,13 @@ jobs: with: targets: wasm32-unknown-unknown + - name: Cache Rust compilation + if: github.event_name == 'pull_request' + id: rust-cache + uses: ./.github/actions/setup-rust-cache + with: + suffix: ${{ github.workflow }}-${{ github.job }}-${{ strategy.job-index || 0 }} + - name: Install wasm-bindgen-cli matching the crate working-directory: . run: | @@ -103,3 +111,9 @@ jobs: fi # Publish the exact tarball exercised by the isolated consumer above. npm publish sidemantic-wasm.tgz --access public --ignore-scripts + + - name: Save completed build caches + if: ${{ !cancelled() && github.event_name == 'pull_request' }} + uses: ./.github/actions/save-build-caches + with: + rust-key: ${{ steps.rust-cache.outputs.key }} diff --git a/.github/workflows/yardstick-upstream.yml b/.github/workflows/yardstick-upstream.yml index e89ad8ab3..f86bccbdb 100644 --- a/.github/workflows/yardstick-upstream.yml +++ b/.github/workflows/yardstick-upstream.yml @@ -11,6 +11,7 @@ on: - cron: "17 10 * * *" pull_request: paths: + - ".github/actions/**" - ".github/workflows/yardstick-upstream.yml" - "docs/compatibility/yardstick.md" - "sidemantic/adapters/yardstick.py" @@ -38,10 +39,21 @@ jobs: steps: - uses: actions/checkout@v4 + - name: Install Rust + uses: dtolnay/rust-toolchain@stable + + - name: Cache Rust compilation + id: rust-cache + uses: ./.github/actions/setup-rust-cache + with: + suffix: ${{ github.workflow }}-${{ github.job }}-${{ strategy.job-index || 0 }} + - name: Install uv - uses: astral-sh/setup-uv@v5 + id: uv + uses: ./.github/actions/setup-uv with: enable-cache: true + suffix: ${{ github.workflow }}-${{ github.job }}-${{ strategy.job-index || 0 }} - name: Set up Python run: uv python install 3.12 @@ -51,3 +63,10 @@ jobs: - name: Replay upstream Yardstick tests run: uv run pytest -q tests/queries/test_yardstick_measures_replay.py -m yardstick_upstream + + - name: Save completed build caches + if: ${{ !cancelled() }} + uses: ./.github/actions/save-build-caches + with: + rust-key: ${{ steps.rust-cache.outputs.key }} + uv-key: ${{ steps.uv.outputs.cache-key }} diff --git a/.gitignore b/.gitignore index dce2b92cd..071d041bc 100644 --- a/.gitignore +++ b/.gitignore @@ -53,3 +53,7 @@ package.json # Rust workspace build artifacts (dax local wheel builds) /target/ + +# GitHub Actions compiler cache and Cargo shim +/.ci-mbx/ +/.ci-cache-bin/ diff --git a/crates/dax-pyo3/pyproject.toml b/crates/dax-pyo3/pyproject.toml index b213f9cb6..8af2d1caa 100644 --- a/crates/dax-pyo3/pyproject.toml +++ b/crates/dax-pyo3/pyproject.toml @@ -15,3 +15,23 @@ module-name = "sidemantic_dax._native" python-source = "python" features = ["pyo3/extension-module", "pyo3/abi3-py311"] exclude = ["dist/*", "target/*"] + +# uv otherwise only checks Python packaging metadata for local wheel rebuilds. +[tool.uv] +cache-keys = [ + { file = "pyproject.toml" }, + { file = "README.md" }, + { file = "Cargo.toml" }, + { file = "../../Cargo.toml" }, + { file = "../../Cargo.lock" }, + { file = "src/**/*" }, + { file = "python/**/*.py" }, + { file = "python/**/*.pyi" }, + { file = "python/**/py.typed" }, + { file = "../dax-parser/Cargo.toml" }, + { file = "../dax-parser/src/**/*" }, + { file = "../../.cargo/config.toml" }, + { file = "../../rust-toolchain*" }, + { env = "RUSTFLAGS" }, + { env = "CARGO_ENCODED_RUSTFLAGS" }, +] diff --git a/docs/rust-runtime.md b/docs/rust-runtime.md index 60034e55e..44fc7298a 100644 --- a/docs/rust-runtime.md +++ b/docs/rust-runtime.md @@ -48,6 +48,21 @@ Rust does not parse these source formats directly: Those remain Python-owned import paths. +## Default runtime and installation + +A normal `uv tool install sidemantic` includes the matching `sidemantic-rs` +package. Native validation, compilation, and semantic SQL rewriting use Rust by +default and report an error if it is unavailable or a capability is unsupported. +Use `--engine python` to select Python, or `--engine auto` to allow fallback for +known unsupported capabilities. Invalid input and unexpected failures still propagate. +`SIDEMANTIC_ENGINE=python` sets a process-wide default; explicit engine selection +and project runtime configuration take precedence. + +Pyodide excludes the native dependency and defaults to Python. The distributed +Python extension uses the lightweight `python` feature; database execution +continues through Python adapters. Rust ADBC execution requires a separate +source build with `python-adbc`. + ## Python API Engine Selection Python users can select the native runtime explicitly: diff --git a/docs/semantic-input.md b/docs/semantic-input.md index af166a1ad..ecddd50dd 100644 --- a/docs/semantic-input.md +++ b/docs/semantic-input.md @@ -351,8 +351,9 @@ synthetic fixtures, not production workload or performance qualification. Rust's `semantic_input` tests check decoding, keys, scope, dialects and rejection without relying on Python preprocessing. -This contract does not change the default engine, retire the Python compiler, -or claim that WASM, the DuckDB extension, and the Python binding already expose +Native Python installations now default to Rust with a matching runtime package; +Pyodide retains Python. This does not retire the Python compiler or claim that +WASM, the DuckDB extension, and the Python binding already expose identical capabilities. Each host needs corresponding acceptance evidence before its default or implementation ownership changes. diff --git a/pyproject.toml b/pyproject.toml index afe0ce787..2c32abde1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -6,6 +6,7 @@ readme = "README.md" license = {file = "LICENSE"} requires-python = ">=3.11" dependencies = [ + "sidemantic-rs==0.12.0; sys_platform != 'emscripten'", "antlr4-python3-runtime>=4.13.2", "sqlglot>=30.1.0", "pyyaml>=6.0", @@ -244,9 +245,14 @@ warn_unused_ignores = true [tool.uv] prerelease = "if-necessary" -required-environments = ["sys_platform == 'linux' and platform_machine == 'x86_64'"] +# Require binary wheels for the CPython versions exercised by CI, without +# imposing a Python upper bound on users or requiring future/PyPy wheels. +required-environments = [ + "sys_platform == 'linux' and platform_machine == 'x86_64' and implementation_name == 'cpython' and platform_python_implementation == 'CPython' and python_version < '3.15'", +] [tool.uv.sources] +sidemantic-rs = { path = "sidemantic-rs" } sidemantic-dax = { path = "crates/dax-pyo3" } [dependency-groups] diff --git a/sidemantic-rs/pyproject.toml b/sidemantic-rs/pyproject.toml index 824482b9d..dd3784c5f 100644 --- a/sidemantic-rs/pyproject.toml +++ b/sidemantic-rs/pyproject.toml @@ -4,11 +4,26 @@ build-backend = "maturin" [project] name = "sidemantic-rs" -version = "0.1.0" +version = "0.12.0" description = "Standalone Rust runtime bindings for Sidemantic" license = "AGPL-3.0-only" requires-python = ">=3.11" [tool.maturin] module-name = "sidemantic_rs" -features = ["python-adbc"] +features = ["python"] + +# uv otherwise only checks Python packaging metadata for local wheel rebuilds. +[tool.uv] +cache-keys = [ + { file = "pyproject.toml" }, + { file = "Cargo.toml" }, + { file = "../Cargo.toml" }, + { file = "../Cargo.lock" }, + { file = "build.rs" }, + { file = "src/**/*" }, + { file = "../.cargo/config.toml" }, + { file = "../rust-toolchain*" }, + { env = "RUSTFLAGS" }, + { env = "CARGO_ENCODED_RUSTFLAGS" }, +] diff --git a/sidemantic-rs/src/bin/sidemantic-mcp.rs b/sidemantic-rs/src/bin/sidemantic-mcp.rs index b4d314a27..100c91716 100644 --- a/sidemantic-rs/src/bin/sidemantic-mcp.rs +++ b/sidemantic-rs/src/bin/sidemantic-mcp.rs @@ -13,9 +13,8 @@ use adbc_core::options::{OptionConnection, OptionDatabase, OptionValue}; use rmcp::{ handler::server::{router::tool::ToolRouter, wrapper::Parameters}, model::{ - Annotated, CallToolResult, ListResourcesResult, PaginatedRequestParams, RawResource, - ReadResourceRequestParams, ReadResourceResult, ResourceContents, ServerCapabilities, - ServerInfo, + CallToolResult, ListResourcesResult, PaginatedRequestParams, ReadResourceRequestParams, + ReadResourceResult, Resource, ResourceContents, ServerCapabilities, ServerInfo, }, schemars, schemars::JsonSchema, @@ -522,21 +521,10 @@ impl ServerHandler for SidemanticMcpServer { _context: RequestContext, ) -> impl std::future::Future> + Send + '_ { std::future::ready(Ok(ListResourcesResult::with_all_items(vec![ - Annotated::new( - RawResource { - uri: CATALOG_RESOURCE_URI.to_string(), - name: "catalog".to_string(), - title: Some("Sidemantic Catalog Metadata".to_string()), - description: Some( - "Postgres-compatible catalog metadata for the semantic layer.".to_string(), - ), - mime_type: Some("application/json".to_string()), - size: None, - icons: None, - meta: None, - }, - None, - ), + Resource::new(CATALOG_RESOURCE_URI, "catalog") + .with_title("Sidemantic Catalog Metadata") + .with_description("Postgres-compatible catalog metadata for the semantic layer.") + .with_mime_type("application/json"), ]))) } diff --git a/sidemantic-rs/src/core/dependency.rs b/sidemantic-rs/src/core/dependency.rs index fb4d7ef4e..d03d383ab 100644 --- a/sidemantic-rs/src/core/dependency.rs +++ b/sidemantic-rs/src/core/dependency.rs @@ -31,8 +31,8 @@ impl SemanticColumnReference { pub fn parse_semantic_expression(sql: &str) -> crate::error::Result { #[cfg(target_arch = "wasm32")] crate::wasm_sql_guard::check(sql, DialectType::DuckDB)?; - let statement = polyglot_sql::parse_one(&format!("SELECT {sql}"), DialectType::DuckDB) - .map_err(|error| crate::error::SidemanticError::SqlParse(error.to_string()))?; + let statement = + crate::semantic_input::dialects::parse(&format!("SELECT {sql}"), DialectType::DuckDB)?; let Expression::Select(mut select) = statement else { return Err(crate::error::SidemanticError::SqlParse( "expected scalar expression".into(), @@ -69,11 +69,18 @@ fn column_references( let ast = serde_json::to_value(expression) .map_err(|error| crate::error::SidemanticError::SqlParse(error.to_string()))?; let mut references = Vec::new(); - let mut stack = vec![(&ast, false)]; - while let Some((node, aggregate_input)) = stack.pop() { + let mut stack = vec![(&ast, false, HashSet::::new())]; + while let Some((node, aggregate_input, bindings)) = stack.pop() { match node { serde_json::Value::Object(fields) => { let kind = (fields.len() == 1).then(|| fields.keys().next().unwrap().as_str()); + if kind == Some("lambda") { + let lambda = &fields["lambda"]; + let mut bindings = bindings; + bindings.extend(lambda_parameter_names(lambda)); + stack.push((&lambda["body"], aggregate_input, bindings)); + continue; + } if allow_subqueries && matches!( kind, @@ -88,67 +95,107 @@ fn column_references( }); } let aggregate_input = aggregate_input - || matches!( - kind, - Some( - "count" - | "sum" - | "avg" - | "min" - | "max" - | "median" - | "mode" - | "stddev" - | "stddev_pop" - | "stddev_samp" - | "variance" - | "var_pop" - | "var_samp" - | "aggregate_function" - | "group_concat" - | "string_agg" - | "list_agg" - | "array_agg" - | "count_if" - | "sum_if" - | "first" - | "last" - | "any_value" - | "approx_distinct" - | "approx_count_distinct" - | "approx_percentile" - | "percentile" - | "logical_and" - | "logical_or" - | "skewness" - | "array_concat_agg" - | "array_unique_agg" - | "bool_xor_agg" - ) - ); + || is_aggregate_ast_node(node) + || matches!(kind, Some("window" | "window_function")); if kind == Some("column") { let column: polyglot_sql::expressions::Column = serde_json::from_value(fields["column"].clone()).map_err(|error| { crate::error::SidemanticError::SqlParse(error.to_string()) })?; + if bindings.contains( + column + .table + .as_ref() + .map_or(column.name.name.as_str(), |table| table.name.as_str()), + ) { + continue; + } references.push(SemanticColumnReference { model: column.table.map(|table| table.name), field: column.name.name, aggregate_input, }); } else { - stack.extend(fields.values().map(|child| (child, aggregate_input))); + stack.extend( + fields + .values() + .map(|child| (child, aggregate_input, bindings.clone())), + ); } } - serde_json::Value::Array(children) => { - stack.extend(children.iter().map(|child| (child, aggregate_input))) - } + serde_json::Value::Array(children) => stack.extend( + children + .iter() + .map(|child| (child, aggregate_input, bindings.clone())), + ), _ => {} } } Ok(references) } +/// Aggregate expression variants in the pinned SQL AST. +pub(crate) fn is_aggregate_ast_kind(kind: &str) -> bool { + matches!( + kind, + "count" + | "sum" + | "avg" + | "min" + | "max" + | "median" + | "mode" + | "stddev" + | "stddev_pop" + | "stddev_samp" + | "variance" + | "var_pop" + | "var_samp" + | "aggregate_function" + | "group_concat" + | "string_agg" + | "list_agg" + | "array_agg" + | "count_if" + | "sum_if" + | "first" + | "last" + | "any_value" + | "approx_distinct" + | "approx_count_distinct" + | "approx_percentile" + | "percentile" + | "logical_and" + | "logical_or" + | "skewness" + | "array_concat_agg" + | "array_unique_agg" + | "bool_xor_agg" + ) +} + +pub(crate) fn is_aggregate_ast_node(node: &serde_json::Value) -> bool { + let Some(fields) = node.as_object().filter(|fields| fields.len() == 1) else { + return false; + }; + let (kind, value) = fields.iter().next().unwrap(); + is_aggregate_ast_kind(kind) + // polyglot 0.1.15 leaves bare DuckDB LIST as a generic function, + // while LIST with DISTINCT/ORDER/FILTER is an AggregateFunction. + || (kind == "function" + && value["quoted"].as_bool() != Some(true) + && value["name"].as_str().is_some_and(|name| name.eq_ignore_ascii_case("LIST"))) +} + +fn lambda_parameter_names(lambda: &serde_json::Value) -> Vec { + lambda["parameters"] + .as_array() + .into_iter() + .flatten() + .filter_map(|parameter| parameter["name"].as_str().map(str::to_owned)) + .collect() +} + /// Replace column nodes in every AST child, including typed functions that the /// pinned polyglot transform visitor does not descend into. pub fn replace_semantic_columns( @@ -176,9 +223,21 @@ fn replace_columns( value: &mut serde_json::Value, replacements: &std::collections::HashMap<(Option, String), String>, skip_subqueries: bool, + bindings: &HashSet, ) -> crate::error::Result<()> { match value { serde_json::Value::Object(fields) => { + if fields.len() == 1 && fields.contains_key("lambda") { + let lambda = fields.get_mut("lambda").unwrap(); + let mut bindings = bindings.clone(); + bindings.extend(lambda_parameter_names(lambda)); + return replace( + &mut lambda["body"], + replacements, + skip_subqueries, + &bindings, + ); + } if skip_subqueries && fields.len() == 1 && fields.keys().any(|name| { @@ -195,6 +254,9 @@ fn replace_columns( serde_json::from_value(fields["column"].clone()) .map_err(|error| SidemanticError::SqlGeneration(error.to_string()))?; let key = (column.table.map(|table| table.name), column.name.name); + if bindings.contains(key.0.as_deref().unwrap_or(&key.1)) { + return Ok(()); + } if let Some(sql) = replacements.get(&key) { *value = serde_json::to_value(Expression::Raw(polyglot_sql::expressions::Raw { @@ -204,13 +266,13 @@ fn replace_columns( } } else { for child in fields.values_mut() { - replace(child, replacements, skip_subqueries)?; + replace(child, replacements, skip_subqueries, bindings)?; } } } serde_json::Value::Array(children) => { for child in children { - replace(child, replacements, skip_subqueries)?; + replace(child, replacements, skip_subqueries, bindings)?; } } _ => {} @@ -219,7 +281,7 @@ fn replace_columns( } let mut value = serde_json::to_value(expression) .map_err(|error| SidemanticError::SqlGeneration(error.to_string()))?; - replace(&mut value, replacements, skip_subqueries)?; + replace(&mut value, replacements, skip_subqueries, &HashSet::new())?; serde_json::from_value(value).map_err(|error| SidemanticError::SqlGeneration(error.to_string())) } @@ -643,6 +705,69 @@ mod tests { #[allow(unused_imports)] use crate::core::model::{Aggregation, Dimension, Model}; + #[test] + fn lambda_bindings_are_local_to_the_body_for_dependencies_and_replacement() { + let sql = "list_transform(orders.items, x -> x.value + external.bias) + x.value"; + let references = semantic_column_references(sql).unwrap(); + let names: Vec<_> = references + .iter() + .map(SemanticColumnReference::name) + .collect(); + assert_eq!(names.len(), 3, "{names:?}"); + for name in ["orders.items", "external.bias", "x.value"] { + assert!(names.contains(&name.to_owned()), "{names:?}"); + } + let replacements = std::collections::HashMap::from([ + ((Some("x".into()), "value".into()), "outer_value".into()), + ((Some("external".into()), "bias".into()), "free_bias".into()), + ]); + let replaced = + replace_semantic_columns(parse_semantic_expression(sql).unwrap(), &replacements) + .unwrap(); + let rendered = polyglot_sql::generate(&replaced, DialectType::DuckDB).unwrap(); + assert_eq!(rendered.matches("outer_value").count(), 1, "{rendered}"); + assert!(rendered.contains("x.value"), "{rendered}"); + assert!(rendered.contains("free_bias"), "{rendered}"); + } + + #[test] + fn list_aggregate_inputs_stay_physical_through_scalar_list_functions() { + let sql = "LIST_AGGREGATE(LIST_TRANSFORM(LIST_DISTINCT(LIST(STRUCT_PACK(k := orders.id, v := orders.amount))), x -> x.v), 'quantile_cont', 0.5) + orders.revenue"; + let references = semantic_column_references(sql).unwrap(); + assert_eq!(references.len(), 3, "{references:?}"); + for reference in references { + assert_eq!( + reference.aggregate_input, + reference.field != "revenue", + "{reference:?}" + ); + } + assert!(validate_row_expression( + &parse_semantic_expression("LIST(amount)").unwrap(), + "test.row_scope" + ) + .is_err()); + let scalar_list = + semantic_column_references("list_transform(orders.amounts, x -> x + 1)").unwrap(); + assert_eq!(scalar_list.len(), 1); + assert!(!scalar_list[0].aggregate_input); + } + + #[test] + fn authored_window_partition_and_order_columns_are_physical_inputs() { + let references = semantic_column_references( + "sum(amount) over (partition by category order by year) + revenue", + ) + .unwrap(); + for reference in references { + assert_eq!( + reference.aggregate_input, + reference.field != "revenue", + "{reference:?}" + ); + } + } + #[test] fn outer_filter_dependencies_do_not_capture_nested_columns_or_literals() { let sql = "orders.status IN (SELECT status FROM allowed WHERE note = 'orders.revenue AND customers.id')"; @@ -767,49 +892,12 @@ pub fn validate_row_expression( match value { serde_json::Value::Object(fields) => { let kind = (fields.len() == 1).then(|| fields.keys().next().unwrap().as_str()); - matches!( - kind, - Some( - "select" - | "subquery" - | "raw" - | "window" - | "window_function" - | "count" - | "sum" - | "avg" - | "min" - | "max" - | "median" - | "mode" - | "stddev" - | "stddev_pop" - | "stddev_samp" - | "variance" - | "var_pop" - | "var_samp" - | "aggregate_function" - | "group_concat" - | "string_agg" - | "list_agg" - | "array_agg" - | "count_if" - | "sum_if" - | "first" - | "last" - | "any_value" - | "approx_distinct" - | "approx_count_distinct" - | "approx_percentile" - | "percentile" - | "logical_and" - | "logical_or" - | "skewness" - | "array_concat_agg" - | "array_unique_agg" - | "bool_xor_agg" - ) - ) || fields.values().any(visit) + (is_aggregate_ast_node(value) + || matches!( + kind, + Some("select" | "subquery" | "raw" | "window" | "window_function") + )) + || fields.values().any(visit) } serde_json::Value::Array(values) => values.iter().any(visit), _ => false, diff --git a/sidemantic-rs/src/core/graph.rs b/sidemantic-rs/src/core/graph.rs index bebd5a666..a51df6676 100644 --- a/sidemantic-rs/src/core/graph.rs +++ b/sidemantic-rs/src/core/graph.rs @@ -220,7 +220,9 @@ impl SemanticGraph { if let Some(dimensions) = preagg.dimensions.as_ref() { for dimension in dimensions { - if model.get_dimension(dimension).is_none() { + if model.get_dimension(dimension).is_none() + && !model.is_foreign_key_dimension(dimension) + { return Err(SidemanticError::Validation(format!( "Pre-aggregation '{}.{}' references unknown dimension '{}'", model.name, preagg.name, dimension @@ -363,6 +365,33 @@ impl SemanticGraph { Some(owner) } + /// Whether this SQL instance is a declared many-to-many junction population. + pub(crate) fn is_bridge_instance(&self, instance: &str) -> bool { + self.relationship_instances + .iter() + .any(|((source, name), target)| { + let Some(model) = self.get_model(source) else { + return false; + }; + model.relationships.iter().any(|relationship| { + if !relationship.active + || relationship.name != *name + || relationship.r#type != RelationshipType::ManyToMany + { + return false; + } + let Some(through) = &relationship.through else { + return false; + }; + if source != &model.name || relationship.target_model.is_some() { + instance == format!("{target}$through") + } else { + instance == through + } + }) + }) + } + pub fn relationship_target_instance( &self, source: &str, @@ -798,12 +827,10 @@ impl SemanticGraph { let fk_keys = rel.foreign_key_columns(); let pk_keys = if rel.primary_key.is_some() || rel.primary_key_columns.is_some() { rel.primary_key_columns() - } else if (instance != canonical || rel.target_model.is_some()) - && matches!( - rel.r#type, - RelationshipType::OneToMany | RelationshipType::OneToOne - ) - { + } else if matches!( + rel.r#type, + RelationshipType::OneToMany | RelationshipType::OneToOne + ) { model.primary_keys() } else { self.models @@ -821,15 +848,10 @@ impl SemanticGraph { (model.primary_keys(), fk_keys.clone()) } } - RelationshipType::OneToOne - if instance != canonical || rel.target_model.is_some() => - { + RelationshipType::ManyToOne => (fk_keys.clone(), pk_keys.clone()), + RelationshipType::OneToMany | RelationshipType::OneToOne => { (pk_keys.clone(), fk_keys.clone()) } - RelationshipType::ManyToOne | RelationshipType::OneToOne => { - (fk_keys.clone(), pk_keys.clone()) - } - RelationshipType::OneToMany => (pk_keys.clone(), fk_keys.clone()), }; self.adjacency.entry(instance.clone()).or_default().push(( @@ -844,7 +866,13 @@ impl SemanticGraph { if rel.r#type == RelationshipType::Cross { None } else { - rel.sql.clone() + // Legacy column-name SQL leaves the keyed join intact. + // Custom predicates use the same placeholder contract + // as semantic-input decoding and the Python graph. + rel.sql + .as_ref() + .filter(|sql| sql.contains("{from}") || sql.contains("{to}")) + .cloned() }, rel.edge_id.clone(), )); @@ -894,6 +922,7 @@ impl SemanticGraph { .sql .as_ref() .filter(|_| rel.r#type != RelationshipType::Cross) + .filter(|sql| sql.contains("{from}") || sql.contains("{to}")) .map(|sql| { sql.replace("{from}", "__TEMP__") .replace("{to}", "{from}") @@ -903,12 +932,10 @@ impl SemanticGraph { let fk_keys = rel.foreign_key_columns(); let pk_keys = if rel.primary_key.is_some() || rel.primary_key_columns.is_some() { rel.primary_key_columns() - } else if (instance != canonical || rel.target_model.is_some()) - && matches!( - rel.r#type, - RelationshipType::OneToMany | RelationshipType::OneToOne - ) - { + } else if matches!( + rel.r#type, + RelationshipType::OneToMany | RelationshipType::OneToOne + ) { model.primary_keys() } else { self.models @@ -926,15 +953,10 @@ impl SemanticGraph { (fk_keys.clone(), model.primary_keys()) } } - RelationshipType::OneToOne - if instance != canonical || rel.target_model.is_some() => - { + RelationshipType::ManyToOne => (pk_keys.clone(), fk_keys.clone()), + RelationshipType::OneToMany | RelationshipType::OneToOne => { (fk_keys.clone(), pk_keys.clone()) } - RelationshipType::ManyToOne | RelationshipType::OneToOne => { - (pk_keys.clone(), fk_keys.clone()) - } - RelationshipType::OneToMany => (fk_keys.clone(), pk_keys.clone()), }; self.adjacency @@ -1067,24 +1089,27 @@ impl SemanticGraph { let model_name = parts[0]; let field_with_granularity = parts[1]; - // Check for granularity suffix (e.g., order_date__month) - let (field_name, granularity) = - if let Some((field, gran)) = field_with_granularity.rsplit_once("__") { - if field.is_empty() || gran.is_empty() { - return Err(SidemanticError::InvalidReference { - reference: reference.to_string(), - }); - } - (field.to_string(), Some(gran.to_string())) - } else { - (field_with_granularity.to_string(), None) - }; - - // Verify model exists - if self.get_model(model_name).is_none() { + let model = self.get_model(model_name).ok_or_else(|| { let available: Vec<&str> = self.models.keys().map(|s| s.as_str()).collect(); - return Err(SidemanticError::model_not_found(model_name, &available)); - } + SidemanticError::model_not_found(model_name, &available) + })?; + + // Exact public fields win over the granularity syntax, including names + // resembling internal helpers such as __fanout_rank_0. + let (field_name, granularity) = if model.get_metric(field_with_granularity).is_some() + || model.get_dimension(field_with_granularity).is_some() + { + (field_with_granularity.to_string(), None) + } else if let Some((field, gran)) = field_with_granularity.rsplit_once("__") { + if field.is_empty() || gran.is_empty() { + return Err(SidemanticError::InvalidReference { + reference: reference.to_string(), + }); + } + (field.to_string(), Some(gran.to_string())) + } else { + (field_with_granularity.to_string(), None) + }; Ok((model_name.to_string(), field_name, granularity)) } @@ -1098,6 +1123,37 @@ mod tests { }; use crate::core::parameter::{Parameter, ParameterType}; + #[test] + fn exact_field_names_take_precedence_over_granularity_suffixes() { + let mut graph = SemanticGraph::new(); + graph + .add_model( + Model::new("orders", "id") + .with_table("orders") + .with_metric(Metric::count("__fanout_rank_0")) + .with_metric(Metric::count("__sidemantic_filtered_2")) + .with_dimension(Dimension::categorical("__sidemantic_filtered_1_raw")) + .with_dimension(Dimension::categorical("literal__month")) + .with_dimension(Dimension::time("day")), + ) + .unwrap(); + for name in [ + "__fanout_rank_0", + "__sidemantic_filtered_2", + "__sidemantic_filtered_1_raw", + "literal__month", + ] { + assert_eq!( + graph.parse_reference(&format!("orders.{name}")).unwrap(), + ("orders".into(), name.into(), None) + ); + } + assert_eq!( + graph.parse_reference("orders.day__month").unwrap(), + ("orders".into(), "day".into(), Some("month".into())) + ); + } + fn role(name: &str, target: &str, key: &str) -> Relationship { let mut relationship = Relationship::many_to_one(name).with_keys(key, "id"); relationship.target_model = Some(target.into()); @@ -1256,7 +1312,11 @@ mod tests { assert_eq!(path.steps[1].from_keys, vec![key.to_string()]); assert_eq!(graph.get_model(&bridge).unwrap().name, "links"); assert_eq!(graph.role_root_owner(&bridge), Some("orders")); + assert!(graph.is_bridge_instance(&bridge)); + assert!(!graph.is_bridge_instance(name)); } + assert!(!graph.is_bridge_instance("links")); + assert!(!graph.is_bridge_instance("orders")); } #[test] @@ -1614,6 +1674,40 @@ mod tests { .contains("references unknown measure 'missing_revenue'")); } + #[test] + fn test_pre_aggregation_dimensions_accept_declared_foreign_keys() { + for (dimension, accepted) in [ + ("status", true), + ("tenant_id", true), + ("customer_id", true), + ("missing", false), + ("customer_id__month", false), + ] { + let mut relationship = Relationship::many_to_one("customers"); + relationship.foreign_key_columns = Some(vec!["tenant_id".into(), "customer_id".into()]); + let preagg = serde_json::from_value(serde_json::json!({ + "name": "by_customer", "measures": ["revenue"], + "dimensions": [dimension] + })) + .unwrap(); + let model = Model::new("orders", "id") + .with_table("orders") + .with_dimension(Dimension::categorical("status")) + .with_metric(Metric::sum("revenue", "amount")) + .with_relationship(relationship) + .with_pre_aggregation(preagg); + let result = SemanticGraph::new().add_model(model); + if accepted { + result.unwrap(); + } else { + assert!(result + .unwrap_err() + .to_string() + .contains("references unknown dimension")); + } + } + } + #[test] fn test_find_join_path() { let graph = create_test_graph(); @@ -1692,6 +1786,35 @@ mod tests { assert_eq!(path.steps[0].to_keys, vec!["customer_uid".to_string()]); } + #[test] + fn one_to_one_uses_local_key_and_remote_foreign_key_in_both_directions() { + for explicit_key in [false, true] { + let mut graph = SemanticGraph::new(); + let mut relationship = Relationship::new("regions"); + relationship.r#type = RelationshipType::OneToOne; + relationship.foreign_key = Some("region_record_id".into()); + if explicit_key { + relationship.primary_key = Some("region_id".into()); + } + graph + .add_model( + Model::new("sales", "region_id") + .with_table("sales") + .with_relationship(relationship), + ) + .unwrap(); + graph + .add_model(Model::new("regions", "region_record_id").with_table("regions")) + .unwrap(); + let forward = graph.find_join_path("sales", "regions").unwrap(); + assert_eq!(forward.steps[0].from_keys, vec!["region_id"]); + assert_eq!(forward.steps[0].to_keys, vec!["region_record_id"]); + let reverse = graph.find_join_path("regions", "sales").unwrap(); + assert_eq!(reverse.steps[0].from_keys, vec!["region_record_id"]); + assert_eq!(reverse.steps[0].to_keys, vec!["region_id"]); + } + } + #[test] fn test_one_to_one_omitted_key_defaults_to_id() { let mut graph = SemanticGraph::new(); @@ -1745,6 +1868,33 @@ mod tests { assert_eq!(path.fan_out_boundary(), Some("orders")); } + #[test] + fn legacy_relationship_sql_preserves_forward_and_reverse_keys() { + let mut graph = SemanticGraph::new(); + let mut relationship = Relationship::many_to_one("customers").with_condition("customer_id"); + relationship.foreign_key = Some("customer_id".into()); + graph + .add_model( + Model::new("orders", "order_id") + .with_table("orders") + .with_relationship(relationship), + ) + .unwrap(); + graph + .add_model(Model::new("customers", "customer_key").with_table("customers")) + .unwrap(); + for (from, to, from_key, to_key) in [ + ("orders", "customers", "customer_id", "customer_key"), + ("customers", "orders", "customer_key", "customer_id"), + ] { + let path = graph.find_join_path(from, to).unwrap(); + assert_eq!(path.steps.len(), 1); + assert!(path.steps[0].custom_condition.is_none()); + assert_eq!(path.steps[0].from_keys, vec![from_key]); + assert_eq!(path.steps[0].to_keys, vec![to_key]); + } + } + #[test] fn test_custom_join_condition() { let mut graph = SemanticGraph::new(); @@ -1844,6 +1994,8 @@ mod tests { let path = graph.find_join_path("orders", "products").unwrap(); assert_eq!(path.steps.len(), 2); + assert!(graph.is_bridge_instance("order_items")); + assert!(!graph.is_bridge_instance("products")); // orders -> order_items assert_eq!(path.steps[0].from_model, "orders"); diff --git a/sidemantic-rs/src/core/key_expression.rs b/sidemantic-rs/src/core/key_expression.rs index 6369a76cf..38bc2636b 100644 --- a/sidemantic-rs/src/core/key_expression.rs +++ b/sidemantic-rs/src/core/key_expression.rs @@ -42,6 +42,19 @@ fn deterministic_scalar(expression: &Expression) -> bool { && deterministic_scalar(&concat.expression) } Expression::Coalesce(arguments) => arguments.expressions.iter().all(deterministic_scalar), + Expression::DateTrunc(trunc) | Expression::TimestampTrunc(trunc) => { + deterministic_scalar(&trunc.this) + } + Expression::Function(function) + if !function.quoted + && function.name.eq_ignore_ascii_case("DATE_TRUNC") + && function.args.len() == 2 => + { + matches!( + &function.args[0], + Expression::Literal(polyglot_sql::expressions::Literal::String(_)) + ) && deterministic_scalar(&function.args[1]) + } _ => false, } } @@ -154,6 +167,26 @@ mod tests { use super::*; use crate::core::Dimension; + #[test] + fn date_bucket_keys_keep_physical_input_scope() { + let model = Model::new("monthly_sales", "month") + .with_dimension(Dimension::new("month").with_sql("DATE_TRUNC('month', order_date)")); + let expression = key_expression(&model, "month", Some("s"), DialectType::DuckDB).unwrap(); + let sql = polyglot_sql::generate(&expression, DialectType::DuckDB).unwrap(); + assert!( + sql.to_ascii_uppercase() + .contains("DATE_TRUNC('MONTH', S.ORDER_DATE)"), + "{sql}" + ); + assert!(is_computed_key(&model, "month").unwrap()); + for input in ["random()", "SUM(order_date)", "other.order_date"] { + let model = Model::new("monthly_sales", "month").with_dimension( + Dimension::new("month").with_sql(format!("DATE_TRUNC('month', {input})")), + ); + assert!(key_expression(&model, "month", None, DialectType::DuckDB).is_err()); + } + } + #[test] fn key_classification_uses_resolved_direction_and_composite_defaults() { use crate::core::{Relationship, RelationshipType}; diff --git a/sidemantic-rs/src/core/mod.rs b/sidemantic-rs/src/core/mod.rs index b206e03ff..b06202f11 100644 --- a/sidemantic-rs/src/core/mod.rs +++ b/sidemantic-rs/src/core/mod.rs @@ -13,6 +13,7 @@ mod segment; pub mod symmetric_agg; mod table_calc; +pub(crate) use dependency::is_aggregate_ast_node; pub use dependency::{ check_circular_dependencies, extract_column_references_from_expr, extract_dependencies, extract_dependencies_with_context, outer_semantic_column_references, parse_semantic_expression, @@ -30,7 +31,7 @@ pub use model::{ pub use parameter::{Parameter, ParameterType}; pub use policy::{AccessRule, PolicyError, PreparedPolicies, SecurityPolicy}; pub(crate) use preaggregation::{ - materialization_sql as preaggregation_materialization_sql, + materialization_sql as preaggregation_materialization_sql, replace_model_placeholder, source_expression as preaggregation_source_expression, }; pub use relative_date::RelativeDate; diff --git a/sidemantic-rs/src/core/model.rs b/sidemantic-rs/src/core/model.rs index 52e613174..af602b7a7 100644 --- a/sidemantic-rs/src/core/model.rs +++ b/sidemantic-rs/src/core/model.rs @@ -190,6 +190,7 @@ pub enum ComparisonType { Wow, // Week over week Dod, // Day over day Qoq, // Quarter over quarter + #[serde(alias = "prior_period")] PriorPeriod, } @@ -1112,6 +1113,16 @@ impl Model { self.dimensions.iter().find(|d| d.name == name) } + /// Relationship foreign keys can be grouped without a separate dimension declaration. + pub fn is_foreign_key_dimension(&self, name: &str) -> bool { + self.relationships.iter().any(|relationship| { + relationship + .foreign_key_columns() + .iter() + .any(|column| column == name) + }) + } + /// Find a metric by name pub fn get_metric(&self, name: &str) -> Option<&Metric> { self.metrics.iter().find(|m| m.name == name) @@ -1137,6 +1148,15 @@ impl Model { mod tests { use super::*; + #[test] + fn prior_period_accepts_python_and_legacy_spellings() { + for name in ["prior_period", "priorperiod"] { + let comparison: ComparisonType = + serde_json::from_value(serde_json::json!(name)).unwrap(); + assert_eq!(comparison, ComparisonType::PriorPeriod); + } + } + #[test] fn test_dimension_sql_expr() { let dim = Dimension::new("status"); diff --git a/sidemantic-rs/src/core/preaggregation.rs b/sidemantic-rs/src/core/preaggregation.rs index b768052bb..12540d3a3 100644 --- a/sidemantic-rs/src/core/preaggregation.rs +++ b/sidemantic-rs/src/core/preaggregation.rs @@ -162,7 +162,7 @@ pub(crate) fn materialization_sql( } /// Rewrite only actual placeholder tokens, leaving literal and subquery text intact. -fn replace_model_placeholder(sql: &str, owner: Option<&str>) -> Result { +pub(crate) fn replace_model_placeholder(sql: &str, owner: Option<&str>) -> Result { use polyglot_sql::{dialects::Dialect, DialectType, TokenType}; let tokens = Dialect::get(DialectType::DuckDB) .tokenize(sql) diff --git a/sidemantic-rs/src/lib.rs b/sidemantic-rs/src/lib.rs index b277c3dc3..12b01ea19 100644 --- a/sidemantic-rs/src/lib.rs +++ b/sidemantic-rs/src/lib.rs @@ -105,13 +105,13 @@ pub use wasm::{ wasm_analyze_migrator_query, wasm_build_preaggregation_refresh_statements, wasm_build_symmetric_aggregate_sql, wasm_calculate_preaggregation_benefit_score, wasm_chart_auto_detect_columns, wasm_chart_encoding_type, wasm_chart_format_label, - wasm_chart_select_type, wasm_compile_with_yaml_query, wasm_detect_adapter_kind, - wasm_dimension_sql_expr_with_yaml, wasm_dimension_with_granularity_with_yaml, - wasm_evaluate_table_calculation_expression, wasm_extract_column_references, - wasm_extract_metric_dependencies_from_yaml, wasm_extract_preaggregation_patterns, - wasm_find_models_for_query, wasm_find_relationship_path_with_yaml, - wasm_format_parameter_value_with_yaml, wasm_generate_catalog_metadata_with_yaml, - wasm_generate_preaggregation_definition, + wasm_chart_select_type, wasm_compile_with_semantic_input, wasm_compile_with_yaml_query, + wasm_detect_adapter_kind, wasm_dimension_sql_expr_with_yaml, + wasm_dimension_with_granularity_with_yaml, wasm_evaluate_table_calculation_expression, + wasm_extract_column_references, wasm_extract_metric_dependencies_from_yaml, + wasm_extract_preaggregation_patterns, wasm_find_models_for_query, + wasm_find_relationship_path_with_yaml, wasm_format_parameter_value_with_yaml, + wasm_generate_catalog_metadata_with_yaml, wasm_generate_preaggregation_definition, wasm_generate_preaggregation_materialization_sql_with_yaml, wasm_generate_preaggregation_name, wasm_generate_time_comparison_sql, wasm_interpolate_sql_with_parameters_with_yaml, wasm_is_relative_date, wasm_is_sql_template, wasm_load_graph_with_sql, @@ -128,7 +128,8 @@ pub use wasm::{ wasm_relationship_primary_key_columns_with_yaml, wasm_relationship_related_key_with_yaml, wasm_relationship_sql_expr_with_yaml, wasm_relative_date_to_range, wasm_render_sql_template, wasm_resolve_metric_inheritance, wasm_resolve_model_inheritance_with_yaml, - wasm_rewrite_with_yaml, wasm_segment_get_sql_with_yaml, wasm_summarize_preaggregation_patterns, + wasm_rewrite_with_semantic_input_context, wasm_rewrite_with_yaml, + wasm_segment_get_sql_with_yaml, wasm_summarize_preaggregation_patterns, wasm_time_comparison_offset_interval, wasm_time_comparison_sql_offset, wasm_trailing_period_sql_interval, wasm_validate_engine_refresh_sql_compatibility, wasm_validate_metric_payload, wasm_validate_model_payload, wasm_validate_models_yaml, diff --git a/sidemantic-rs/src/python.rs b/sidemantic-rs/src/python.rs index b5a63d2f6..add971dca 100644 --- a/sidemantic-rs/src/python.rs +++ b/sidemantic-rs/src/python.rs @@ -104,7 +104,7 @@ static REGISTRY_CONTEXTVAR: PyOnceLock> = PyOnceLock::new(); pyo3::create_exception!( sidemantic_rs, UnsupportedSemanticFeaturesError, - PyRuntimeError + PyValueError ); pyo3::create_exception!(sidemantic_rs, SecurityError, PyRuntimeError); pyo3::create_exception!(sidemantic_rs, QueryValidationError, PyValueError); @@ -124,9 +124,10 @@ fn semantic_input_error(py: Python<'_>, error: SidemanticError) -> PyErr { } error } - SidemanticError::Validation(_) | SidemanticError::ValidationIssue { .. } => { - QueryValidationError::new_err(error.to_string()) - } + SidemanticError::Validation(_) + | SidemanticError::ValidationIssue { .. } + | SidemanticError::NoJoinPath { .. } + | SidemanticError::SqlParse(_) => QueryValidationError::new_err(error.to_string()), SidemanticError::InvalidConfig(_) => PyValueError::new_err(error.to_string()), _ => PyRuntimeError::new_err(error.to_string()), } @@ -515,12 +516,14 @@ fn validate_query_references( /// Generate materialization SQL for a model pre-aggregation using sidemantic-rs schema. #[pyfunction] fn generate_preaggregation_materialization_sql( + py: Python<'_>, yaml: &str, model_name: &str, preagg_name: &str, ) -> PyResult { generate_preaggregation_materialization_sql_with_yaml_native(yaml, model_name, preagg_name) .map_err(|e| match e { + SidemanticError::UnsupportedSemanticFeatures { .. } => semantic_input_error(py, e), SidemanticError::Validation(_) | SidemanticError::YamlParse(_) | SidemanticError::InvalidConfig(_) diff --git a/sidemantic-rs/src/runtime.rs b/sidemantic-rs/src/runtime.rs index e3eb4b6f5..08bc44444 100644 --- a/sidemantic-rs/src/runtime.rs +++ b/sidemantic-rs/src/runtime.rs @@ -5804,6 +5804,12 @@ pub fn validate_query_references( let mut errors = Vec::new(); for metric_ref in metrics { + // Graph metric names are opaque, and may themselves contain dots. + if graph.get_metric(metric_ref).is_some() + || context.top_level_metric_names.contains(metric_ref) + { + continue; + } if let Some((model_name, metric_name)) = metric_ref.split_once('.') { if graph.get_model(model_name).is_none() { errors.push(format!( @@ -5852,11 +5858,10 @@ pub fn validate_query_references( )); continue; } - if graph - .get_model(model_name) - .and_then(|model| model.get_dimension(dim_name)) - .is_none() - { + if graph.get_model(model_name).is_some_and(|model| { + model.get_dimension(dim_name).is_none() + && !crate::core::semantic_key_names(graph, model).contains(dim_name) + }) { errors.push(format!( "Dimension '{dim_name}' not found in model '{model_name}' (referenced in '{dim_ref_for_lookup}')" )); @@ -5870,6 +5875,16 @@ pub fn validate_query_references( let mut model_names: BTreeSet = BTreeSet::new(); for metric_ref in metrics { + if graph.get_metric(metric_ref).is_some() + || context.top_level_metric_names.contains(metric_ref) + { + if let Some(sql_ref) = context.top_level_metric_sql_refs.get(metric_ref) { + if let Some((model_name, _)) = sql_ref.split_once('.') { + model_names.insert(model_name.to_string()); + } + } + continue; + } if let Some((model_name, _)) = metric_ref.split_once('.') { model_names.insert(model_name.to_string()); continue; @@ -5919,6 +5934,51 @@ mod tests { use super::*; use std::time::{SystemTime, UNIX_EPOCH}; + #[test] + fn validation_resolves_exact_graph_names_and_declared_relationship_keys() { + let yaml = r#" +models: + - name: orders + table: orders + primary_key: id + metrics: + - name: revenue + agg: sum + sql: amount + - name: order_count + agg: count + relationships: + - name: customers + type: many_to_one + foreign_key: customer_id + - name: customers + table: customers + primary_key: id +metrics: + - name: finance.revenue_per_order + type: ratio + numerator: orders.revenue + denominator: orders.order_count + - name: company.sales.revenue + sql: orders.revenue +"#; + let runtime = SidemanticRuntime::from_yaml(yaml).unwrap(); + for metric in ["finance.revenue_per_order", "company.sales.revenue"] { + let errors = + runtime.validate_query_references(&[metric.into()], &["orders.customer_id".into()]); + assert!(errors.is_empty(), "{errors:?}"); + } + let errors = runtime.validate_query_references( + &["company.sales.missing".into()], + &["orders.unknown_key".into()], + ); + assert_eq!(errors.len(), 2, "{errors:?}"); + assert!(errors.iter().any(|error| error.contains("unknown_key"))); + assert!(errors + .iter() + .any(|error| error.contains("company.sales.missing"))); + } + #[test] fn test_export_osi_yaml_accepts_out_of_order_graph_metrics() { // A ratio metric listed before the graph metrics it references must not diff --git a/sidemantic-rs/src/runtime/parameters.rs b/sidemantic-rs/src/runtime/parameters.rs index a808b829f..3d243b31b 100644 --- a/sidemantic-rs/src/runtime/parameters.rs +++ b/sidemantic-rs/src/runtime/parameters.rs @@ -91,6 +91,42 @@ fn string_literal(value: &str, dialect: DialectType) -> std::result::Result std::result::Result { + let text = text + .strip_prefix("E:") + .or_else(|| text.strip_prefix("e:")) + .ok_or("Invalid SQL escape-string token")?; + let mut characters = text.chars(); + let mut result = String::new(); + while let Some(character) = characters.next() { + if character != '\\' { + result.push(character); + continue; + } + let escaped = characters.next().ok_or("Invalid SQL escape-string token")?; + let decoded = match escaped { + '\\' | '\'' => escaped, + 'a' => '\u{0007}', + 'b' => '\u{0008}', + 'f' => '\u{000c}', + 'n' => '\n', + 'r' => '\r', + 't' => '\t', + 'v' => '\u{000b}', + other => { + // SQLGlot preserves escapes outside its recognized table. + result.push('\\'); + other + } + }; + result.push(decoded); + } + Ok(result) +} + /// Wrap bare-name output expressions so declared string/unquoted/yesno types /// remain available to the formatter. Tokenization leaves conditions, complex /// expressions, quoted Jinja strings, comments and raw blocks untouched. @@ -223,20 +259,24 @@ fn replace_outputs( .ok_or("Invalid SQL token span")?; if matches!( token.token_type, - TokenType::String | TokenType::DollarString | TokenType::ByteString + TokenType::String + | TokenType::DollarString + | TokenType::ByteString + | TokenType::EscapeString ) { let raw = sql.get(start..end).ok_or("Invalid SQL token span")?; - if token.token_type == TokenType::ByteString - || matches!( - dialect, - DialectType::MySQL - | DialectType::BigQuery - | DialectType::Snowflake - | DialectType::Spark - | DialectType::Databricks - | DialectType::Hive - ) - { + if matches!( + token.token_type, + TokenType::ByteString | TokenType::EscapeString + ) || matches!( + dialect, + DialectType::MySQL + | DialectType::BigQuery + | DialectType::Snowflake + | DialectType::Spark + | DialectType::Databricks + | DialectType::Hive + ) { let prefix = &sql[start..position]; if !(prefix.len() - prefix.trim_end_matches('\\').len()).is_multiple_of(2) { return Err( @@ -247,22 +287,28 @@ fn replace_outputs( let value = match literals.entry((start, end)) { std::collections::btree_map::Entry::Occupied(entry) => entry.into_mut(), std::collections::btree_map::Entry::Vacant(entry) => { - let normalized = dialects::fragment(raw, dialect, dialects::Fragment::Scalar) - .map_err(|error| error.to_string())?; - let expression = crate::core::parse_semantic_expression(&normalized) - .map_err(|error| error.to_string())?; - let text = match expression { - Expression::Literal(Literal::String(text)) => text, - Expression::Literal(Literal::DollarString(text)) => { - polyglot_sql::tokens::parse_dollar_string_token(&text).1 - } - _ => { - return Err( - "Parameter output requires an ordinary SQL string literal".into() - ) - } - }; - entry.insert(text) + if token.token_type == TokenType::EscapeString { + entry.insert(escape_string_text(&token.text)?) + } else { + let normalized = + dialects::fragment(raw, dialect, dialects::Fragment::Scalar) + .map_err(|error| error.to_string())?; + let expression = crate::core::parse_semantic_expression(&normalized) + .map_err(|error| error.to_string())?; + let text = match expression { + Expression::Literal(Literal::String(text)) => text, + Expression::Literal(Literal::DollarString(text)) => { + polyglot_sql::tokens::parse_dollar_string_token(&text).1 + } + _ => { + return Err( + "Parameter output requires an ordinary SQL string literal" + .into(), + ) + } + }; + entry.insert(text) + } } }; *value = value.replace(&output.marker, &output.text); @@ -291,6 +337,45 @@ mod tests { serde_json::from_value(json!({"name":name, "type":kind})).unwrap() } + #[test] + fn escape_string_outputs_keep_static_escapes_separate_from_values() { + for kind in ["string", "date"] { + let parameter = parameter("value", kind); + let definitions = HashMap::from([("value".to_owned(), ¶meter)]); + for dialect in [DialectType::DuckDB, DialectType::PostgreSQL] { + for value in ["signup", "\\' OR 1=1 -- ", "O'Reilly\\folder"] { + let values = HashMap::from([( + "value".to_owned(), + serde_yaml::Value::String(value.into()), + )]); + for (template, expected) in [ + ("{# c #}E'{{ value }}'", value.to_owned()), + ( + r"{# c #}e'é prefix\'{{ value }}'", + format!("é prefix'{value}"), + ), + ( + r"{# c #}E'line\n{{ value }}\\{{ value }}'", + format!("line\n{value}\\{value}"), + ), + ] { + let rendered = render(template, &definitions, &values, dialect).unwrap(); + assert_eq!(rendered, string_literal(&expected, dialect).unwrap()); + let expression = crate::core::parse_semantic_expression(&rendered).unwrap(); + assert_eq!(expression, Expression::Literal(Literal::String(expected))); + } + } + let values = + HashMap::from([("value".to_owned(), serde_yaml::Value::String("ok".into()))]); + for template in [r"{# c #}E'\{{ value }}'", r"{# c #}e'\\\{{ value }}'"] { + assert!(render(template, &definitions, &values, dialect) + .unwrap_err() + .contains("unpaired SQL escape")); + } + } + } + } + #[test] fn output_literals_preserve_quotes_backslashes_and_quoted_context() { let parameter = parameter("value", "string"); diff --git a/sidemantic-rs/src/semantic_input.rs b/sidemantic-rs/src/semantic_input.rs index f0df6a586..41b9bf2fb 100644 --- a/sidemantic-rs/src/semantic_input.rs +++ b/sidemantic-rs/src/semantic_input.rs @@ -20,6 +20,8 @@ use crate::sql::{QueryRewriter, SemanticQuery, SqlGenerator}; mod calculations; mod dates; pub(crate) mod dialects; +mod fragments; +mod inheritance; mod literals; mod policies; @@ -177,13 +179,20 @@ fn expression_language(raw: &mut Map, path: &str) -> Result<()> { } } } + // DAX source is provenance once the host has supplied an executable SQL, + // table, or simple aggregate translation. Never run untranslated DAX here. + let translated = ["sql", "table", "agg"].iter().any(|field| { + raw.get(*field) + .and_then(Value::as_str) + .is_some_and(|value| !value.is_empty()) + }); if let Some(value) = raw.remove("dax") { - if !value.is_null() { + if !value.is_null() && !translated { return Err(unsupported(format!("{path}.dax"))); } } if let Some(value) = raw.remove("expression_language") { - if !value.is_null() && value != "sql" { + if !value.is_null() && value != "sql" && !(value == "dax" && translated) { return Err(unsupported(format!("{path}.expression_language"))); } } @@ -517,6 +526,16 @@ fn decode_metric( }; raw.insert("type".into(), json!(kind)); } + // Imported definitions may retain `derived` alongside a concrete agg. + // The aggregation still consumes physical row inputs, as in Python. + if raw.get("type").and_then(Value::as_str) == Some("derived") + && raw + .get("agg") + .and_then(Value::as_str) + .is_some_and(|agg| agg != "expression") + { + raw.insert("type".into(), json!("simple")); + } if complete && raw.get("agg").is_some_and(|value| !value.is_null()) { return Err(invalid( path, @@ -552,7 +571,10 @@ fn decode_metric( exemplar.logical_data_type = Some(String::new()); exemplar.sql_is_complete = true; let metric = project(raw, exemplar, path)?; - if metric.agg == Some(crate::core::Aggregation::ApproxCountDistinct) && !model_local { + if metric.agg == Some(crate::core::Aggregation::ApproxCountDistinct) + && !model_local + && !(metric.r#type == crate::core::MetricType::Cohort && owner.is_some()) + { return Err(unsupported("metric.approx_count_distinct_model_scope")); } if metric.non_additive_dimension.is_some() { @@ -573,6 +595,16 @@ fn decode_metric( fn decode_relationship(value: Value, path: &str) -> Result { let mut raw = object(value, path)?; + // Python's native relationship contract reserves SQL join predicates for + // {from}/{to} expressions. A legacy SQL field such as "id" does not + // replace the declared FK/PK join with a bare boolean expression. + if raw + .get("sql") + .and_then(Value::as_str) + .is_some_and(|sql| !sql.contains("{from}") && !sql.contains("{to}")) + { + raw.remove("sql"); + } if raw.get("type") == Some(&json!("many_to_many")) && raw.get("through").is_none_or(Value::is_null) && raw.get("foreign_key").is_none_or(Value::is_null) @@ -603,14 +635,12 @@ fn decode_relationship(value: Value, path: &str) -> Result { fn decode_model(value: Value, path: &str) -> Result { let mut raw = object(value, path)?; expression_language(&mut raw, path)?; - for (field, capability) in [ - ("schema_exposure", "model.schema_exposure"), - ("auto_dimensions", "model.auto_dimensions"), - ("extends", "model.inheritance"), - ] { - reject_active(&mut raw, field, capability)?; - } + reject_active(&mut raw, "extends", "model.inheritance")?; // These declarations are retained and checked separately from core models. + // Schema exposure is applied by host introspection before dimensions enter + // the handoff. Native compilation uses only those explicit dimensions. + raw.remove("schema_exposure"); + raw.remove("auto_dimensions"); raw.remove("security"); raw.remove("invariant_filters"); if let Some(value) = raw.remove("pre_aggregations") { @@ -699,7 +729,7 @@ fn decode_model(value: Value, path: &str) -> Result { .as_deref() .is_none_or(|sql| sql.is_empty() || sql == "*") { - if primary_keys.len() != 1 { + if primary_keys.is_empty() { return Err(unsupported("metric.count_distinct_primary_key")); } // Keep the default distinct input distinct from explicit raw SQL. @@ -731,16 +761,65 @@ fn validate_semantic_dependencies(graph: &SemanticGraph, graph_metrics: &[Metric let mut dependencies: HashMap> = HashMap::new(); for (name, metric, context) in definitions { let mut metric_dependencies = Vec::new(); + let source_qualifier = context + .and_then(|owner| graph.get_model(owner)) + .filter(|model| model.sql.is_none()) + .and_then(|model| model.table.as_deref()) + .map(|table| dialects::parse(&format!("SELECT * FROM {table}"), DialectType::DuckDB)) + .transpose()? + .and_then(|expression| match expression { + polyglot_sql::Expression::Select(select) => select.from, + _ => None, + }) + .and_then(|from| from.expressions.into_iter().next()) + .and_then(|expression| match expression { + polyglot_sql::Expression::Table(table) => Some(table.name.name), + _ => None, + }); + let deferred_input_aliases: Vec<&str> = metric + .metadata + .as_ref() + .and_then(|metadata| metadata.get("input_metrics")) + .and_then(Value::as_array) + .into_iter() + .flatten() + .filter(|input| { + ["offset_window", "offset_to_grain", "filter"] + .iter() + .any(|field| input.get(*field).is_some_and(|value| !value.is_null())) + }) + .filter_map(|input| input.get("alias").and_then(Value::as_str)) + .collect(); + if metric.r#type == crate::core::MetricType::Cumulative { + // Cumulative SQL is a base metric reference, not a scalar SQL + // expression. Imports can retain unresolved bases; the temporal + // generator resolves them when selected. Keep known edges here so + // cycles remain invalid even before a query is compiled. + if let Some(reference) = metric.sql.as_ref().or(metric.base_metric.as_ref()) { + let mut dependency = reference.clone(); + if let Some(model_name) = context { + if graph + .get_model(model_name) + .is_some_and(|model| model.get_metric(reference).is_some()) + { + dependency = format!("{model_name}.{reference}"); + } + } + metric_dependencies.push(dependency); + } + } // `base` is the generated period-output relation, not a model. Resolve // its metric name with the same ownership and cycle rules as other refs. let window_dependency = SqlGenerator::window_output_dependency(metric)?; for expression in [ // Cohort SQL consumes inner-result aliases; the cohort generator // validates that separate namespace instead of metric dependencies. - metric - .sql - .as_deref() - .filter(|_| metric.r#type != crate::core::MetricType::Cohort), + metric.sql.as_deref().filter(|_| { + !matches!( + metric.r#type, + crate::core::MetricType::Cohort | crate::core::MetricType::Cumulative + ) + }), metric.numerator.as_deref(), metric.denominator.as_deref(), window_dependency.as_deref(), @@ -748,21 +827,42 @@ fn validate_semantic_dependencies(graph: &SemanticGraph, graph_metrics: &[Metric .into_iter() .flatten() { - for column in crate::core::semantic_column_references(expression)? { - let model_name = column.model.as_deref().or(context); + let expression = crate::core::replace_model_placeholder(expression, context)?; + for column in crate::core::semantic_column_references(&expression)? { + // MetricFlow retains modified input aliases for round-trip; + // they are not ordinary metric definitions. Unrelated queries + // remain valid, while selected derived SQL still resolves every + // input through the generator and rejects unsupported aliases. + if column.model.is_none() && deferred_input_aliases.contains(&column.field.as_str()) + { + continue; + } + let physical_input = metric.r#type == crate::core::MetricType::Simple + || metric.agg.is_some() + || metric.sql_is_complete + || column.aggregate_input; + // Complete aggregates may use the generated source CTE qualifier. + // Resolve that alias only for physical inputs, and prefer a real + // model of that name when one exists. + let model_name = column.model.as_deref().or(context).map(|name| { + if physical_input && graph.get_model(name).is_none() { + if source_qualifier.as_deref() == Some(name) { + context.unwrap_or(name) + } else { + name.strip_suffix("_cte").unwrap_or(name) + } + } else { + name + } + }); if let Some(model_name) = model_name { - let model = graph + graph .get_model(model_name) .ok_or_else(|| invalid(&name, format!("unknown model '{model_name}'")))?; - if column.aggregate_input - && model - .get_dimension(&column.field) - .is_some_and(|dimension| dimension.sql_expr() != column.field) - { - return Err(unsupported("metric.raw_computed_column")); - } + // Complete aggregate SQL names physical columns. A semantic + // dimension with the same name does not redefine that input. } - if metric.r#type == crate::core::MetricType::Simple || column.aggregate_input { + if physical_input { continue; } let dependency = if let Some(model_name) = &column.model { @@ -854,6 +954,7 @@ impl SemanticInput { return Err(invalid("version", "supported semantic input version is 1")); } let input_dialect = dialects::parse_dialect(&envelope.input_dialect)?; + envelope.models = inheritance::resolve(&envelope.models)?; let policies = policies::decode_with_dialect(&envelope.models, input_dialect)?; let deferred_segment_dialects = dialects::normalize(&mut envelope, input_dialect)?; let unsupported_capabilities: Vec = envelope @@ -877,6 +978,8 @@ impl SemanticInput { | "relationship.inactive" | "model.security" | "model.invariant_filters" + | "model.schema_exposure" + | "preaggregation.lambda" | "visibility" ) }) @@ -928,10 +1031,10 @@ impl SemanticInput { continue; } if !keys.contains_key(relationship.related_model()) { - return Err(invalid( - "relationships", - format!("unknown target {}", relationship.related_model()), - )); + // Models can be loaded independently of their join targets. + // The graph retains this declaration but only builds edges + // between loaded models; queries still resolve every reference. + continue; } if relationship.r#type == crate::core::RelationshipType::ManyToMany && relationship.through.is_some() @@ -985,9 +1088,13 @@ impl SemanticInput { // An explicit primary_key records a local/remote key pair. (foreign, primary) }; - if relationship.sql.is_none() - && (local.is_empty() || remote.is_empty() || local.len() != remote.len()) - { + if relationship.sql.is_none() && (local.is_empty() || remote.is_empty()) { + // An incomplete direct edge is not traversable, but it + // must not block queries against the local model. + relationship.active = false; + continue; + } + if relationship.sql.is_none() && local.len() != remote.len() { return Err(invalid( "relationships", "direct many-to-many join key arity mismatch", @@ -1015,8 +1122,7 @@ impl SemanticInput { if primary.is_empty() { let primary_model = if relationship.r#type == crate::core::RelationshipType::OneToMany - || (relationship.target_model.is_some() - && relationship.r#type == crate::core::RelationshipType::OneToOne) + || relationship.r#type == crate::core::RelationshipType::OneToOne { &model.name } else { @@ -1109,6 +1215,8 @@ struct QueryInput { #[serde(default)] use_preaggregations: bool, #[serde(default)] + allow_non_additive_unsafe: bool, + #[serde(default)] skip_default_time_dimensions: bool, preagg_database: Option, preagg_schema: Option, @@ -1140,12 +1248,45 @@ fn prepare_query_input(mut query: QueryInput, input: &mut SemanticInput) -> Resu ) .map_err(|error| invalid("query.parameter_values", error))? .iter() - .map(|sql| dialects::fragment(sql, dialect, dialects::Fragment::Scalar)) + .map(|sql| { + fragments::validate_request_expression(sql, dialect, false)?; + dialects::fragment(sql, dialect, dialects::Fragment::Scalar) + }) .collect::>>()?; + let known_names: Vec<_> = query + .aliases + .values() + .chain(query.metrics.iter()) + .chain(query.dimensions.iter()) + .map(String::as_str) + .collect(); query.order_by = query .order_by .iter() - .map(|sql| dialects::fragment(sql, dialect, dialects::Fragment::Order)) + .map(|sql| { + // Selected references and output aliases are names, not authored SQL. + // Bind before dialect parsing and policy collection so spaces and + // ordering keywords inside an alias retain their literal meaning. + let (field, suffix) = crate::sql::split_order_field(sql, &known_names); + // Keep canonical semantic names in the request. Specialized + // planners and table calculations bind these names before rendering + // SQL identifiers; quoting here changes the lookup key. + let render_reference = + |reference: &str| format!("{reference} {suffix}").trim_end().to_owned(); + if let Some((reference, _)) = query.aliases.iter().find(|(_, alias)| *alias == field) { + return Ok(render_reference(reference)); + } + if query + .metrics + .iter() + .chain(&query.dimensions) + .any(|name| name == field) + { + return Ok(render_reference(field)); + } + fragments::validate_request_expression(sql, dialect, true)?; + dialects::fragment(sql, dialect, dialects::Fragment::Order) + }) .collect::>>()?; let mut prepared_segments = std::collections::HashSet::new(); for reference in &query.segments { @@ -1235,6 +1376,7 @@ fn compile_semantic_input(input_json: &str, query_json: &str) -> Result offset: payload.offset, ungrouped: payload.ungrouped, use_preaggregations: payload.use_preaggregations, + allow_non_additive_unsafe: payload.allow_non_additive_unsafe, skip_default_time_dimensions: payload.skip_default_time_dimensions, preagg_database: payload.preagg_database, preagg_schema: payload.preagg_schema, @@ -1285,14 +1427,18 @@ fn validate_semantic_input(input_json: &str, query_json: &str) -> Result Result>, #[serde(default)] enforce_visibility: bool, + #[serde(default)] + use_preaggregations: bool, + #[serde(default)] + allow_non_additive_unsafe: bool, } pub fn rewrite_with_semantic_input_context( @@ -1404,6 +1555,8 @@ fn rewrite_semantic_input_diagnostics( .any(|policy| policy.security.is_some() || !policy.invariant_filters.is_empty()); let requires_policies = context.user_attributes.is_some() || security_controls; let prepare = |graph: &SemanticGraph, query: &mut SemanticQuery| { + query.use_preaggregations = context.use_preaggregations; + query.allow_non_additive_unsafe = context.allow_non_additive_unsafe; query.prepared_policies = policies::prepare_for_rewrite( graph, &input.policies, @@ -1419,7 +1572,7 @@ fn rewrite_semantic_input_diagnostics( // their source names too, before the rewriter allocates any user CTE. let policy_definitions = serde_json::to_string(&input.policies) .map_err(|error| invalid("rewrite.policy_definitions", error))?; - if requires_policies { + if requires_policies || context.use_preaggregations || context.allow_non_additive_unsafe { rewriter = rewriter.with_query_preparer(&prepare, &policy_definitions, security_controls); } @@ -1445,6 +1598,245 @@ mod tests { }) } + #[test] + fn unloaded_relationship_targets_do_not_block_local_queries() { + let mut source = input(); + source["models"][0]["relationships"] = json!([ + {"name":"customers", "type":"many_to_one", "foreign_key":"customer_id"} + ]); + let sql = + compile_with_semantic_input(&source.to_string(), r#"{"metrics":["orders.revenue"]}"#) + .unwrap(); + assert!(!sql.contains("JOIN"), "{sql}"); + assert!(compile_with_semantic_input( + &source.to_string(), + r#"{"metrics":["orders.revenue"],"dimensions":["customers.region"]}"#, + ) + .is_err()); + } + + #[test] + fn translated_dax_remains_provenance_without_blocking_sql() { + let mut source = input(); + source["models"][0]["dax"] = json!("FILTER(Orders, TRUE())"); + source["models"][0]["expression_language"] = json!("dax"); + source["models"][0]["dimensions"][0]["dax"] = json!("UPPER(Orders[status])"); + source["models"][0]["dimensions"][0]["expression_language"] = json!("dax"); + source["models"][0]["dimensions"][0]["sql"] = json!("UPPER(status)"); + source["models"][0]["metrics"][0]["dax"] = json!("SUM(Orders[amount])"); + source["models"][0]["metrics"][0]["expression_language"] = json!("dax"); + let sql = + compile_with_semantic_input(&source.to_string(), r#"{"metrics":["orders.revenue"]}"#) + .unwrap(); + assert!(sql.contains("SUM("), "{sql}"); + assert_eq!( + SemanticInput::from_json(&source.to_string()) + .unwrap() + .source, + source + ); + let metric = source["models"][0]["metrics"][0].as_object_mut().unwrap(); + metric.remove("sql"); + metric.remove("agg"); + assert!(matches!( + SemanticInput::from_json(&source.to_string()), + Err(SidemanticError::UnsupportedSemanticFeatures { .. }) + )); + } + + #[test] + fn incomplete_direct_many_to_many_does_not_block_local_queries() { + let mut source = input(); + source["models"][0]["relationships"] = + json!([{"name":"regions", "type":"many_to_many", "foreign_key":"order_id"}]); + source["models"].as_array_mut().unwrap().push(json!({ + "name":"regions", "table":"regions", "primary_key":"order_id", + "dimensions":[{"name":"label", "type":"categorical"}] + })); + let sql = + compile_with_semantic_input(&source.to_string(), r#"{"metrics":["orders.revenue"]}"#) + .unwrap(); + assert!(!sql.contains("JOIN"), "{sql}"); + assert!(compile_with_semantic_input( + &source.to_string(), + r#"{"metrics":["orders.revenue"],"dimensions":["regions.label"]}"# + ) + .is_err()); + assert_eq!( + SemanticInput::from_json(&source.to_string()) + .unwrap() + .source, + source + ); + } + + #[test] + fn dependency_validation_binds_model_tokens_without_rewriting_literals() { + let mut source = input(); + source["models"][0]["metrics"] = json!([ + {"name":"average", "sql":"sum({model}.amount) / count(*)", "sql_is_complete":true}, + {"name":"literal", "sql":"sum(CASE WHEN '{model}.missing' = 'x' THEN 1 ELSE 0 END)", "sql_is_complete":true} + ]); + let query = r#"{"metrics":["orders.average"]}"#; + assert!(validate_with_semantic_input(&source.to_string(), query) + .unwrap() + .is_empty()); + } + + #[test] + fn unresolved_cumulative_base_only_blocks_queries_that_select_it() { + let mut source = input(); + source["models"][0]["metrics"] + .as_array_mut() + .unwrap() + .push(json!({ + "name":"running", "type":"cumulative", "agg":"sum", "sql":"hours" + })); + source["models"][0]["dimensions"] + .as_array_mut() + .unwrap() + .push(json!({ + "name":"day", "type":"time", "granularity":"day" + })); + let sql = + compile_with_semantic_input(&source.to_string(), r#"{"metrics":["orders.revenue"]}"#) + .unwrap(); + assert!(sql.contains("SUM("), "{sql}"); + assert!(compile_with_semantic_input( + &source.to_string(), + r#"{"metrics":["orders.running"],"dimensions":["orders.day"]}"#, + ) + .is_err()); + source["models"][0]["metrics"] + .as_array_mut() + .unwrap() + .push(json!({ + "name":"hours", "type":"cumulative", "sql":"running" + })); + assert!(matches!( + SemanticInput::from_json(&source.to_string()), + Err(SidemanticError::CircularDependency(_)) + )); + } + + #[test] + fn deferred_imported_input_alias_does_not_block_unrelated_metrics() { + let mut source = input(); + source["models"][0]["metrics"].as_array_mut().unwrap().push(json!({ + "name":"growth", "type":"derived", "sql":"revenue - prior_revenue", + "metadata":{"input_metrics":[{"name":"revenue", "alias":"prior_revenue", "offset_window":"7 days"}]} + })); + let sql = + compile_with_semantic_input(&source.to_string(), r#"{"metrics":["orders.revenue"]}"#) + .unwrap(); + assert!(sql.contains("SUM("), "{sql}"); + assert!(compile_with_semantic_input( + &source.to_string(), + r#"{"metrics":["orders.growth"]}"# + ) + .is_err()); + assert_eq!( + SemanticInput::from_json(&source.to_string()) + .unwrap() + .source, + source + ); + } + + #[test] + fn caller_filters_cannot_borrow_trusted_segment_or_policy_scope() { + let predicate = "status IN (SELECT status FROM allowed_statuses)"; + for temporal in [false, true] { + let mut source = input(); + source["models"][0]["segments"] = json!([{"name":"allowed", "sql":predicate}]); + source["models"][0]["dimensions"] + .as_array_mut() + .unwrap() + .push(json!({ + "name":"day", "type":"time", "granularity":"day" + })); + source["models"][0]["metrics"] + .as_array_mut() + .unwrap() + .push(json!({ + "name":"running", "type":"cumulative", "sql":"orders.revenue" + })); + let mut query = if temporal { + json!({"metrics":["orders.running"], "dimensions":["orders.day"]}) + } else { + json!({"metrics":["orders.revenue"]}) + }; + query["segments"] = json!(["orders.allowed"]); + let sql = compile_with_semantic_input(&source.to_string(), &query.to_string()).unwrap(); + assert!(sql.contains("allowed_statuses"), "{sql}"); + query.as_object_mut().unwrap().remove("segments"); + source["models"][0]["invariant_filters"] = json!([predicate]); + let sql = compile_with_semantic_input(&source.to_string(), &query.to_string()).unwrap(); + assert!(sql.contains("allowed_statuses"), "{sql}"); + query["filters"] = json!([predicate]); + for result in [ + compile_with_semantic_input(&source.to_string(), &query.to_string()).map(|_| ()), + validate_with_semantic_input(&source.to_string(), &query.to_string()).map(|_| ()), + ] { + let error = result.unwrap_err(); + assert!( + error.to_string().contains("physical data sources"), + "{error}" + ); + } + } + } + + #[test] + fn complete_ordered_set_aggregate_uses_physical_sort_input() { + let mut source = input(); + source["models"][0]["metrics"] = json!([{ + "name":"p95", "sql_is_complete":true, + "sql":"PERCENTILE_CONT(0.95) WITHIN GROUP (ORDER BY orders.amount)" + }]); + let sql = compile_with_semantic_input(&source.to_string(), r#"{"metrics":["orders.p95"]}"#) + .unwrap(); + assert!( + sql.contains("WITHIN GROUP") || sql.contains("QUANTILE_CONT"), + "{sql}" + ); + assert!(sql.contains("amount"), "{sql}"); + } + + #[test] + fn physical_metric_input_can_use_its_declared_source_table_name() { + let mut source = input(); + source["models"][0]["table"] = json!("sales"); + source["models"][0]["metrics"][0]["sql"] = json!("sales.amount"); + let sql = + compile_with_semantic_input(&source.to_string(), r#"{"metrics":["orders.revenue"]}"#) + .unwrap(); + assert!(sql.contains("amount AS revenue_raw"), "{sql}"); + assert!(sql.contains("SUM(orders_cte.revenue_raw)"), "{sql}"); + assert!(sql.contains("FROM sales"), "{sql}"); + source["models"][0]["metrics"][0]["sql"] = json!("other.amount"); + assert!(SemanticInput::from_json(&source.to_string()).is_err()); + } + + #[test] + fn schema_exposure_uses_only_host_discovered_dimensions() { + let mut source = input(); + source["required_capabilities"] = json!(["model.schema_exposure"]); + source["models"][0]["schema_exposure"] = json!({"private":["secret"]}); + source["models"][0]["auto_dimensions"] = json!(true); + let sql = + compile_with_semantic_input(&source.to_string(), r#"{"dimensions":["orders.status"]}"#) + .unwrap(); + assert!(!sql.contains("secret"), "{sql}"); + assert!(compile_with_semantic_input( + &source.to_string(), + r#"{"dimensions":["orders.secret"]}"#, + ) + .is_err()); + let decoded = SemanticInput::from_json(&source.to_string()).unwrap(); + assert_eq!(decoded.source, source); + } + #[test] fn resolved_explore_anchor_is_shared_by_compilation_and_validation() { let mut source = input(); @@ -1580,6 +1972,83 @@ mod tests { assert_eq!(model.get_segment("unused").unwrap().sql, "{% if missing"); } + #[test] + fn selected_order_names_remain_canonical_before_sql_rendering() { + let mut source = input(); + source["models"][0]["dimensions"] = json!([ + {"name":"Order Status", "type":"categorical", "sql":"status"}, + {"name":"Rank DESC", "type":"categorical", "sql":"status"} + ]); + let mut input = SemanticInput::decode_scoped(&source.to_string(), true).unwrap(); + for (order, expected) in [ + ("orders.Order Status DESC", "orders.Order Status DESC"), + ("orders.Rank DESC", "orders.Rank DESC"), + ("Public Status ASC", "orders.Order Status ASC"), + ] { + let request = json!({ + "metrics":["orders.revenue"], + "dimensions":["orders.Order Status", "orders.Rank DESC"], + "aliases":{"orders.Order Status":"Public Status"}, + "order_by":[order] + }) + .to_string(); + let query = prepare_query_input(query_input(&request).unwrap(), &mut input).unwrap(); + assert_eq!(query.order_by, vec![expected]); + // Exercise policy dependency parsing and final output binding too. + let sql = compile_with_semantic_input(&source.to_string(), &request).unwrap(); + assert!(sql.contains("ORDER BY"), "{sql}"); + } + } + + #[test] + fn explicit_aggregation_and_authored_windows_accept_physical_columns() { + let mut source = input(); + source["models"][0]["metrics"] = json!([ + {"name":"value", "type":"derived", "agg":"sum", "sql":"amount * quantity"}, + {"name":"running", "type":"derived", "sql":"sum(amount) over (order by year)"} + ]); + let decoded = SemanticInput::from_json(&source.to_string()).unwrap(); + assert_eq!( + decoded + .graph + .get_model("orders") + .unwrap() + .get_metric("value") + .unwrap() + .r#type, + crate::core::MetricType::Simple, + ); + let sql = + compile_with_semantic_input(&source.to_string(), r#"{"metrics":["orders.value"]}"#) + .unwrap(); + assert!(sql.contains("SUM("), "{sql}"); + assert!(sql.contains("quantity"), "{sql}"); + } + + #[test] + fn dotted_graph_aggregate_name_declares_an_existing_source_model() { + let mut source = input(); + source["metrics"] = json!([ + {"name":"orders.p95.amount", "agg":"max", "sql":"amount"} + ]); + let sql = compile_with_semantic_input( + &source.to_string(), + &json!({ + "metrics":["orders.p95.amount"], + "order_by":["orders.p95.amount DESC"] + }) + .to_string(), + ) + .unwrap(); + assert!(sql.contains("\"orders.p95.amount\" DESC"), "{sql}"); + source["metrics"][0]["name"] = json!("missing.p95.amount"); + assert!(compile_with_semantic_input( + &source.to_string(), + r#"{"metrics":["missing.p95.amount"]}"#, + ) + .is_err()); + } + #[test] fn handoff_preserves_unknown_keys_and_original_declarations() { let source = input(); @@ -1612,10 +2081,10 @@ mod tests { )); } let mut source = input(); - source["models"][0]["schema_exposure"] = json!({}); + source["models"][0]["extends"] = json!("base_orders"); assert!(matches!( SemanticInput::from_json(&source.to_string()), - Err(SidemanticError::UnsupportedSemanticFeatures { .. }) + Err(SidemanticError::ValidationIssue { .. }) )); let mut source = input(); source["models"][0]["invariant_filters"] = json!(["tenant_id = 1"]); @@ -1653,6 +2122,7 @@ mod tests { .remove("unexpected"); source["models"][0]["pre_aggregations"][0]["union_with_source_data"] = json!(true); source["models"][0]["pre_aggregations"][0]["rollups"] = json!(["orders.historical"]); + source["required_capabilities"] = json!(["preaggregation.lambda"]); let decoded = SemanticInput::from_json(&source.to_string()).unwrap(); assert!( decoded.graph.get_model("orders").unwrap().pre_aggregations[0].union_with_source_data @@ -1664,6 +2134,82 @@ mod tests { ); } + #[test] + fn sql_rewrite_preaggregations_are_opt_in_and_respect_policies() { + let mut source = input(); + source["models"][0]["pre_aggregations"] = json!([ + {"name":"total", "measures":["revenue"]} + ]); + for query in [ + "SELECT revenue FROM orders", + "SELECT revenue AS total FROM orders", + "SELECT * FROM (SELECT revenue FROM orders) AS totals", + ] { + let sql = + rewrite_with_semantic_input_context(&source.to_string(), query, "{}").unwrap(); + assert!(!sql.contains("orders_preagg_total"), "{sql}"); + assert!(!sql.contains("used_preagg=true"), "{sql}"); + dialects::parse(&sql, DialectType::DuckDB).unwrap(); + let sql = rewrite_with_semantic_input_context( + &source.to_string(), + query, + r#"{"use_preaggregations":true}"#, + ) + .unwrap(); + assert!(sql.contains("orders_preagg_total"), "{sql}"); + assert!(sql.contains("used_preagg=true"), "{sql}"); + dialects::parse(&sql, DialectType::DuckDB).unwrap(); + } + source["models"][0]["invariant_filters"] = json!(["not deleted"]); + let sql = rewrite_with_semantic_input_context( + &source.to_string(), + "SELECT revenue FROM orders", + r#"{"use_preaggregations":true}"#, + ) + .unwrap(); + assert!(!sql.contains("orders_preagg_total"), "{sql}"); + assert!(!sql.contains("used_preagg=true"), "{sql}"); + dialects::parse(&sql, DialectType::DuckDB).unwrap(); + assert!(sql.contains("NOT deleted"), "{sql}"); + } + + #[test] + fn raw_inheritance_preserves_source_and_enforces_inherited_policies() { + let mut source = input(); + source["models"][0]["security"] = json!({"row_filters":["tenant = {{ user.tenant }}"]}); + source["models"][0]["invariant_filters"] = json!(["not deleted"]); + source["models"].as_array_mut().unwrap().push(json!({ + "name":"vip_orders", "extends":"orders", "security":null, + "invariant_filters":["amount > 100"] + })); + let decoded = SemanticInput::from_json(&source.to_string()).unwrap(); + assert_eq!(decoded.source, source); + let parent = decoded.graph.get_model("orders").unwrap(); + let child = decoded.graph.get_model("vip_orders").unwrap(); + assert_eq!(child.table, parent.table); + assert_eq!(child.metrics.len(), parent.metrics.len()); + assert!(matches!( + compile_with_semantic_input( + &source.to_string(), + r#"{"metrics":["vip_orders.revenue"]}"# + ), + Err(SidemanticError::Security(_)) + )); + let sql = compile_with_semantic_input( + &source.to_string(), + r#"{"metrics":["vip_orders.revenue"],"user_attributes":{"tenant":1}}"#, + ) + .unwrap(); + assert!(sql.contains("tenant = 1"), "{sql}"); + assert!(sql.contains("NOT deleted"), "{sql}"); + assert!(sql.contains("amount > 100"), "{sql}"); + source["models"][1]["unexpected"] = json!(true); + assert!(matches!( + SemanticInput::from_json(&source.to_string()), + Err(SidemanticError::ValidationIssue { .. }) + )); + } + #[test] fn handoff_policies_are_enforced_and_cannot_be_forged() { let mut source = input(); @@ -1975,6 +2521,40 @@ mod tests { assert!(sql.contains("NULLIF")); } + #[test] + fn legacy_relationship_sql_keeps_keyed_join_in_both_directions() { + let mut source = input(); + source["models"][0]["primary_key"] = json!("id"); + source["models"][0]["relationships"] = json!([ + {"name":"items", "type":"one_to_many", "sql":"id", "foreign_key":"order_id"} + ]); + source["models"].as_array_mut().unwrap().push(json!({ + "name":"items", "table":"items", "primary_key":"item_id", + "dimensions":[{"name":"category", "type":"categorical"}] + })); + let decoded = SemanticInput::from_json(&source.to_string()).unwrap(); + for (from, to, local, remote) in [ + ("orders", "items", "id", "order_id"), + ("items", "orders", "order_id", "id"), + ] { + let path = decoded.graph.find_join_path(from, to).unwrap(); + assert_eq!(path.steps[0].from_keys, vec![local]); + assert_eq!(path.steps[0].to_keys, vec![remote]); + assert!(path.steps[0].custom_condition.is_none()); + } + let sql = compile_with_semantic_input( + &source.to_string(), + r#"{"metrics":["orders.revenue"],"dimensions":["items.category"]}"#, + ) + .unwrap(); + // Either operand may anchor the query's dimension population. + assert!( + sql.contains("orders_cte.id = items_cte.order_id") + || sql.contains("items_cte.order_id = orders_cte.id"), + "{sql}" + ); + } + #[test] fn handoff_relationships_preserve_composite_keys_and_roles() { let mut source = input(); @@ -2057,10 +2637,10 @@ mod tests { } #[test] - fn count_distinct_uses_only_a_known_single_key() { + fn count_distinct_uses_known_single_and_composite_keys() { let mut source = input(); source["models"][0]["metrics"] = json!([{"name":"unique_orders", "agg":"count_distinct"}]); - for key in [Value::Null, json!(["tenant_id", "order_id"])] { + for key in [Value::Null, json!([])] { source["models"][0]["primary_key"] = key; assert!(matches!( SemanticInput::from_json(&source.to_string()), @@ -2076,6 +2656,20 @@ mod tests { assert!(sql.contains("COUNT(DISTINCT")); assert!(sql.contains("order_id")); assert!(!sql.contains("CONCAT")); + source["models"][0]["primary_key"] = json!(["tenant_key", "order_key"]); + source["models"][0]["dimensions"] = json!([ + {"name":"tenant_key", "type":"numeric", "sql":"tenant_id + 10"}, + {"name":"order_key", "type":"numeric", "sql":"order_id + 100"} + ]); + let sql = compile_with_semantic_input( + &source.to_string(), + r#"{"metrics":["orders.unique_orders"]}"#, + ) + .unwrap(); + assert!(sql.contains("COUNT(DISTINCT"), "{sql}"); + assert!(sql.contains("CONCAT"), "{sql}"); + assert!(sql.contains("tenant_id + 10"), "{sql}"); + assert!(sql.contains("order_id + 100"), "{sql}"); } #[test] @@ -2484,11 +3078,62 @@ mod tests { source["models"][0]["metrics"][0]["sql"] = json!("SUM(amount)"); source["models"][0]["dimensions"] = json!([{"name":"amount", "type":"numeric", "sql":"amount * 10"}]); - let error = SemanticInput::from_json(&source.to_string()).err(); + let sql = + compile_with_semantic_input(&source.to_string(), r#"{"metrics":["orders.paid"]}"#) + .unwrap(); + assert!(sql.contains("SUM("), "{sql}"); + assert!(!sql.contains("* 10"), "{sql}"); + } + + #[test] + fn compilation_binds_model_placeholders_in_metric_dependency_walks() { + for metric in [ + json!({"name":"value", "agg":"sum", "sql":"({model}.amount)"}), + json!({"name":"value", "sql":"SUM({model}.amount)", "sql_is_complete":true}), + json!({"name":"value", "type":"derived", "sql":"{model}.revenue * 2"}), + ] { + let mut source = input(); + source["models"][0]["metrics"] + .as_array_mut() + .unwrap() + .push(metric); + let sql = + compile_with_semantic_input(&source.to_string(), r#"{"metrics":["orders.value"]}"#) + .unwrap(); + assert!(!sql.contains("{model}"), "{sql}"); + assert!(sql.contains("orders_cte"), "{sql}"); + } + } + + #[test] + fn complete_aggregates_accept_source_cte_aliases_without_weakening_metric_binding() { + let mut source = input(); + source["models"][0]["metrics"] = json!([ + {"name":"average", "sql":"AVG(orders_cte.amount)", "sql_is_complete":true} + ]); + let sql = + compile_with_semantic_input(&source.to_string(), r#"{"metrics":["orders.average"]}"#) + .unwrap(); + // Complete aggregates may project their physical inputs into an inner + // population before averaging. Validate the resulting query, not that + // planner's temporary input name. + assert!(sql.contains("AVG("), "{sql}"); assert!( - matches!(&error, Some(SidemanticError::UnsupportedSemanticFeatures { capabilities }) if capabilities == &vec!["metric.raw_computed_column"]), - "{error:?}" + polyglot_sql::parse_one(&sql, DialectType::DuckDB).is_ok(), + "{sql}" ); + + source["models"][0]["metrics"][0]["sql"] = json!("AVG(missing_cte.amount)"); + assert!(matches!( + SemanticInput::from_json(&source.to_string()), + Err(SidemanticError::ValidationIssue { .. }) + )); + source["models"][0]["metrics"][0] = + json!({"name":"average", "type":"derived", "sql":"orders_cte.revenue"}); + assert!(matches!( + SemanticInput::from_json(&source.to_string()), + Err(SidemanticError::ValidationIssue { .. }) + )); } #[test] diff --git a/sidemantic-rs/src/semantic_input/dates.rs b/sidemantic-rs/src/semantic_input/dates.rs index d5d4267fc..a7fb27691 100644 --- a/sidemantic-rs/src/semantic_input/dates.rs +++ b/sidemantic-rs/src/semantic_input/dates.rs @@ -18,8 +18,7 @@ pub(super) fn normalize_transpiled( if source != DialectType::Snowflake || target != DialectType::DuckDB { return Ok(sql.to_owned()); } - let expression = polyglot_sql::parse_one(sql, target) - .map_err(|error| SidemanticError::SqlParse(error.to_string()))?; + let expression = super::dialects::parse(sql, target)?; let convert_error = |error: serde_json::Error| SidemanticError::SqlGeneration(error.to_string()); let mut value = serde_json::to_value(expression).map_err(convert_error)?; diff --git a/sidemantic-rs/src/semantic_input/dialects.rs b/sidemantic-rs/src/semantic_input/dialects.rs index 17fe9d993..cf37115d5 100644 --- a/sidemantic-rs/src/semantic_input/dialects.rs +++ b/sidemantic-rs/src/semantic_input/dialects.rs @@ -86,8 +86,164 @@ pub(crate) fn parse(sql: &str, source: DialectType) -> Result { let sql = super::literals::source_sql(sql, source)?; #[cfg(target_arch = "wasm32")] crate::wasm_sql_guard::check(&sql, source)?; - polyglot_sql::parse_one(&sql, source) - .map_err(|error| SidemanticError::SqlParse(error.to_string())) + let mut statements = parse_many(&sql, source)?; + if statements.len() != 1 { + return Err(SidemanticError::SqlParse( + "Expected one SQL statement".into(), + )); + } + Ok(statements.remove(0)) +} + +/// QUANTILE_CONT/DISC are absent from the pinned parser's aggregate registry. +/// Parse their argument lists with its generic aggregate grammar, then restore +/// the actual name in the AST. ORDER BY, DISTINCT, FILTER and window clauses +/// must survive; deleting the aggregate ordering changes percentile semantics. +pub(crate) fn parse_many(sql: &str, dialect: DialectType) -> Result> { + use polyglot_sql::expressions::AggregateFunction; + use polyglot_sql::tokens::TokenType; + + let mut replacements: HashMap = HashMap::new(); + let mut prepared = sql.to_owned(); + if sql.to_ascii_uppercase().contains("QUANTILE_") { + let stream = Dialect::get(dialect) + .tokenize(sql) + .map_err(|error| SidemanticError::SqlParse(error.to_string()))?; + let offsets = sql + .char_indices() + .map(|(index, _)| index) + .chain([sql.len()]) + .collect::>(); + let mut prefix = "__sidemantic_quantile_".to_owned(); + while sql.to_ascii_lowercase().contains(&prefix) { + prefix.push('_'); + } + prepared.clear(); + let mut cursor = 0; + let mut index = 0; + while index + 1 < stream.len() { + let token = &stream[index]; + if token.token_type == TokenType::Identifier + || !["QUANTILE_CONT", "QUANTILE_DISC"] + .contains(&token.text.to_ascii_uppercase().as_str()) + || stream[index + 1].token_type != TokenType::LParen + || index > 0 && stream[index - 1].token_type == TokenType::Dot + { + index += 1; + continue; + } + let open = index + 1; + let mut end = open + 1; + let mut depth = 1; + while end < stream.len() { + match stream[end].token_type { + TokenType::LParen => depth += 1, + TokenType::RParen => depth -= 1, + _ => {} + } + if depth == 0 { + break; + } + end += 1; + } + if end == stream.len() { + return Err(SidemanticError::SqlParse( + "Unclosed quantile aggregate".into(), + )); + } + let body = &sql[offsets[stream[open].span.end]..offsets[stream[end].span.start]]; + // RESERVOIR_SAMPLE uses the generic aggregate grammar without a + // specialized argument parser in the pinned registry. Only + // this root node is renamed; nested calls keep their own identity. + let mut parsed = parse_many(&format!("SELECT RESERVOIR_SAMPLE({body})"), dialect)?; + let Expression::Select(select) = parsed.remove(0) else { + unreachable!() + }; + let Expression::AggregateFunction(mut aggregate) = select.expressions[0].clone() else { + return Err(SidemanticError::SqlParse( + "Expected quantile aggregate arguments".into(), + )); + }; + aggregate.name = token.text.to_ascii_uppercase(); + let placeholder = format!("{prefix}{}", replacements.len()); + replacements.insert(placeholder.clone(), *aggregate); + prepared.push_str(&sql[cursor..offsets[token.span.start]]); + prepared.push_str(&format!("{placeholder}()")); + cursor = offsets[stream[end].span.end]; + index = end + 1; + } + prepared.push_str(&sql[cursor..]); + } + let statements = polyglot_sql::parse(&prepared, dialect) + .map_err(|error| SidemanticError::SqlParse(error.to_string()))?; + let complete_conditionals = + dialect == DialectType::DuckDB && prepared.to_ascii_uppercase().contains("IF"); + if replacements.is_empty() && !complete_conditionals { + return Ok(statements); + } + fn restore( + value: &mut Value, + replacements: &HashMap, + complete_conditionals: bool, + ) -> Result<()> { + match value { + Value::Object(fields) => { + for child in fields.values_mut() { + restore(child, replacements, complete_conditionals)?; + } + if complete_conditionals { + if let Some(function) = fields.get_mut("if_func").and_then(Value::as_object_mut) + { + if function.get("false_value").is_none_or(Value::is_null) { + // The parser accepts IF(condition, value), but the + // DuckDB emitter preserves that invalid two-argument + // call. Its omitted false branch has NULL semantics. + function.insert( + "false_value".into(), + serde_json::to_value(Expression::null()).map_err(|error| { + SidemanticError::SqlParse(error.to_string()) + })?, + ); + } + } + } + for kind in ["function", "aggregate_function"] { + let Some(function) = fields.get(kind) else { + continue; + }; + let Some(name) = function.get("name").and_then(Value::as_str) else { + continue; + }; + let Some(saved) = replacements.get(&name.to_ascii_lowercase()) else { + continue; + }; + let mut aggregate = saved.clone(); + if kind == "aggregate_function" { + let wrapper: AggregateFunction = + serde_json::from_value(function.clone()) + .map_err(|error| SidemanticError::SqlParse(error.to_string()))?; + aggregate.filter = wrapper.filter; + aggregate.ignore_nulls = wrapper.ignore_nulls.or(aggregate.ignore_nulls); + } + *value = + serde_json::to_value(Expression::AggregateFunction(Box::new(aggregate))) + .map_err(|error| SidemanticError::SqlParse(error.to_string()))?; + break; + } + } + Value::Array(children) => { + for child in children { + restore(child, replacements, complete_conditionals)?; + } + } + _ => {} + } + Ok(()) + } + let mut value = serde_json::to_value(statements) + .map_err(|error| SidemanticError::SqlParse(error.to_string()))?; + restore(&mut value, &replacements, complete_conditionals)?; + serde_json::from_value(value).map_err(|error| SidemanticError::SqlParse(error.to_string())) } pub(crate) fn query(sql: &str, source: DialectType) -> Result { @@ -358,6 +514,73 @@ pub(super) fn normalize( mod tests { use super::*; + #[test] + fn ordered_quantiles_preserve_arguments_order_filter_and_window() { + for name in ["quantile_cont", "quantile_disc"] { + let sql = format!("SELECT {name}(DISTINCT value, 0.25 ORDER BY sort_key DESC NULLS FIRST) FILTER (WHERE included) OVER (PARTITION BY category), 'é QUANTILE_CONT(x ORDER BY y)' AS label"); + let parsed = parse(&sql, DialectType::DuckDB).unwrap(); + let generated = polyglot_sql::generate(&parsed, DialectType::DuckDB).unwrap(); + assert!( + generated.contains(&format!( + "{}(DISTINCT value, 0.25 ORDER BY sort_key DESC NULLS FIRST)", + name.to_ascii_uppercase() + )), + "{generated}" + ); + assert!( + generated.contains("FILTER(WHERE included)") + || generated.contains("FILTER (WHERE included)"), + "{generated}" + ); + assert!(generated.contains("PARTITION BY category"), "{generated}"); + assert!( + generated.contains("'é QUANTILE_CONT(x ORDER BY y)'"), + "{generated}" + ); + let reparsed = parse(&generated, DialectType::DuckDB).unwrap(); + assert_eq!( + polyglot_sql::generate(&reparsed, DialectType::DuckDB).unwrap(), + generated + ); + } + } + + #[test] + fn omitted_conditional_false_branches_are_explicit_nulls_for_duckdb() { + for (input, expected) in [ + ("IF(flag, 1)", "IF(flag, 1, NULL)"), + ("COUNT(IF(flag, 1))", "COUNT(IF(flag, 1, NULL))"), + ( + "IF(flag, IF(other, 1), 0)", + "IF(flag, IF(other, 1, NULL), 0)", + ), + ("IF(flag, 1, 0)", "IF(flag, 1, 0)"), + ] { + let parsed = parse(&format!("SELECT {input}"), DialectType::DuckDB).unwrap(); + let generated = polyglot_sql::generate(&parsed, DialectType::DuckDB).unwrap(); + assert_eq!(generated, format!("SELECT {expected}")); + } + let sql = "SELECT 'IF(flag, 1)' AS label"; + let parsed = parse(sql, DialectType::DuckDB).unwrap(); + assert_eq!( + polyglot_sql::generate(&parsed, DialectType::DuckDB).unwrap(), + sql + ); + } + + #[test] + fn nested_quantiles_keep_each_function_identity() { + let parsed = parse("SELECT quantile_cont((SELECT quantile_disc(value, 0.5 ORDER BY value) FROM raw), 0.25 ORDER BY rank)", DialectType::DuckDB).unwrap(); + let generated = polyglot_sql::generate(&parsed, DialectType::DuckDB).unwrap(); + assert!(generated.contains("QUANTILE_CONT("), "{generated}"); + assert!( + generated.contains("QUANTILE_DISC(value, 0.5 ORDER BY value)"), + "{generated}" + ); + assert!(!generated.contains("RESERVOIR_SAMPLE"), "{generated}"); + assert!(!generated.contains("__sidemantic_quantile_"), "{generated}"); + } + #[test] fn nested_query_emission_uses_production_stack_on_standard_test_thread() { // Construct the AST directly to isolate final emission from parsing. diff --git a/sidemantic-rs/src/semantic_input/fragments.rs b/sidemantic-rs/src/semantic_input/fragments.rs new file mode 100644 index 000000000..3c0092d71 --- /dev/null +++ b/sidemantic-rs/src/semantic_input/fragments.rs @@ -0,0 +1,297 @@ +//! Caller expressions cannot introduce physical reads or escape their clause. +//! Trusted segment and security definitions do not cross this request boundary. + +use polyglot_sql::{expressions::Select, DialectType, Expression}; +use serde_json::Value; + +use crate::error::Result; + +// Keep this contract aligned with sidemantic/sql/fragment.py. Unknown/UDF +// functions belong in trusted model SQL, not caller-supplied filters. +const SCALAR_FUNCTIONS: &str = " +ABS ACOS ASIN ATAN ATAN2 CEIL CEILING FLOOR ROUND SIGN SQRT CBRT POWER POW EXP LN LOG LOG2 LOG10 +SIN COS TAN COT DEGREES RADIANS PI MOD GREATEST LEAST COALESCE NULLIF IF IIF CASE CAST TRY_CAST +LOWER UPPER LENGTH CHAR_LENGTH CHARACTER_LENGTH CONCAT CONCAT_WS SUBSTRING SUBSTR LEFT RIGHT +TRIM LTRIM RTRIM REPLACE REPEAT REVERSE LPAD RPAD SPLIT SPLIT_PART STARTS_WITH ENDS_WITH +CONTAINS POSITION STR_POSITION REGEXP_LIKE REGEXP_REPLACE REGEXP_EXTRACT REGEXP_SPLIT +COUNT SUM AVG MIN MAX MEDIAN STDDEV STDDEV_POP STDDEV_SAMP VARIANCE VAR_POP VAR_SAMP +DATE TIME TIMESTAMP DATE_TRUNC TIMESTAMP_TRUNC DATETIME_TRUNC TIME_TRUNC DATE_ADD DATE_SUB +DATE_DIFF DATEDIFF TIMESTAMP_ADD TIMESTAMP_SUB TIMESTAMP_DIFF EXTRACT YEAR MONTH DAY +DAY_OF_MONTH DAY_OF_WEEK DAY_OF_YEAR WEEK WEEK_OF_YEAR QUARTER HOUR MINUTE SECOND +CURRENT_DATE CURRENT_TIME CURRENT_TIMESTAMP CURRENT_DATETIME TIME_TO_STR STR_TO_TIME +TS_OR_DS_TO_DATE TS_OR_DS_TO_TIMESTAMP TS_OR_DS_TO_DATE_STR TIME_TO_UNIX UNIX_TO_TIME +DATE_TO_DATE_STR LAST_DAY DATE_FROM_PARTS TIMESTAMP_FROM_PARTS INTERVAL +ARRAY ARRAY_SIZE ARRAY_LENGTH ARRAY_CONTAINS ARRAY_SLICE ARRAY_TO_STRING +JSON_EXTRACT JSON_EXTRACT_SCALAR JSONB_EXTRACT JSONB_EXTRACT_SCALAR JSON_TYPE +STRUCT MAP EXISTS ISNULL IFNULL NVL"; + +fn normalized_name(name: &str) -> String { + name.chars() + .filter(|character| *character != '_') + .flat_map(char::to_uppercase) + .collect() +} + +fn allowed_function(name: &str) -> bool { + let name = normalized_name(name); + SCALAR_FUNCTIONS + .split_whitespace() + .any(|allowed| normalized_name(allowed) == name) +} + +fn validate_nodes(value: &Value, path: &str) -> Result<()> { + // The pinned public AST walker omits typed function children. Inspect the + // serialized tree, deserializing enum nodes to distinguish them from their + // payload objects (which can also have a single field). + if let Some(fields) = value.as_object().filter(|fields| fields.len() == 1) { + if let Ok(expression) = serde_json::from_value::(value.clone()) { + let kind = fields.keys().next().unwrap().as_str(); + match expression { + Expression::Table(_) => { + return Err(super::invalid( + path, + "Query expressions cannot introduce physical data sources", + )); + } + Expression::Select(select) if select.into.is_some() => { + return Err(super::invalid( + path, + "Query expressions cannot introduce physical data sources", + )); + } + Expression::Function(function) => { + if !allowed_function(&function.name) { + return Err(super::invalid( + path, + format!( + "Function {} is not allowed in query expressions", + function.name + ), + )); + } + } + Expression::AggregateFunction(function) => { + if !allowed_function(&function.name) { + return Err(super::invalid( + path, + format!( + "Function {} is not allowed in query expressions", + function.name + ), + )); + } + } + Expression::MethodCall(_) => { + return Err(super::invalid( + path, + "Qualified functions are not allowed in query expressions", + )); + } + _ => { + let structural = matches!( + kind, + "literal" + | "boolean" + | "null" + | "identifier" + | "column" + | "star" + | "select" + | "union" + | "intersect" + | "except" + | "subquery" + | "values" + | "alias" + | "and" + | "or" + | "xor" + | "add" + | "sub" + | "mul" + | "div" + | "eq" + | "neq" + | "lt" + | "lte" + | "gt" + | "gte" + | "like" + | "i_like" + | "bitwise_and" + | "bitwise_or" + | "bitwise_xor" + | "not" + | "neg" + | "bitwise_not" + | "in" + | "between" + | "is_null" + | "is_true" + | "is_false" + | "is" + | "from" + | "join" + | "where" + | "group_by" + | "having" + | "order_by" + | "ordered" + | "limit" + | "offset" + | "data_type" + | "tuple" + | "paren" + | "var" + | "dot" + | "bracket" + | "at_time_zone" + | "window" + | "window_function" + | "over" + | "within_group" + | "when" + | "whens" + ); + let canonical = match kind { + "if_func" => "IF", + "safe_cast" => "TRY_CAST", + _ => kind, + }; + if !structural && !allowed_function(canonical) { + return Err(super::invalid( + path, + format!("SQL expression {kind} is not allowed in query expressions"), + )); + } + } + } + } + } + match value { + Value::Object(fields) => { + for child in fields.values() { + validate_nodes(child, path)?; + } + } + Value::Array(children) => { + for child in children { + validate_nodes(child, path)?; + } + } + _ => {} + } + Ok(()) +} + +pub(super) fn validate_request_expression( + sql: &str, + dialect: DialectType, + order: bool, +) -> Result<()> { + let (path, prefix) = if order { + ("query.order_by", "SELECT 1 ORDER BY ") + } else { + ("query.filters", "SELECT 1 WHERE ") + }; + let parsed = super::dialects::parse(&format!("{prefix}{sql}"), dialect) + .map_err(|error| super::invalid(path, format!("Invalid query expression: {error}")))?; + let Expression::Select(mut select) = parsed else { + return Err(super::invalid( + path, + "Query expression contains disallowed SQL", + )); + }; + let expression = if order { + let clause = select + .order_by + .take() + .filter(|clause| clause.expressions.len() == 1) + .ok_or_else(|| super::invalid(path, "Expected one ordering expression"))?; + Expression::OrderBy(Box::new(clause)) + } else { + select + .where_clause + .take() + .ok_or_else(|| super::invalid(path, "Expected one query expression"))? + .this + }; + select.expressions.clear(); + select.leading_comments.clear(); + select.post_select_comments.clear(); + if *select != Select::new() { + return Err(super::invalid( + path, + "Query expression contains extra clauses", + )); + } + let value = serde_json::to_value(expression).map_err(|error| super::invalid(path, error))?; + validate_nodes(&value, path) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn caller_filters_reject_reads_functions_and_clause_escapes() { + super::super::with_semantic_stack(|| { + for sql in [ + "EXISTS (SELECT 1 FROM secret_table)", + "events.user_id IN (SELECT user_id FROM events_raw)", + "EXISTS (SELECT 1 FROM read_csv('/tmp/private.csv'))", + "1 = 1 UNION SELECT 1", + "1 = 1 ORDER BY 1", + "1 = 1; SELECT 2", + "(SELECT pg_read_file('/tmp/private')) IS NOT NULL", + "readfile('/tmp/private') IS NOT NULL", + "lo_get(123) IS NOT NULL", + "custom_schema.abs('/tmp/private') IS NOT NULL", + "coalesce(readfile('/tmp/private'), '') != ''", + ] { + assert!( + validate_request_expression(sql, DialectType::DuckDB, false).is_err(), + "{sql}" + ); + } + for sql in [ + "count; SELECT 2", + "count DESC LIMIT 1", + "count, user_id", + "random()", + ] { + assert!( + validate_request_expression(sql, DialectType::DuckDB, true).is_err(), + "{sql}" + ); + } + Ok(()) + }) + .unwrap(); + } + + #[test] + fn caller_filters_keep_scalar_subqueries_literals_and_known_functions() { + super::super::with_semantic_stack(|| { + for sql in [ + "EXISTS (SELECT 1 WHERE 2 > 1)", + "event_type != '; -- FROM secret_table'", + "coalesce(user_id, 0) >= 1", + "CASE WHEN user_id > 0 THEN user_id ELSE 0 END > 0", + "lower(event_type) = 'signup'", + "COUNT(DISTINCT user_id) > 0", + "CAST(user_id AS VARCHAR) != ''", + ] { + validate_request_expression(sql, DialectType::DuckDB, false)?; + } + validate_request_expression("count DESC NULLS LAST", DialectType::DuckDB, true)?; + validate_request_expression( + "CASE WHEN user_id > 0 THEN user_id ELSE 0 END DESC", + DialectType::DuckDB, + true, + )?; + Ok(()) + }) + .unwrap(); + } +} diff --git a/sidemantic-rs/src/semantic_input/inheritance.rs b/sidemantic-rs/src/semantic_input/inheritance.rs new file mode 100644 index 000000000..efb297ffb --- /dev/null +++ b/sidemantic-rs/src/semantic_input/inheritance.rs @@ -0,0 +1,194 @@ +//! Resolve raw declarations before projecting away policy and importer fields. + +use std::collections::{HashMap, HashSet}; + +use serde_json::{Map, Value}; + +use super::{invalid, object, DESCRIPTIVE_FIELDS}; +use crate::error::Result; + +pub(super) fn resolve(models: &[Value]) -> Result> { + let mut declarations = HashMap::new(); + let mut names = Vec::new(); + for (index, model) in models.iter().enumerate() { + let path = format!("models[{index}]"); + let raw = object(model.clone(), &path)?; + let name = raw + .get("name") + .and_then(Value::as_str) + .ok_or_else(|| invalid(&path, "model requires a name"))? + .to_owned(); + if declarations.insert(name.clone(), raw).is_some() { + return Err(invalid(&path, format!("duplicate model '{name}'"))); + } + names.push(name); + } + let mut resolved = HashMap::new(); + let mut visiting = HashSet::new(); + for name in &names { + resolve_model(name, &declarations, &mut resolved, &mut visiting)?; + } + Ok(names + .into_iter() + .map(|name| Value::Object(resolved.remove(&name).unwrap())) + .collect()) +} + +fn resolve_model( + name: &str, + declarations: &HashMap>, + resolved: &mut HashMap>, + visiting: &mut HashSet, +) -> Result> { + if let Some(model) = resolved.get(name) { + return Ok(model.clone()); + } + let path = format!("models.{name}.extends"); + let mut child = declarations + .get(name) + .cloned() + .ok_or_else(|| invalid(&path, format!("unknown parent model '{name}'")))?; + if !visiting.insert(name.to_owned()) { + return Err(invalid(&path, "circular model inheritance")); + } + let model = match child.remove("extends") { + None | Some(Value::Null) => child, + Some(Value::String(parent)) => { + let parent = resolve_model(&parent, declarations, resolved, visiting)?; + merge(child, parent, name)? + } + Some(_) => return Err(invalid(&path, "expected a parent model name")), + }; + visiting.remove(name); + resolved.insert(name.to_owned(), model.clone()); + Ok(model) +} + +fn merge( + mut child: Map, + mut parent: Map, + name: &str, +) -> Result> { + // Match Python merge_model: named collections override by name; invariant + // filters always accumulate, and the parent's security remains authoritative. + for field in [ + "dimensions", + "metrics", + "relationships", + "segments", + "pre_aggregations", + "invariant_filters", + ] { + let path = format!("models.{name}.{field}"); + let mut items = Vec::::new(); + let mut positions = HashMap::::new(); + for value in [parent.remove(field), child.remove(field)] + .into_iter() + .flatten() + { + let entries = value + .as_array() + .ok_or_else(|| invalid(&path, "expected an array"))?; + for entry in entries { + if field == "invariant_filters" { + items.push(entry.clone()); + continue; + } + let item_name = entry + .get("name") + .and_then(Value::as_str) + .ok_or_else(|| invalid(&path, "collection item requires a name"))?; + if let Some(position) = positions.get(item_name) { + items[*position] = entry.clone(); + } else { + positions.insert(item_name.to_owned(), items.len()); + items.push(entry.clone()); + } + } + } + parent.insert(field.to_owned(), Value::Array(items)); + } + for field in [ + "name", + "table", + "sql", + "source_uri", + "description", + "primary_key", + "unique_keys", + "default_time_dimension", + "default_grain", + "freshness", + "metadata", + "auto_dimensions", + "schema_exposure", + "meta", + ] { + if let Some(value) = child.remove(field) { + parent.insert(field.to_owned(), value); + } + } + // Python retains these fields from the parent, including inherited policy. + // Preserve unexpected child fields so the normal decoder still rejects them. + for (field, value) in child { + if !matches!(field.as_str(), "security" | "dax" | "expression_language") + && !DESCRIPTIVE_FIELDS.contains(&field.as_str()) + { + parent.insert(field, value); + } + } + Ok(parent) +} + +#[cfg(test)] +mod tests { + use super::*; + use serde_json::json; + + #[test] + fn inheritance_is_transitive_and_keeps_parent_policy_and_named_order() { + let models = vec![ + json!({"name":"leaf", "extends":"child", "invariant_filters":["amount < 100"]}), + json!({"name":"base", "table":"sales", "primary_key":"id", + "security":{"row_filters":["tenant = 1"]}, + "invariant_filters":["not deleted"], + "dimensions":[{"name":"id"},{"name":"amount","sql":"raw_amount"}]}), + json!({"name":"child", "extends":"base", "primary_key":null, + "security":null, "invariant_filters":["amount > 10"], + "dimensions":[{"name":"amount","sql":"net_amount"},{"name":"region"}]}), + ]; + let output = resolve(&models).unwrap(); + let leaf = &output[0]; + assert_eq!(leaf["name"], "leaf"); + assert_eq!(leaf["table"], "sales"); + assert!(leaf["primary_key"].is_null()); + assert_eq!(leaf["security"], models[1]["security"]); + assert_eq!( + leaf["invariant_filters"], + json!(["not deleted", "amount > 10", "amount < 100"]) + ); + assert_eq!( + leaf["dimensions"], + json!([{"name":"id"},{"name":"amount","sql":"net_amount"},{"name":"region"}]) + ); + assert!(leaf.get("extends").is_none()); + } + + #[test] + fn invalid_inheritance_fails_with_validation_errors() { + for models in [ + vec![json!({"name":"a", "extends":"missing"})], + vec![ + json!({"name":"a", "extends":"b"}), + json!({"name":"b", "extends":"a"}), + ], + vec![json!({"name":"a", "extends":5})], + vec![json!({"name":"a"}), json!({"name":"a"})], + ] { + assert!(matches!( + resolve(&models), + Err(crate::error::SidemanticError::ValidationIssue { .. }) + )); + } + } +} diff --git a/sidemantic-rs/src/semantic_input/policies.rs b/sidemantic-rs/src/semantic_input/policies.rs index 24ddf89d8..cdbb3d708 100644 --- a/sidemantic-rs/src/semantic_input/policies.rs +++ b/sidemantic-rs/src/semantic_input/policies.rs @@ -156,7 +156,7 @@ fn prepare_with_dialects( population.expression(filter, None)?; } for order in &query.order_by { - for column in order_columns(order)? { + for column in order_columns(order, query)? { population.column(&column, None)?; } } @@ -503,7 +503,7 @@ fn check_visibility( columns.extend(outer_columns(parse_semantic_expression(filter)?)?); } for order in &query.order_by { - columns.extend(order_columns(order)?); + columns.extend(order_columns(order, query)?); } for column in columns { if let Some(model) = column @@ -530,11 +530,30 @@ fn check_visibility( Ok(()) } -fn order_columns(order: &str) -> Result> { +fn order_columns(order: &str, query: &SemanticQuery) -> Result> { + let known: Vec<_> = query + .metrics + .iter() + .chain(&query.dimensions) + .map(String::as_str) + .collect(); + let (field, suffix) = crate::sql::split_order_field(order, &known); + let framed = if known.contains(&field) { + // Selected fields are semantic names, including names with spaces. + // Quote only this parser input, leaving the request's binding keys intact. + let quote = |name: &str| format!("\"{}\"", name.replace('"', "\"\"")); + let field = field.split_once('.').map_or_else( + || quote(field), + |(model, field)| format!("{}.{}", quote(model), quote(field)), + ); + format!("{field} {suffix}") + } else { + order.to_owned() + }; #[cfg(target_arch = "wasm32")] - crate::wasm_sql_guard::check(order, DialectType::DuckDB)?; + crate::wasm_sql_guard::check(&framed, DialectType::DuckDB)?; let expression = - polyglot_sql::parse_one(&format!("SELECT 1 ORDER BY {order}"), DialectType::DuckDB) + polyglot_sql::parse_one(&format!("SELECT 1 ORDER BY {framed}"), DialectType::DuckDB) .map_err(|error| SidemanticError::SqlParse(error.to_string()))?; outer_columns(expression) } diff --git a/sidemantic-rs/src/sql/generator.rs b/sidemantic-rs/src/sql/generator.rs index ff27eee07..83c38e4dc 100644 --- a/sidemantic-rs/src/sql/generator.rs +++ b/sidemantic-rs/src/sql/generator.rs @@ -6,6 +6,7 @@ mod cohort; mod conversion; mod fanout_aggregate; mod fanout_complete; +mod imported_totals; mod join_kind; mod options; mod retention; @@ -48,6 +49,8 @@ pub struct SemanticQuery { pub limit: Option, pub offset: Option, pub ungrouped: bool, + /// Explicit compatibility escape hatch: aggregate every snapshot row. + pub allow_non_additive_unsafe: bool, pub use_preaggregations: bool, pub preagg_database: Option, pub preagg_schema: Option, @@ -300,10 +303,9 @@ impl<'a> SqlGenerator<'a> { .invariant_filters .values() .any(|filters| !filters.is_empty()) - && !query.ungrouped && query.table_calculations.is_empty() { - if required_models.len() > 1 { + if required_models.len() > 1 && !query.ungrouped { let anchor = query .consumption_base_model .as_deref() @@ -319,7 +321,11 @@ impl<'a> SqlGenerator<'a> { )? { return Ok(sql); } - } else if let Some(model_name) = required_models.iter().next() { + } else if let Some(model_name) = required_models + .iter() + .next() + .filter(|_| required_models.len() == 1) + { if let Some(preagg_sql) = self.try_use_preaggregation( model_name, &metric_refs, @@ -330,6 +336,7 @@ impl<'a> SqlGenerator<'a> { query.offset, query.preagg_database.as_deref(), query.preagg_schema.as_deref(), + query.ungrouped, )? { return Ok(preagg_sql); } @@ -409,13 +416,13 @@ impl<'a> SqlGenerator<'a> { { continue; } - let window_sql = self.normalize_cte_source_expression(window_expr); + let window_sql = self.normalize_cte_source_expression(window_expr, model)?; raw_model_columns .entry(model_name.clone()) .or_default() .push(format!( "{window_sql} AS {}", - self.quote_identifier(&dimension.name) + self.quote_identifier(&Self::window_dimension_alias(dimension)) )); raw_model_aliases .entry(model_name.clone()) @@ -431,8 +438,10 @@ impl<'a> SqlGenerator<'a> { let metric = self.metric_for_model_with_source(&model_name, &metric_name, graph_metric)?; let raw_alias = self.metric_raw_alias(model, &metric_name, metric); - let mut raw_expr = - self.normalize_cte_source_expression(&self.metric_raw_expression(metric, model)?); + let mut raw_expr = self.normalize_cte_source_expression( + &self.metric_raw_expression(metric, model)?, + model, + )?; if !metric.filters.is_empty() { let metric_filter = self.normalize_metric_filters( &metric.filters, @@ -466,7 +475,9 @@ impl<'a> SqlGenerator<'a> { })?; let raw_expr = model .get_dimension(&column_name) - .map(|dimension| self.normalize_cte_source_expression(dimension.sql_expr())) + .and_then(|dimension| dimension.sql.as_deref()) + .map(|sql| self.normalize_cte_source_expression(sql, model)) + .transpose()? .unwrap_or_else(|| self.quote_identifier(&column_name)); raw_model_columns .entry(model_name.clone()) @@ -577,21 +588,38 @@ impl<'a> SqlGenerator<'a> { } else { identity } + } else if dimension.window.is_some() { + let column = format!( + "{}.{}", + alias, + self.quote_identifier(&Self::window_dimension_alias(dimension)) + ); + if let Some(granularity) = dim_ref + .granularity + .as_deref() + .or(dimension.granularity.as_deref()) + { + self.date_trunc_sql(granularity, &column)? + } else { + column + } } else if let Some(granularity) = dim_ref .granularity .as_deref() .or(dimension.granularity.as_deref()) { + let source = dimension + .sql + .clone() + .unwrap_or_else(|| self.quote_identifier(&dimension.name)); self.normalize_select_expression( - &self.date_trunc_sql(granularity, dimension.sql_expr())?, + &self.date_trunc_sql(granularity, &source)?, &alias, ) - } else if dimension.window.is_some() { - format!("{}.{}", alias, self.quote_identifier(&dimension.name)) } else { - self.dimension_select_expression(dimension, &alias) + self.dimension_select_expression(dimension, &alias)? } - } else if Self::is_relationship_foreign_key_dimension(model, &dim_ref.name) { + } else if model.is_foreign_key_dimension(&dim_ref.name) { format!("{}.{}", alias, self.quote_identifier(&dim_ref.name)) } else { let available: Vec<&str> = @@ -720,7 +748,10 @@ impl<'a> SqlGenerator<'a> { }, MetricType::Derived => { // For derived metrics, we need to expand referenced metrics - self.expand_derived_metric(metric.sql_expr(), &metric_ref.model)? + self.expand_derived_metric( + &self.imported_calculation_expression(metric, &metric_ref.model)?, + &metric_ref.model, + )? } MetricType::Ratio => { // For ratio metrics, expand numerator and denominator @@ -761,7 +792,7 @@ impl<'a> SqlGenerator<'a> { sql.push('\n'); // FROM clause - let source_start = sql.len(); + let mut source_start = sql.len(); sql.push_str(&format!( "FROM {}_cte AS {}\n", base_model, @@ -815,11 +846,15 @@ impl<'a> SqlGenerator<'a> { } else if cte_where_filters .get(&step.to_model) .is_some_and(|filters| !filters.is_empty()) - || query - .prepared_policies - .filters_for_model(&step.to_model) - .next() - .is_some() + // Junction policies restrict which links are visible; + // they must not remove source rows with no visible link. + // Explicit query filters above still constrain the domain. + || (!self.graph.is_bridge_instance(&step.to_model) + && query + .prepared_policies + .filters_for_model(&step.to_model) + .next() + .is_some()) { "INNER JOIN" } else { @@ -837,6 +872,26 @@ impl<'a> SqlGenerator<'a> { sql.push_str(&format!("WHERE {}\n", filter_sql.join(" AND "))); } + // BSL all() aggregates the same filtered, joined population before + // grouping. In particular, distinct totals cannot sum group counts. + if select_parts + .iter() + .any(|part| part.to_ascii_lowercase().contains("__bsl_all(")) + { + let source = sql[source_start..].to_string(); + for part in &mut select_parts { + if part.to_ascii_lowercase().contains("__bsl_all(") { + *part = self.expand_imported_totals(part, &source)?; + } + } + sql.truncate(select_start); + sql.push_str("SELECT\n"); + sql.push_str(&select_parts.join(",\n")); + sql.push('\n'); + source_start = sql.len(); + sql.push_str(&source); + } + if !aggregate_ranks.is_empty() { let (ranked, rewritten_having) = self.ranked_aggregate_source( &select_parts, @@ -1150,7 +1205,7 @@ impl<'a> SqlGenerator<'a> { return Ok(()); }; let Some(dimension) = model.get_dimension(dimension_name) else { - if Self::is_relationship_foreign_key_dimension(model, dimension_name) { + if model.is_foreign_key_dimension(dimension_name) { return Err(SidemanticError::Validation(format!( "Cannot apply granularity to non-time dimension '{dimension_name}'" ))); @@ -1351,7 +1406,11 @@ impl<'a> SqlGenerator<'a> { .flatten() .chain(metric.filters.iter().map(String::as_str)) { - for column in semantic_column_references(fragment)? { + let fragment = crate::core::replace_model_placeholder( + fragment, + self.graph.metric_owner(reference), + )?; + for column in semantic_column_references(&fragment)? { if let Some(model) = &column.model { if self.graph.get_model(model).is_none() { return Err(SidemanticError::InvalidConfig(format!( @@ -1394,17 +1453,32 @@ impl<'a> SqlGenerator<'a> { } } } - if owners.len() != 1 { + // A graph-level aggregate can name its source through its public name, + // e.g. events.p95.latency with physical SQL input `latency`. + if owners.is_empty() && metric.agg.is_some() { + if let Some((model, _)) = reference.split_once('.') { + if self.graph.get_model(model).is_some() { + owners.insert(model.to_owned()); + } + } + } + if owners.is_empty() { return Err(SidemanticError::UnsupportedSemanticFeatures { capabilities: vec![format!("metric.graph_scope.{reference}")], }); } - Ok(owners.into_iter().collect()) + // This is dependency discovery, not assignment of a synthetic owner. + // Independent aggregate planning and public alias resolution need the + // full source set; source-local callers check for a single owner. + let mut owners: Vec<_> = owners.into_iter().collect(); + owners.sort(); + Ok(owners) } fn metric_reference_tokens(&self, expression: &str) -> Result> { if self.graph.has_strict_metric_scope() { - return Ok(semantic_column_references(expression)? + let expression = crate::core::replace_model_placeholder(expression, None)?; + return Ok(semantic_column_references(&expression)? .into_iter() .filter(|column| !column.aggregate_input) .map(|column| column.name()) @@ -1670,7 +1744,8 @@ impl<'a> SqlGenerator<'a> { models: &mut HashSet, ) -> Result<()> { if self.graph.has_strict_metric_scope() { - for column in semantic_column_references(expr)? { + let expr = crate::core::replace_model_placeholder(expr, None)?; + for column in semantic_column_references(&expr)? { if let Some(model) = column.model { if self.graph.get_model(&model).is_some() { models.insert(model); @@ -1876,7 +1951,10 @@ impl<'a> SqlGenerator<'a> { let metric = self.metric_for_ref(metric_ref)?; if self.graph.has_strict_metric_scope() && metric.r#type == MetricType::Derived { - for column in semantic_column_references(metric.sql_expr())? { + let expression = self.imported_calculation_expression(metric, &metric_ref.model)?; + let expression = + crate::core::replace_model_placeholder(&expression, Some(&metric_ref.model))?; + for column in semantic_column_references(&expression)? { if column.aggregate_input { deps.insert(( column.model.unwrap_or_else(|| metric_ref.model.clone()), @@ -1964,7 +2042,8 @@ impl<'a> SqlGenerator<'a> { deps: &mut HashSet<(String, String)>, ) -> Result<()> { if self.graph.has_strict_metric_scope() { - for column in semantic_column_references(expr)? { + let expr = crate::core::replace_model_placeholder(expr, Some(default_model))?; + for column in semantic_column_references(&expr)? { let model = column.model.unwrap_or_else(|| default_model.to_string()); if self.graph.get_model(&model).is_some() { deps.insert((model, column.field)); @@ -2125,6 +2204,20 @@ impl<'a> SqlGenerator<'a> { } let names: Vec<_> = known_fields.iter().map(String::as_str).collect(); let (head, suffix) = crate::sql::split_order_field(item, &names); + // Boundary dialect normalization preserves identifier quotes. Resolve + // the parsed column name, while retaining the original SQL if it is + // not a selected semantic field. + let normalized_head = if names.contains(&head) { + None + } else if let Ok(Expression::Column(column)) = parse_semantic_expression(head) { + Some(match column.table { + Some(table) => format!("{}.{}", table.name, column.name.name), + None => column.name.name, + }) + } else { + None + }; + let reference = normalized_head.as_deref().unwrap_or(head); // Resolve the field and NULL placement from the same suffix. Python's // ordinary SQLGlot builder defaults to ascending NULLs first, descending last. let suffix = if suffix.contains("NULLS") { @@ -2138,14 +2231,14 @@ impl<'a> SqlGenerator<'a> { }; for metric_ref in metric_refs { - if metric_ref.name == head || metric_ref.alias == head { + if metric_ref.name == reference || metric_ref.alias == reference { let alias = self.output_alias(&metric_ref.model, &metric_ref.alias, alias_collisions); return format!("{}{}", self.quote_identifier(&alias), suffix); } } - let Ok((model, field, granularity)) = self.graph.parse_reference(head) else { + let Ok((model, field, granularity)) = self.graph.parse_reference(reference) else { return format!("{head}{suffix}"); }; @@ -2308,6 +2401,7 @@ impl<'a> SqlGenerator<'a> { if window_models.is_empty() { non_window_filters.push(filter.clone()); } else { + let filter = self.window_row_filter(filter, &window_models)?; for model_name in window_models { window_filters_by_model .entry(model_name) @@ -2573,7 +2667,11 @@ impl<'a> SqlGenerator<'a> { cohort_metrics.push(metric_ref.clone()); } _ => { - let explicit_ref = format!("{}.{}", metric_ref.model, metric_ref.name); + let explicit_ref = if metric_ref.graph_metric { + metric_ref.name.clone() + } else { + format!("{}.{}", metric_ref.model, metric_ref.name) + }; if seen_metrics.insert(explicit_ref.clone()) { base_metrics.push(explicit_ref); } @@ -3869,15 +3967,6 @@ impl<'a> SqlGenerator<'a> { } } - fn is_relationship_foreign_key_dimension(model: &Model, dimension_name: &str) -> bool { - model.relationships.iter().any(|relationship| { - relationship - .foreign_key_columns() - .iter() - .any(|column| column == dimension_name) - }) - } - fn interval_sql(&self, num: &str, unit: &str) -> String { format!("INTERVAL '{num} {unit}'") } @@ -3957,6 +4046,16 @@ impl<'a> SqlGenerator<'a> { let mut seen_models = HashSet::new(); for metric_ref in metrics { + // Graph calculations are scalar unless dimensions were requested. + // Only cumulative graph metrics inherit a dependency's time grain + // because their window needs an ordering dimension. + if self + .graph + .get_metric(metric_ref) + .is_some_and(|metric| metric.r#type != MetricType::Cumulative) + { + continue; + } let model_name = if let Some((model_name, _, _)) = self.exact_metric_reference(metric_ref)? { model_name @@ -4269,6 +4368,7 @@ impl<'a> SqlGenerator<'a> { offset: Option, preagg_database: Option<&str>, preagg_schema: Option<&str>, + ungrouped: bool, ) -> Result> { let model = self.graph.get_model(model_name).ok_or_else(|| { let available: Vec<&str> = self.graph.models().map(|m| m.name.as_str()).collect(); @@ -4291,6 +4391,25 @@ impl<'a> SqlGenerator<'a> { return Ok(None); }; query_metric_names.extend(filter_plan.metrics.iter().cloned()); + if ungrouped + && (!filter_plan.aggregates.is_empty() + || metric_refs.iter().any(|reference| { + model.get_metric(&reference.name).is_none_or(|metric| { + metric.r#type != MetricType::Simple + || !matches!( + metric.agg, + Some( + Aggregation::Sum + | Aggregation::Count + | Aggregation::Min + | Aggregation::Max + ) + ) + }) + })) + { + return Ok(None); + } // Ordering is limited to selected semantic outputs on this route. let mut rewritten_order = Vec::new(); @@ -4327,6 +4446,17 @@ impl<'a> SqlGenerator<'a> { let mut best_match: Option<(crate::core::PreAggregation, i32)> = None; for preagg in &model.pre_aggregations { + if ungrouped { + let keys = model.primary_keys(); + if keys.is_empty() + || !preagg + .dimensions + .as_ref() + .is_some_and(|dimensions| keys.iter().all(|key| dimensions.contains(key))) + { + continue; + } + } if !self.preaggregation_can_satisfy_query( model, preagg, @@ -4366,6 +4496,7 @@ impl<'a> SqlGenerator<'a> { offset, preagg_database, preagg_schema, + ungrouped, )?; Ok(Some(format!("{preagg_sql}\n-- used_preagg=true"))) @@ -4982,6 +5113,7 @@ impl<'a> SqlGenerator<'a> { offset: Option, preagg_database: Option<&str>, preagg_schema: Option<&str>, + ungrouped: bool, ) -> Result { let preagg_table = self.preaggregation_source(model, preagg, preagg_database, preagg_schema)?; @@ -5023,14 +5155,17 @@ impl<'a> SqlGenerator<'a> { )); } for metric_ref in metric_refs { - let expression = self - .preaggregation_metric_expression( + let expression = if ungrouped { + self.quote_identifier(&format!("{}_raw", metric_ref.name)) + } else { + self.preaggregation_metric_expression( model, preagg, &metric_ref.name, &mut HashSet::new(), ) - .expect("matcher checked aggregate state"); + .expect("matcher checked aggregate state") + }; select_parts.push(format!( "{expression} AS {}", self.quote_identifier(&metric_ref.alias) @@ -5043,7 +5178,7 @@ impl<'a> SqlGenerator<'a> { if !filters.is_empty() { sql.push_str(&format!("\nWHERE {}", filters.join(" AND "))); } - if !dimension_refs.is_empty() { + if !ungrouped && !dimension_refs.is_empty() { let group_by: Vec<_> = (1..=dimension_refs.len()) .map(|index| index.to_string()) .collect(); @@ -5075,37 +5210,35 @@ impl<'a> SqlGenerator<'a> { ) -> Result<(Vec, Vec)> { let mut where_filters = Vec::new(); let mut having_filters = Vec::new(); - let ref_re = regex::Regex::new(r"\b([A-Za-z_][A-Za-z0-9_]*)\.([A-Za-z_][A-Za-z0-9_]*)\b") - .expect("valid model.field regex"); - for filter in filters { - let mut rewritten = filter.clone(); - let mut uses_metric = false; - - for cap in ref_re.captures_iter(filter) { - let Some(model_match) = cap.get(1) else { - continue; - }; - let Some(field_match) = cap.get(2) else { - continue; - }; - let model_name = model_match.as_str(); - let field_name = field_match.as_str(); - let Some(model) = self.graph.get_model(model_name) else { - continue; + let mut replacements = HashMap::new(); + for column in crate::core::outer_semantic_column_references(filter)? { + let reference = column.name(); + let metric = if self.graph.get_metric(&reference).is_some() { + self.exact_metric_reference(&reference)? + } else if let Some(model) = column.model.as_deref() { + self.graph.get_model(model).and_then(|model| { + model + .get_metric(&column.field) + .map(|_| (model.name.clone(), column.field.clone(), false)) + }) + } else { + self.exact_metric_reference(&reference)? }; - if model.get_metric(field_name).is_none() { - continue; + if let Some((model, name, _)) = metric { + replacements.insert( + (column.model, column.field), + self.quote_identifier(&self.output_alias(&model, &name, collisions)), + ); } - - uses_metric = true; - let replacement = self.output_alias(model_name, field_name, collisions); - let full_ref = format!("{model_name}.{field_name}"); - rewritten = rewritten.replace(&full_ref, &replacement); } - - if uses_metric { - having_filters.push(rewritten); + if !replacements.is_empty() { + having_filters.push(self.emit_expression( + &crate::core::replace_outer_semantic_columns( + parse_semantic_expression(filter)?, + &replacements, + )?, + )?); } else { where_filters.push(filter.clone()); } @@ -5235,6 +5368,62 @@ impl<'a> SqlGenerator<'a> { .is_empty() } + /// A metric beside a window dimension in a row predicate denotes its input, + /// before aggregation. Keep that predicate out of the child's HAVING clause. + fn window_row_filter(&self, filter: &str, owners: &HashSet) -> Result { + let mut replacements = HashMap::new(); + for column in crate::core::outer_semantic_column_references(filter)? { + let Some(owner) = column.model.as_deref() else { + continue; + }; + let Some(model) = self.graph.get_model(owner) else { + continue; + }; + let Some(metric) = model.get_metric(&column.field) else { + continue; + }; + if !owners.contains(owner) + || metric.r#type != MetricType::Simple + || metric.sql_is_complete + { + return Err(SidemanticError::UnsupportedSemanticFeatures { + capabilities: vec!["aggregation.mixed_row_aggregate_filter".into()], + }); + } + let alias = self.model_alias(owner); + let mut raw = self + .metric_raw_expression(metric, model)? + .replace("{model}", owner); + if !metric.filters.is_empty() { + let predicate = self.normalize_metric_filters(&metric.filters, owner, &alias)?; + raw = format!("CASE WHEN {predicate} THEN {raw} END"); + } + let mut inputs = HashMap::new(); + for input in semantic_column_references(&raw)? { + if input.model.as_deref().is_some_and(|source| { + source != owner && source != alias && source != model.table_name() + }) { + return Err(SidemanticError::UnsupportedSemanticFeatures { + capabilities: vec!["aggregation.mixed_row_aggregate_filter".into()], + }); + } + inputs.insert( + (input.model, input.field.clone()), + format!("{alias}.{}", self.quote_identifier(&input.field)), + ); + } + let raw = self.emit_expression(&crate::core::replace_semantic_columns( + parse_semantic_expression(&raw)?, + &inputs, + )?)?; + replacements.insert((column.model, column.field), format!("({raw})")); + } + self.emit_expression(&crate::core::replace_outer_semantic_columns( + parse_semantic_expression(filter)?, + &replacements, + )?) + } + fn filter_window_dimension_models( &self, filter: &str, @@ -5292,10 +5481,21 @@ impl<'a> SqlGenerator<'a> { capabilities: vec!["filter.computed_key_source".into()], }); } - let source = if computed_keys.contains(&column.field) { - self.key_sql(model, &column.field, None)? - } else if let Some(dimension) = model.get_dimension(&column.field) { - let source = dimension.sql_expr().replace("{model}", model_name); + let (field, grain) = column + .field + .rsplit_once("__") + .filter(|(field, _)| model.get_dimension(field).is_some()) + .map_or((column.field.as_str(), None), |(field, grain)| { + (field, Some(grain)) + }); + let mut source = if computed_keys.contains(field) { + self.key_sql(model, field, None)? + } else if let Some(dimension) = model.get_dimension(field) { + let source = dimension + .sql + .clone() + .unwrap_or_else(|| self.quote_identifier(&dimension.name)) + .replace("{model}", model_name); let mut inputs = HashMap::new(); for input in semantic_column_references(&source)? { if input @@ -5319,6 +5519,9 @@ impl<'a> SqlGenerator<'a> { } else { self.quote_identifier(&column.field) }; + if let Some(grain) = grain { + source = self.date_trunc_sql(grain, &source)?; + } let replacement = match polyglot_sql::parse_one(&format!("SELECT {source}"), self.dialect) { Ok(Expression::Select(select)) @@ -5391,8 +5594,25 @@ impl<'a> SqlGenerator<'a> { Ok(rendered.join(" AND ")) } - fn normalize_cte_source_expression(&self, expr: &str) -> String { - expr.replace("{model}.", "").replace("{model}", "") + fn normalize_cte_source_expression(&self, expr: &str, model: &Model) -> Result { + let expr = expr.replace("{model}.", "").replace("{model}", ""); + let mut replacements = HashMap::new(); + for column in semantic_column_references(&expr)? { + if column.model.as_deref().is_some_and(|owner| { + owner == model.name + || owner == self.model_alias(&model.name) + || owner == model.table_name() + }) { + replacements.insert( + (column.model, column.field.clone()), + self.quote_identifier(&column.field), + ); + } + } + self.emit_expression(&crate::core::replace_semantic_columns( + parse_semantic_expression(&expr)?, + &replacements, + )?) } fn normalize_select_expression(&self, expr: &str, alias: &str) -> String { @@ -5403,13 +5623,39 @@ impl<'a> SqlGenerator<'a> { &self, dimension: &crate::core::Dimension, alias: &str, - ) -> String { - let expr = dimension.sql_expr(); - if expr.contains("{model}") { - self.normalize_select_expression(expr, alias) - } else { - format!("{}.{}", alias, expr) + ) -> Result { + if dimension.sql.is_none() { + // A default dimension is a physical identifier, not authored SQL. + // Parsing names such as "Order Date" as expressions loses that + // distinction (and can interpret the second word as an alias). + return Ok(format!( + "{alias}.{}", + self.quote_identifier(&dimension.name) + )); } + let expr = self.normalize_select_expression(dimension.sql_expr(), alias); + // Qualify the input columns, not the complete expression: prefixing + // CASE, arithmetic or a function call produces invalid SQL. + let mut replacements = HashMap::new(); + for column in semantic_column_references(&expr)? { + if column.model.is_none() + || column.model.as_deref() == alias.strip_suffix("_cte") + || column.model.as_deref().is_some_and(|owner| { + self.graph + .get_model(alias.strip_suffix("_cte").unwrap_or(alias)) + .is_some_and(|model| model.sql.is_none() && owner == model.table_name()) + }) + { + replacements.insert( + (column.model, column.field.clone()), + format!("{alias}.{}", self.quote_identifier(&column.field)), + ); + } + } + self.emit_expression(&crate::core::replace_semantic_columns( + parse_semantic_expression(&expr)?, + &replacements, + )?) } fn is_simple_identifier(identifier: &str) -> bool { @@ -5419,15 +5665,32 @@ impl<'a> SqlGenerator<'a> { } fn quote_identifier(&self, identifier: &str) -> String { - if Self::is_simple_identifier(identifier) { - identifier.to_string() - } else { - // A quoted identifier is a leaf AST node with no unsupported operations. + // Rewriter leaves use DuckDB before final target emission. Preserve + // case there too so a PostgreSQL wrapper can reference the same alias. + let preserve_case = matches!( + self.dialect, + DialectType::DuckDB | DialectType::PostgreSQL | DialectType::Redshift + ) && identifier + .chars() + .any(|character| character.is_ascii_uppercase()); + if Self::is_simple_identifier(identifier) && !preserve_case { + // The SQL generator owns the dialect's reserved-word table. polyglot_sql::generate( - &Expression::Identifier(Identifier::quoted(identifier)), + &Expression::Identifier(Identifier::new(identifier)), self.dialect, ) - .expect("quoted identifier generation is infallible") + .expect("identifier generation is infallible") + } else { + // Keep the name literal: the pinned generator otherwise splits + // ASC/DESC and index-prefix suffixes even inside quoted identifiers. + let dialect = polyglot_sql::Dialect::get(self.dialect); + let style = &dialect.generator_config().identifier_quote_style; + format!( + "{}{}{}", + style.start, + identifier.replace(style.end, &format!("{}{}", style.end, style.end)), + style.end + ) } } @@ -5554,9 +5817,11 @@ impl<'a> SqlGenerator<'a> { }); } MetricType::Simple => self.simple_metric_reference_sql(metric, &metric_name, &alias)?, - MetricType::Derived => { - self.expand_derived_metric_inner(metric.sql_expr(), &model_name, visited)? - } + MetricType::Derived => self.expand_derived_metric_inner( + &self.imported_calculation_expression(metric, &model_name)?, + &model_name, + visited, + )?, MetricType::Ratio => { let num_ref = metric.numerator.as_deref().unwrap_or("1"); let den_ref = metric.denominator.as_deref().unwrap_or("1"); @@ -5673,8 +5938,9 @@ impl<'a> SqlGenerator<'a> { default_model: &str, visited: &mut HashSet<(String, String, bool)>, ) -> Result { - let parsed = parse_semantic_expression(expression)?; - let columns = semantic_column_references(expression)?; + let expression = crate::core::replace_model_placeholder(expression, Some(default_model))?; + let parsed = parse_semantic_expression(&expression)?; + let columns = semantic_column_references(&expression)?; let has_aggregate = columns.iter().any(|column| column.aggregate_input); let mut replacements = HashMap::new(); for column in columns { @@ -5813,6 +6079,33 @@ impl<'a> SqlGenerator<'a> { fn expand_filter_with_polyglot(&self, filter: &str) -> Result { let parsed = self.parse_where_expr(filter)?; let mut keys = HashMap::new(); + for column in crate::core::outer_semantic_column_references(filter)? { + let matches: Vec<_> = self + .graph + .models() + .filter_map(|model| { + if column.model.as_deref().is_some_and(|owner| { + owner.strip_suffix("_cte").unwrap_or(owner) != model.name + }) { + return None; + } + model + .get_dimension(&column.field) + .filter(|dimension| dimension.window.is_some()) + .map(|dimension| (model, dimension)) + }) + .collect(); + if let [(model, dimension)] = matches.as_slice() { + keys.insert( + (column.model, column.field), + format!( + "{}.{}", + self.model_alias(&model.name), + self.quote_identifier(&Self::window_dimension_alias(dimension)) + ), + ); + } + } if self.has_computed_key_models(&self.find_filter_models(&[filter.to_string()]))? { for column in semantic_column_references(filter)? { if let Some(model) = column @@ -5851,7 +6144,10 @@ impl<'a> SqlGenerator<'a> { sql: format!( "{}.{}", self.model_alias(&model.name), - dimension.sql_expr() + dimension + .sql + .clone() + .unwrap_or_else(|| self.quote_identifier(&dimension.name)) ), })); } @@ -6057,6 +6353,261 @@ mod tests { Relationship, }; + #[test] + fn implicit_spaced_dimensions_remain_identifiers_in_grains_and_filters() { + let mut graph = SemanticGraph::new(); + graph + .add_model( + Model::new("Sales", "id") + .with_table("sales") + .with_dimension(Dimension::time("Order Date").with_granularity("month")) + .with_dimension(Dimension::categorical("Order Status")) + .with_metric(Metric::sum("qty", "quantity")), + ) + .unwrap(); + let generator = SqlGenerator::new(&graph); + for (dimension, filter) in [ + ("Order Date", None), + ("Order Status", None), + ("Order Status", Some("Sales.\"Order Status\" = 'shipped'")), + ] { + let mut query = SemanticQuery::new() + .with_metrics(vec!["Sales.qty".into()]) + .with_dimensions(vec![format!("Sales.{dimension}")]); + query.order_by = vec![format!("Sales.{dimension}")]; + query.filters = filter.into_iter().map(str::to_owned).collect(); + let sql = generator.generate(&query).unwrap(); + crate::semantic_input::dialects::parse(&sql, DialectType::DuckDB).unwrap(); + assert!(sql.contains(&format!("\"{dimension}\"")), "{sql}"); + if filter.is_some() { + assert!(sql.contains("\"Order Status\" = 'shipped'"), "{sql}"); + } + } + } + + #[test] + fn source_qualifiers_bind_at_cte_and_dimension_projection_boundaries() { + for source_sql in [None, Some("SELECT * FROM sales")] { + let mut model = Model::new("sales_model", "id") + .with_table("sales") + .with_dimension(Dimension::categorical("sales_id").with_sql("sales.id")) + .with_metric(Metric::sum("amount", "sales_model.amount * 2")); + model.sql = source_sql.map(str::to_string); + let mut graph = SemanticGraph::new(); + graph.add_model(model).unwrap(); + let generator = SqlGenerator::new(&graph); + let model = graph.get_model("sales_model").unwrap(); + let expression = generator + .normalize_cte_source_expression( + "sales_model.amount + CASE WHEN sales.amount > 0 THEN 1 ELSE 0 END", + model, + ) + .unwrap(); + assert!(semantic_column_references(&expression) + .unwrap() + .iter() + .all(|column| column.model.is_none())); + let sql = generator + .generate(&SemanticQuery::new().with_metrics(vec!["sales_model.amount".into()])) + .unwrap(); + assert!(!sql.contains("sales_model.amount * 2"), "{sql}"); + if source_sql.is_none() { + let sql = generator + .generate( + &SemanticQuery::new() + .with_metrics(vec!["sales_model.amount".into()]) + .with_dimensions(vec!["sales_model.sales_id".into()]), + ) + .unwrap(); + assert!(sql.contains("sales_model_cte.id AS sales_id"), "{sql}"); + } + } + } + + #[test] + fn reserved_complete_aggregate_inputs_remain_quoted() { + let mut graph = SemanticGraph::new(); + graph + .add_model( + Model::new("orders", "id") + .with_table("orders") + .with_metric(Metric::derived("total", "SUM({model}.\"select\")")), + ) + .unwrap(); + let generator = SqlGenerator::new(&graph); + let sql = generator + .generate(&SemanticQuery::new().with_metrics(vec!["orders.total".into()])) + .unwrap(); + crate::semantic_input::dialects::parse(&sql, DialectType::DuckDB).unwrap(); + assert!(sql.contains("\"select\""), "{sql}"); + } + + #[test] + fn identifier_quoting_uses_reserved_words_and_keeps_literal_suffixes() { + let graph = SemanticGraph::new(); + for (dialect, quote) in [(DialectType::DuckDB, '"'), (DialectType::BigQuery, '`')] { + let generator = SqlGenerator::new(&graph).with_dialect(dialect); + assert_eq!( + generator.quote_identifier("select"), + format!("{quote}select{quote}") + ); + for name in ["Order Date", "value DESC", "value(10)"] { + assert_eq!( + generator.quote_identifier(name), + format!("{quote}{name}{quote}") + ); + } + } + for dialect in [ + DialectType::DuckDB, + DialectType::PostgreSQL, + DialectType::Redshift, + ] { + let generator = SqlGenerator::new(&graph).with_dialect(dialect); + assert_eq!(generator.quote_identifier("CaseSubject"), "\"CaseSubject\""); + assert_eq!(generator.quote_identifier("subject"), "subject"); + } + } + + #[test] + fn computed_dimensions_qualify_each_physical_input() { + for expression in [ + "CASE WHEN amount < 20 THEN 'small' ELSE NULL END", + "COALESCE(amount, 0) + adjustment", + "{model}.amount + adjustment", + "orders.amount + adjustment", + ] { + let mut graph = SemanticGraph::new(); + graph + .add_model( + Model::new("orders", "id") + .with_table("raw_orders") + .with_dimension(Dimension::categorical("band").with_sql(expression)) + .with_metric(Metric::count("rows")), + ) + .unwrap(); + let generator = SqlGenerator::new(&graph); + let dimension = graph + .get_model("orders") + .unwrap() + .get_dimension("band") + .unwrap(); + let rendered = generator + .dimension_select_expression(dimension, "orders_cte") + .unwrap(); + let columns = semantic_column_references(&rendered).unwrap(); + assert!(!columns.is_empty(), "{rendered}"); + assert!( + columns + .iter() + .all(|column| column.model.as_deref() == Some("orders_cte")), + "{rendered}" + ); + for ungrouped in [false, true] { + let query = SemanticQuery::new() + .with_metrics(vec!["orders.rows".into()]) + .with_dimensions(vec!["orders.band".into()]) + .with_ungrouped(ungrouped); + let sql = generator.generate(&query).unwrap(); + polyglot_sql::parse_one(&sql, DialectType::DuckDB).unwrap(); + assert!(sql.contains(&rendered), "{sql}"); + } + } + } + + #[test] + fn graph_metric_predicate_is_having_and_preserves_string_literals() { + let mut graph = SemanticGraph::new(); + graph + .add_model( + Model::new("orders", "id") + .with_table("orders") + .with_dimension(Dimension::categorical("region")), + ) + .unwrap(); + let mut count = Metric::count("one"); + count.filters = vec!["status = 'paid'".into()]; + graph.add_metric_unvalidated(count).unwrap(); + graph + .set_metric_scopes(HashMap::from([("one".into(), "orders".into())])) + .unwrap(); + let query = SemanticQuery::new() + .with_metrics(vec!["one".into()]) + .with_dimensions(vec!["orders.region".into()]) + .with_filters(vec!["one > 0".into(), "orders.region <> 'one'".into()]); + let sql = SqlGenerator::new(&graph).generate(&query).unwrap(); + assert!(sql.contains("HAVING one > 0"), "{sql}"); + assert!(!sql.contains("WHERE one > 0"), "{sql}"); + assert!(sql.contains("region <> 'one'"), "{sql}"); + } + + #[test] + fn grained_filter_binds_physical_expression_without_selected_alias() { + let mut graph = SemanticGraph::new(); + graph + .add_model( + Model::new("events", "id") + .with_table("events") + .with_dimension(Dimension::time("created_at").with_sql("occurred_at")) + .with_dimension(Dimension::new("gross").with_sql("unit_price * quantity")) + .with_metric(Metric::sum("revenue", "amount")), + ) + .unwrap(); + for dialect in [DialectType::DuckDB, DialectType::PostgreSQL] { + let sql = SqlGenerator::new(&graph) + .with_dialect(dialect) + .generate( + &SemanticQuery::new() + .with_metrics(vec!["events.revenue".into()]) + .with_filters(vec![ + "events.created_at__month = DATE '2024-02-01'".into(), + "events.gross >= 20".into(), + ]), + ) + .unwrap(); + assert!(!sql.contains("created_at__month"), "{sql}"); + assert!( + sql.to_ascii_uppercase() + .contains("DATE_TRUNC('MONTH', OCCURRED_AT)"), + "{sql}" + ); + assert!(sql.contains("(unit_price * quantity) >= 20"), "{sql}"); + polyglot_sql::parse_one(&sql, dialect).unwrap(); + } + } + + #[test] + fn scalar_graph_metrics_do_not_inherit_model_default_time_dimension() { + let mut graph = SemanticGraph::new(); + let mut model = Model::new("orders", "id") + .with_table("orders") + .with_dimension(Dimension::time("day")) + .with_metric(Metric::sum("revenue", "amount")); + model.default_time_dimension = Some("day".into()); + model.default_grain = Some("day".into()); + graph.add_model(model).unwrap(); + graph + .add_metric_unvalidated(Metric::derived("total_revenue", "orders.revenue")) + .unwrap(); + graph.set_metric_scopes(HashMap::new()).unwrap(); + let generator = SqlGenerator::new(&graph); + assert!(generator + .apply_default_time_dimensions(&["total_revenue".into()], &[]) + .unwrap() + .is_empty()); + assert_eq!( + generator + .apply_default_time_dimensions(&["orders.revenue".into()], &[]) + .unwrap(), + vec!["orders.day__day"] + ); + let sql = generator + .generate(&SemanticQuery::new().with_metrics(vec!["total_revenue".into()])) + .unwrap(); + assert!(!sql.contains("GROUP BY"), "{sql}"); + assert!(!sql.contains("DATE_TRUNC"), "{sql}"); + } + #[test] fn computed_time_key_projection_applies_grain_and_timezone() { let mut dimension = Dimension::time("event_time").with_sql("CAST(raw_time AS TIMESTAMP)"); @@ -6177,6 +6728,57 @@ mod tests { assert!(!sql.contains("orders_preagg_daily"), "{sql}"); } + #[test] + fn ungrouped_rollups_require_complete_primary_key_and_per_row_state() { + let source = serde_json::json!({ + "name":"orders", "table":"raw_orders", "primary_key":"id", + "dimensions":[{"name":"id", "type":"categorical"}, {"name":"status", "type":"categorical"}], + "metrics":[{"name":"revenue", "agg":"sum", "sql":"amount"}, {"name":"average", "agg":"avg", "sql":"amount"}], + "pre_aggregations":[{"name":"detail", "dimensions":["id", "status"], "measures":["revenue", "average"]}] + }); + for (keys, dimensions, metric, filter, expected) in [ + (vec!["id"], vec!["id", "status"], "revenue", None, true), + (vec!["id"], vec!["status"], "revenue", None, false), + (vec![], vec!["id", "status"], "revenue", None, false), + (vec!["id", "status"], vec!["id"], "revenue", None, false), + ( + vec!["id", "status"], + vec!["id", "status"], + "revenue", + None, + true, + ), + (vec!["id"], vec!["id", "status"], "average", None, false), + ( + vec!["id"], + vec!["id", "status"], + "revenue", + Some("orders.revenue > 1"), + false, + ), + ] { + let mut model: Model = serde_json::from_value(source.clone()).unwrap(); + model.primary_key = keys.first().copied().unwrap_or_default().into(); + model.primary_key_columns = keys.iter().map(|key| (*key).into()).collect(); + model.pre_aggregations[0].dimensions = + Some(dimensions.iter().map(|name| (*name).into()).collect()); + let mut graph = SemanticGraph::new(); + graph.add_model(model).unwrap(); + let mut query = SemanticQuery::new() + .with_metrics(vec![format!("orders.{metric}")]) + .with_dimensions(vec!["orders.id".into()]) + .with_ungrouped(true) + .with_use_preaggregations(true); + query.filters = filter.into_iter().map(str::to_owned).collect(); + let sql = SqlGenerator::new(&graph).generate(&query).unwrap(); + assert_eq!(sql.contains("used_preagg=true"), expected, "{sql}"); + if expected { + assert!(!sql.contains("GROUP BY"), "{sql}"); + assert!(sql.contains("revenue_raw AS revenue"), "{sql}"); + } + } + } + #[test] fn distinct_rollup_requires_exact_projected_grain() { let mut model = rollup_model(); @@ -6565,9 +7167,9 @@ mod tests { let sql = generator.generate(&query).unwrap(); - assert!(sql.contains("sales.amount AS revenue_raw"), "{sql}"); + assert!(sql.contains("amount AS revenue_raw"), "{sql}"); + assert!(sql.contains("SUM(sales_cte.revenue_raw)"), "{sql}"); assert!(sql.contains("FROM sales"), "{sql}"); - assert!(!sql.contains("\n amount AS revenue_raw"), "{sql}"); assert!(!sql.contains("FROM orders"), "{sql}"); } @@ -6597,8 +7199,8 @@ mod tests { let query = SemanticQuery::new().with_metrics(vec!["conversion_rate".into()]); let sql = generator.generate(&query).unwrap(); - assert!(sql.contains("orders.signups AS signups_raw"), "{sql}"); - assert!(sql.contains("orders.visitors AS visitors_raw"), "{sql}"); + assert!(sql.contains("signups AS signups_raw"), "{sql}"); + assert!(sql.contains("visitors AS visitors_raw"), "{sql}"); } #[test] @@ -6813,6 +7415,35 @@ models: ); } + #[test] + fn test_quoted_semantic_order_by_rewrites_to_output_aliases() { + let graph = create_test_graph(); + let query = SemanticQuery::new() + .with_metrics(vec!["orders.revenue".into()]) + .with_dimensions(vec!["orders.status".into()]) + .with_order_by(vec![ + "\"orders\".\"revenue\" DESC NULLS FIRST".into(), + "\"orders\".\"status\" ASC NULLS LAST".into(), + ]); + let sql = SqlGenerator::new(&graph).generate(&query).unwrap(); + let statement = polyglot_sql::parse_one(&sql, SOURCE_DIALECT).unwrap(); + let Expression::Select(select) = statement else { + panic!("expected SELECT: {sql}"); + }; + let order = select.order_by.unwrap(); + for (item, expected) in order.expressions.iter().zip(["revenue", "status"]) { + let Expression::Column(column) = &item.this else { + panic!("expected result column: {sql}"); + }; + assert!(column.table.is_none(), "{sql}"); + assert_eq!(column.name.name, expected, "{sql}"); + } + assert!( + sql.contains("revenue DESC NULLS FIRST, status ASC NULLS LAST"), + "{sql}" + ); + } + #[test] fn test_time_comparison_calendar_lookup_partitions_by_non_time_dimensions() { let mut graph = SemanticGraph::new(); diff --git a/sidemantic-rs/src/sql/generator/aggregate_plan.rs b/sidemantic-rs/src/sql/generator/aggregate_plan.rs index 66a984a80..82d19e421 100644 --- a/sidemantic-rs/src/sql/generator/aggregate_plan.rs +++ b/sidemantic-rs/src/sql/generator/aggregate_plan.rs @@ -28,6 +28,7 @@ struct Plan<'a, 'g> { expressions: HashMap, active: HashSet, cross_source_calculation: bool, + inline_aggregates: bool, } fn unsupported(capability: &str) -> SidemanticError { @@ -36,10 +37,158 @@ fn unsupported(capability: &str) -> SidemanticError { } } +fn count_aggregate(expression: &Expression) -> bool { + match expression { + Expression::Count(_) + | Expression::CountIf(_) + | Expression::ApproxDistinct(_) + | Expression::ApproxCountDistinct(_) => true, + Expression::Filter(filter) => count_aggregate(&filter.this), + Expression::WithinGroup(group) => count_aggregate(&group.this), + _ => false, + } +} + impl<'a, 'g> Plan<'a, 'g> { + /// Split authored aggregate calls before expanding scalar metric references. + /// Each call must read one source; arithmetic combines its grouped output. + fn expand_calculation( + &mut self, + node: &mut serde_json::Value, + context: Option<&str>, + bindings: &HashSet, + ) -> Result<()> { + let aggregate_node = crate::core::is_aggregate_ast_node(node); + if let serde_json::Value::Object(fields) = node { + let kind = (fields.len() == 1).then(|| fields.keys().next().unwrap().as_str()); + if kind == Some("lambda") { + let lambda = fields.get_mut("lambda").unwrap(); + let mut bindings = bindings.clone(); + if let Some(parameters) = lambda["parameters"].as_array() { + bindings.extend( + parameters + .iter() + .filter_map(|parameter| parameter["name"].as_str().map(str::to_owned)), + ); + } + return self.expand_calculation(&mut lambda["body"], context, &bindings); + } + if matches!( + kind, + Some("window" | "window_function" | "select" | "subquery" | "raw") + ) { + return Err(unsupported("calculation_shape")); + } + let aggregate = aggregate_node || matches!(kind, Some("filter" | "within_group")); + if aggregate || kind == Some("column") { + let expression: Expression = serde_json::from_value(node.clone()) + .map_err(|error| SidemanticError::SqlGeneration(error.to_string()))?; + let sql = self.generator.emit_expression(&expression)?; + let replacement = if aggregate { + self.inline_aggregates = true; + let columns = semantic_column_references(&sql)?; + let mut owners = HashSet::new(); + for column in columns { + // Hoisting an aggregate out of a lambda would change + // the binding of its local inputs. + if bindings.contains(column.model.as_deref().unwrap_or(&column.field)) { + return Err(unsupported("calculation_shape")); + } + let owner = column + .model + .as_deref() + .or(context) + .ok_or_else(|| unsupported("unscoped_leaf"))?; + if self.generator.graph.get_model(owner).is_none() { + return Err(unsupported("cross_source_raw_input")); + } + owners.insert(owner.to_string()); + } + if owners.is_empty() { + owners.extend(context.map(str::to_string)); + } + if owners.len() != 1 { + return Err(unsupported("cross_source_raw_input")); + } + let model = owners.into_iter().next().unwrap(); + if !self.models.contains(&model) { + self.models.push(model.clone()); + } + let alias = format!("__sidemantic_metric_{}", self.leaves.len()); + let mut metric = Metric::derived(&alias, sql); + metric.sql_is_complete = true; + self.leaves.push(Leaf { + reference: format!("{model}.{alias}"), + model: model.clone(), + metric, + alias: alias.clone(), + }); + let output = format!( + "{}.{}", + self.generator.quote_identifier(&format!("{model}_preagg")), + self.generator.quote_identifier(&alias), + ); + if count_aggregate(&expression) { + format!("COALESCE({output}, 0)") + } else { + output + } + } else { + let column = semantic_column_references(&sql)?.remove(0); + if bindings.contains(column.model.as_deref().unwrap_or(&column.field)) { + return Ok(()); + } + self.expand(&column.name(), context)? + }; + *node = + serde_json::to_value(parse_semantic_expression(&format!("({replacement})"))?) + .map_err(|error| SidemanticError::SqlGeneration(error.to_string()))?; + return Ok(()); + } + } + match node { + serde_json::Value::Object(fields) => { + for child in fields.values_mut() { + self.expand_calculation(child, context, bindings)?; + } + } + serde_json::Value::Array(children) => { + for child in children { + self.expand_calculation(child, context, bindings)?; + } + } + _ => {} + } + Ok(()) + } + + fn graph_metric_context(&self, reference: &str, metric: &Metric) -> Result> { + if let Some(owner) = self.generator.graph.metric_owner(reference) { + return Ok(Some(owner.to_string())); + } + // Imported graph aggregates can bind their source in qualified SQL + // without an explicit owner annotation. Scalar calculations stay unowned. + if metric.agg.is_some() && !metric.sql_is_complete { + let owners = self + .generator + .graph_metric_owner_models(reference, metric)?; + if owners.len() == 1 { + return Ok(owners.into_iter().next()); + } + } + Ok(None) + } + fn resolve(&self, reference: &str, context: Option<&str>) -> Result> { let graph = self.generator.graph; if let Some((model_name, name)) = reference.split_once('.') { + if let Some(metric) = graph.get_metric(reference) { + return Ok(Some(ResolvedMetric { + reference: reference.to_string(), + context: self.graph_metric_context(reference, metric)?, + metric: metric.clone(), + })); + } return Ok(graph.get_model(model_name).and_then(|model| { model.get_metric(name).map(|metric| ResolvedMetric { reference: reference.to_string(), @@ -63,7 +212,7 @@ impl<'a, 'g> Plan<'a, 'g> { if let Some(metric) = graph.get_metric(reference) { return Ok(Some(ResolvedMetric { reference: reference.to_string(), - context: graph.metric_owner(reference).map(str::to_string), + context: self.graph_metric_context(reference, metric)?, metric: metric.clone(), })); } @@ -108,8 +257,10 @@ impl<'a, 'g> Plan<'a, 'g> { { return Err(unsupported("calculation_filters")); } - let leaf_type = if metric.sql_is_complete { + let leaf_type = if metric.sql_is_complete && resolved.context.is_some() { MetricType::Simple + } else if metric.sql_is_complete { + MetricType::Derived } else { metric.r#type.clone() }; @@ -196,16 +347,12 @@ impl<'a, 'g> Plan<'a, 'g> { metric.name )) })?; - let mut replacements = HashMap::new(); - for column in semantic_column_references(sql)? { - if column.aggregate_input { - return Err(unsupported("inline_aggregate")); - } - let expanded = self.expand(&column.name(), resolved.context.as_deref())?; - replacements.insert((column.model, column.field), format!("({expanded})")); - } - let expression = - replace_semantic_columns(parse_semantic_expression(sql)?, &replacements)?; + let sql = crate::core::replace_model_placeholder(sql, resolved.context.as_deref())?; + let mut ast = serde_json::to_value(parse_semantic_expression(&sql)?) + .map_err(|error| SidemanticError::SqlGeneration(error.to_string()))?; + self.expand_calculation(&mut ast, resolved.context.as_deref(), &HashSet::new())?; + let expression = serde_json::from_value(ast) + .map_err(|error| SidemanticError::SqlGeneration(error.to_string()))?; self.generator.emit_expression(&expression)? } _ => return Err(unsupported("calculation_shape")), @@ -247,6 +394,36 @@ fn dimension_alias(index: usize) -> String { format!("__sidemantic_dimension_{index}") } +fn independent_source_path( + graph: &SemanticGraph, + from: &str, + to: &str, + dimensions: &[DimensionRef], +) -> Result> { + match graph.find_join_path(from, to) { + Ok(path) => Ok(Some(path)), + Err(SidemanticError::AmbiguousJoinPath { .. }) + if dimensions.iter().any(|dimension| { + [from, to].into_iter().all(|source| { + graph + .find_join_path(source, &dimension.model) + .is_ok_and(|path| { + path.steps + .iter() + .all(|step| step.relationship_type != RelationshipType::Cross) + }) + }) + }) => + { + // Both children have a unique keyed route to a requested grouping + // model. Their aggregate outputs meet there; no source-to-source + // row join needs to choose among the alternate graph routes. + Ok(None) + } + Err(error) => Err(error), + } +} + /// Normalize the ordinary compiler's public outputs at the child boundary. /// Each child has its own collision set because it selects different measures. fn child_projection( @@ -307,6 +484,7 @@ pub(super) fn try_generate( expressions: HashMap::new(), active: HashSet::new(), cross_source_calculation: false, + inline_aggregates: false, }; let mut outputs = Vec::new(); for reference in &query.metrics { @@ -340,8 +518,11 @@ pub(super) fn try_generate( } } let mut effective_query; - let query = if plan.models.len() == 1 && !query.skip_default_time_dimensions { + let query = if !query.skip_default_time_dimensions { effective_query = query.clone(); + // Defaults belong to selected model metric references, not every source + // discovered while expanding graph calculations. Resolve them once for + // the whole query before creating independent aggregate children. effective_query.dimensions = generator.apply_default_time_dimensions(&query.metrics, &query.dimensions)?; effective_query.skip_default_time_dimensions = true; @@ -350,13 +531,22 @@ pub(super) fn try_generate( query }; let dimensions = generator.parse_dimension_refs(&query.dimensions)?; + // Inline splitting exists to combine independent sources. The ordinary + // single-source compiler binds semantic dimension inputs and owns authored + // aggregate/window expressions without manufacturing physical columns. + if plan.inline_aggregates && plan.models.len() < 2 { + return Ok(None); + } // Cartesian products require every participating source even in a child // selecting only one source's measure: an empty sibling annihilates rows. // Keyed independent aggregates keep their existing separate populations. let mut population_models = query.required_population_models.clone(); for (index, from) in plan.models.iter().enumerate() { for to in plan.models.iter().skip(index + 1) { - let path = generator.graph.find_join_path(from, to)?; + let Some(path) = independent_source_path(generator.graph, from, to, &dimensions)? + else { + continue; + }; if path .steps .iter() @@ -417,19 +607,9 @@ pub(super) fn try_generate( if query.ungrouped || !query.table_calculations.is_empty() { return Err(unsupported("cross_grain_query_shape")); } - if !query.skip_default_time_dimensions - && plan.models.iter().any(|model| { - generator - .graph - .get_model(model) - .is_some_and(|model| model.default_time_dimension.is_some()) - }) - { - return Err(unsupported("cross_grain_default_time_dimension")); - } // Independent populations still require a declared, supported model graph. for model in plan.models.iter().skip(1) { - generator.graph.find_join_path(&plan.models[0], model)?; + independent_source_path(generator.graph, &plan.models[0], model, &dimensions)?; } for dimension in &dimensions { if generator @@ -467,10 +647,92 @@ pub(super) fn try_generate( }) .collect(); let mut row_filters = Vec::new(); + let mut window_filters: HashMap> = HashMap::new(); let mut aggregate_filters = Vec::new(); for filter in filters { let sql = generator.emit_expression(&filter)?; let columns = crate::core::outer_semantic_column_references(&sql)?; + let window_owners: HashSet<_> = columns + .iter() + .filter_map(|column| { + let owner = column.model.as_deref()?; + generator + .graph + .get_model(owner)? + .get_dimension(&column.field)? + .window + .as_ref() + .map(|_| owner.to_string()) + }) + .collect(); + if !window_owners.is_empty() { + // Window predicates filter source rows after window evaluation and + // before that source's aggregate. Metric references in the same + // predicate therefore refer to their row inputs, not outer totals. + let mut replacements = HashMap::new(); + for column in &columns { + if let Some(metric) = plan.resolve(&column.name(), None)? { + let owner = metric + .context + .as_deref() + .ok_or_else(|| unsupported("mixed_row_aggregate_filter"))?; + if !window_owners.contains(owner) + || metric.metric.r#type != MetricType::Simple + || metric.metric.sql_is_complete + { + return Err(unsupported("mixed_row_aggregate_filter")); + } + let source_model = generator.graph.get_model(owner).unwrap(); + let mut raw = generator.metric_raw_expression(&metric.metric, source_model)?; + if !metric.metric.filters.is_empty() { + let predicate = generator.normalize_metric_filters( + &metric.metric.filters, + owner, + &generator.model_alias(owner), + )?; + raw = format!("CASE WHEN {predicate} THEN {raw} END"); + } + raw = raw.replace("{model}", owner); + let mut inputs = HashMap::new(); + for input in semantic_column_references(&raw)? { + if input.model.as_deref().is_some_and(|qualifier| { + qualifier != owner + && qualifier != generator.model_alias(owner) + && qualifier != source_model.table_name() + }) { + return Err(unsupported("mixed_row_aggregate_filter")); + } + // Bind physical inputs directly to the CTE. Going back + // through semantic names could expand a same-named + // computed dimension instead of the metric's row input. + let qualified = format!( + "{}.{}", + generator.model_alias(owner), + generator.quote_identifier(&input.field) + ); + inputs.insert((input.model, input.field), qualified); + } + let raw = generator.emit_expression(&replace_semantic_columns( + parse_semantic_expression(&raw)?, + &inputs, + )?)?; + replacements.insert( + (column.model.clone(), column.field.clone()), + format!("({raw})"), + ); + } + } + let predicate = generator.emit_expression( + &crate::core::replace_outer_semantic_columns(filter, &replacements)?, + )?; + for owner in window_owners { + window_filters + .entry(owner) + .or_default() + .push(predicate.clone()); + } + continue; + } let mut replacements = HashMap::new(); let mut has_metric = false; let mut has_raw = false; @@ -512,6 +774,9 @@ pub(super) fn try_generate( child.required_population_models = population_models.clone(); child.metrics = leaves.iter().map(|leaf| leaf.reference.clone()).collect(); child.filters = row_filters.clone(); + child + .filters + .extend(window_filters.get(model).into_iter().flatten().cloned()); child.segments.clear(); child.order_by.clear(); child.limit = None; @@ -534,13 +799,22 @@ pub(super) fn try_generate( &dimensions, &metrics, fanout_models.contains(model), + plan.models.len() > 1, )? } else { let mut projection = child_projection(generator, &dimensions, &leaves)?; if query.with_totals && !dimensions.is_empty() { projection.push("__sidemantic_source._is_total AS _is_total".into()); } - let sql = generator.generate_from_model(&child, Some(model))?; + // Independent sources retain unmatched rows. Grouping order must + // not choose which population contributes to the calculation. + let anchor = if plan.models.len() > 1 { + Some(model.clone()) + } else { + generator + .query_base_model(&dimensions, &generator.parse_metric_refs(&child.metrics)?) + }; + let sql = generator.generate_from_model(&child, anchor.as_deref())?; format!( "SELECT {}\nFROM (\n{sql}\n) AS __sidemantic_source", projection.join(", ") @@ -791,6 +1065,282 @@ mod tests { assert_valid_sql(&sql); } + #[test] + fn cross_source_graph_alias_does_not_require_a_single_owner() { + let graph = graph(); + let generator = SqlGenerator::new(&graph); + let mut query = SemanticQuery::new().with_metrics(vec!["ratio".into()]); + query.aliases.insert("ratio".into(), "value".into()); + let sql = generator.generate(&query).unwrap(); + assert!(sql.contains("orders_preagg AS"), "{sql}"); + assert!(sql.contains("customers_preagg AS"), "{sql}"); + assert!(sql.contains("AS \"value\""), "{sql}"); + assert_valid_sql(&sql); + } + + #[test] + fn dotted_graph_calculation_is_not_reinterpreted_as_a_model_path() { + let mut graph = graph(); + graph + .add_metric_unvalidated(Metric::derived("business.ratio", "ratio * 2")) + .unwrap(); + let sql = compile(&graph, &["business.ratio"], &[]).unwrap(); + assert!(sql.contains("orders_preagg AS"), "{sql}"); + assert!(sql.contains("customers_preagg AS"), "{sql}"); + assert!(sql.contains("AS \"business.ratio\""), "{sql}"); + assert_valid_sql(&sql); + } + + #[test] + fn grouped_children_preserve_each_source_domain() { + let graph = graph(); + for dimension in ["customers.region", "orders.region"] { + let query = SemanticQuery::new() + .with_metrics(vec!["ratio".into()]) + .with_dimensions(vec![dimension.into()]); + let sql = SqlGenerator::new(&graph).generate(&query).unwrap(); + for owner in ["orders", "customers"] { + assert!( + sql.contains(&format!("FROM {owner}_cte AS {owner}_cte")), + "{sql}" + ); + } + assert_valid_sql(&sql); + } + } + + #[test] + fn inline_cross_source_aggregates_are_split_before_scalar_arithmetic() { + let mut graph = graph(); + for (name, expression) in [ + ( + "inline_ratio", + "COUNT(orders.id) * 1.0 / NULLIF(COUNT(customers.id), 0)", + ), + ( + "inline_sum", + "SUM(orders.amount) + SUM(CASE WHEN customers.id > 1 THEN customers.id ELSE 0 END)", + ), + ] { + let mut metric = Metric::derived(name, expression); + metric.sql_is_complete = true; + graph.add_metric_unvalidated(metric).unwrap(); + let sql = compile(&graph, &[name], &[]).unwrap(); + assert!(sql.contains("orders_preagg AS"), "{sql}"); + assert!(sql.contains("customers_preagg AS"), "{sql}"); + assert!(sql.contains("CROSS JOIN customers_preagg"), "{sql}"); + assert_valid_sql(&sql); + } + } + + #[test] + fn aggregate_splitting_preserves_lambda_bindings_and_expands_free_metrics() { + let mut graph = graph(); + graph + .add_metric_unvalidated(Metric::derived( + "median_plus_count", + concat!( + "LIST_AGGREGATE(LIST_TRANSFORM(LIST_DISTINCT(", + "LIST(STRUCT_PACK(k := orders.id, v := orders.amount))), ", + "x -> x.v), 'quantile_cont', 0.5) + customers.customer_count", + ), + )) + .unwrap(); + let sql = compile(&graph, &["median_plus_count"], &[]).unwrap(); + assert!(sql.contains("x.v"), "{sql}"); + assert!(sql.contains("customers_preagg"), "{sql}"); + assert_valid_sql(&sql); + } + + #[test] + fn inline_filtered_distinct_count_restores_absent_sources_to_zero() { + let mut graph = graph(); + let mut metric = Metric::derived( + "filtered_count_ratio", + "COUNT(DISTINCT orders.id) FILTER (WHERE orders.amount > 10) / NULLIF(COUNT(customers.id), 0)", + ); + metric.sql_is_complete = true; + graph.add_metric_unvalidated(metric).unwrap(); + let sql = compile(&graph, &["filtered_count_ratio"], &[]).unwrap(); + assert!( + sql.contains("COALESCE(orders_preagg.__sidemantic_metric_"), + "{sql}" + ); + assert!( + sql.contains("FILTER(WHERE") || sql.contains("FILTER (WHERE"), + "{sql}" + ); + assert_valid_sql(&sql); + } + + #[test] + fn inline_cross_source_metric_filters_are_not_discarded() { + let mut graph = graph(); + let mut metric = Metric::derived( + "filtered_inline", + "SUM(orders.amount) + COUNT(customers.id)", + ); + metric.sql_is_complete = true; + metric.filters.push("orders.amount > 10".into()); + graph.add_metric_unvalidated(metric).unwrap(); + let generator = SqlGenerator::new(&graph); + let mut plan = Plan { + generator: &generator, + leaves: Vec::new(), + models: Vec::new(), + expressions: HashMap::new(), + active: HashSet::new(), + cross_source_calculation: false, + inline_aggregates: false, + }; + assert!(matches!(plan.expand("filtered_inline", None), + Err(SidemanticError::UnsupportedSemanticFeatures { capabilities }) + if capabilities == vec!["aggregation.calculation_filters"])); + } + + #[test] + fn inline_single_source_and_window_aggregates_keep_existing_compiler() { + for sql in [ + "COUNT(orders.virtual_row)", + "SUM(orders.amount) / NULLIF(SUM(SUM(orders.amount)) OVER (), 0)", + ] { + let mut graph = graph(); + let mut orders = graph.get_model("orders").unwrap().clone(); + orders + .dimensions + .push(Dimension::categorical("virtual_row").with_sql("1")); + graph.replace_model(orders).unwrap(); + let mut metric = Metric::derived("inline", sql); + metric.sql_is_complete = true; + graph.add_metric_unvalidated(metric).unwrap(); + let query = SemanticQuery::new().with_metrics(vec!["inline".into()]); + assert!(try_generate(&SqlGenerator::new(&graph), &query) + .unwrap() + .is_none()); + } + } + + #[test] + fn imported_aggregate_leaves_use_qualified_inputs_and_requested_group_routes() { + let mut graph = SemanticGraph::new(); + for name in ["clicks", "impressions"] { + let mut model = Model::new(name, "id").with_table(name); + for target in ["campaigns", "publishers"] { + let mut relationship = Relationship::many_to_one(target); + relationship.foreign_key = Some(format!("{target}_id")); + model.relationships.push(relationship); + } + graph.add_model(model).unwrap(); + } + for name in ["campaigns", "publishers"] { + graph + .add_model( + Model::new(name, "id") + .with_table(name) + .with_dimension(Dimension::categorical("name")), + ) + .unwrap(); + } + let mut click_count = Metric::count("click_count"); + click_count.sql = Some("clicks.id".into()); + graph.add_metric_unvalidated(click_count).unwrap(); + let mut ratio = Metric::derived( + "ctr", + "COUNT(clicks.id) * 1.0 / NULLIF(COUNT(impressions.id), 0)", + ); + ratio.sql_is_complete = true; + graph.add_metric_unvalidated(ratio).unwrap(); + graph.set_metric_scopes(HashMap::new()).unwrap(); + let query = SemanticQuery::new() + .with_metrics(vec!["click_count".into(), "ctr".into()]) + .with_dimensions(vec!["campaigns.name".into()]); + let sql = SqlGenerator::new(&graph).generate(&query).unwrap(); + assert!(sql.contains("clicks_preagg AS"), "{sql}"); + assert!(sql.contains("impressions_preagg AS"), "{sql}"); + assert!(!sql.contains("publishers_cte"), "{sql}"); + assert_valid_sql(&sql); + let mut ungrouped = query; + ungrouped.dimensions.clear(); + assert!(matches!( + SqlGenerator::new(&graph).generate(&ungrouped), + Err(SidemanticError::AmbiguousJoinPath { .. }) + )); + } + + #[test] + fn legacy_window_metric_predicate_is_applied_before_aggregation() { + let mut legacy = SemanticGraph::new(); + for model in graph().models() { + legacy.add_model(model.clone()).unwrap(); + } + let mut orders = legacy.get_model("orders").unwrap().clone(); + let mut next_status = Dimension::categorical("next_status").with_sql("status"); + next_status.window = Some("LEAD(status) OVER (ORDER BY id)".into()); + orders.dimensions.push(next_status); + legacy.replace_model(orders).unwrap(); + let query = SemanticQuery::new() + .with_metrics(vec![ + "orders.revenue".into(), + "customers.customer_count".into(), + ]) + .with_dimensions(vec!["orders.region".into()]) + .with_filters(vec![ + "orders.next_status = 'complete' OR orders.revenue > 100".into(), + ]); + let sql = SqlGenerator::new(&legacy).generate(&query).unwrap(); + let (orders_sql, customers_sql) = sql.split_once("customers_preagg AS (").unwrap(); + assert!(orders_sql.contains("'complete'"), "{sql}"); + assert!(!orders_sql.contains("HAVING"), "{sql}"); + assert!(!customers_sql.contains("'complete'"), "{sql}"); + assert_valid_sql(&sql); + } + + #[test] + fn complete_count_without_inputs_retains_owner_join_and_dimension_domain() { + let mut graph = graph(); + let mut orders = graph.get_model("orders").unwrap().clone(); + let mut count = Metric::derived("opaque_count", "COUNT(*)"); + count.sql_is_complete = true; + orders.metrics.push(count); + graph.replace_model(orders).unwrap(); + let query = SemanticQuery::new() + .with_metrics(vec!["orders.opaque_count".into()]) + .with_dimensions(vec!["customers.region".into()]); + let sql = SqlGenerator::new(&graph).generate(&query).unwrap(); + assert!(sql.contains("COUNT(*) AS __sidemantic_metric_0"), "{sql}"); + assert!(sql.contains("FROM customers_cte AS customers_cte"), "{sql}"); + assert!(sql.contains("LEFT JOIN orders_cte AS orders_cte"), "{sql}"); + assert_valid_sql(&sql); + } + + #[test] + fn mixed_window_predicate_filters_only_its_source_before_aggregation() { + let mut graph = graph(); + let mut orders = graph.get_model("orders").unwrap().clone(); + let mut next_status = crate::core::Dimension::categorical("next_status").with_sql("status"); + next_status.window = Some("LEAD(status) OVER (ORDER BY id)".into()); + orders.dimensions.push(next_status); + graph.replace_model(orders).unwrap(); + let query = SemanticQuery::new() + .with_metrics(vec![ + "orders.revenue".into(), + "customers.customer_count".into(), + ]) + .with_dimensions(vec!["orders.region".into()]) + .with_filters(vec![ + "orders.next_status = 'complete' OR orders.revenue > 100".into(), + ]); + let sql = SqlGenerator::new(&graph).generate(&query).unwrap(); + let (orders_sql, rest) = sql.split_once("customers_preagg AS (").unwrap(); + assert!(orders_sql.contains("'complete'"), "{sql}"); + assert!( + orders_sql.contains("amount) > 100") || orders_sql.contains("amount > 100"), + "{sql}" + ); + assert!(!rest.contains("'complete'"), "{sql}"); + assert_valid_sql(&sql); + } + #[test] fn proxies_and_local_names_expand_to_aggregate_owners() { let graph = graph(); @@ -984,4 +1534,52 @@ mod tests { assert!(!sql.contains("orders_preagg AS")); assert_valid_sql(&sql); } + + #[test] + fn default_time_dimensions_follow_selected_metrics_across_sources() { + let mut graph = graph(); + for (owner, field, grain) in [ + ("orders", "ordered_at", "month"), + ("customers", "signed_up_at", "week"), + ] { + let mut model = graph.get_model(owner).unwrap().clone(); + model.dimensions.push(Dimension::time(field)); + model.default_time_dimension = Some(field.into()); + model.default_grain = Some(grain.into()); + graph.replace_model(model).unwrap(); + } + for (metrics, dimensions, skip, expected_month, expected_week) in [ + (vec!["ratio"], vec![], false, false, false), + ( + vec!["orders.revenue", "customers.customer_count"], + vec![], + false, + true, + true, + ), + ( + vec!["orders.revenue", "customers.customer_count"], + vec!["orders.ordered_at__day"], + false, + false, + true, + ), + ( + vec!["orders.revenue", "customers.customer_count"], + vec![], + true, + false, + false, + ), + ] { + let query = SemanticQuery::new() + .with_metrics(metrics.into_iter().map(str::to_string).collect()) + .with_dimensions(dimensions.into_iter().map(str::to_string).collect()) + .with_skip_default_time_dimensions(skip); + let sql = SqlGenerator::new(&graph).generate(&query).unwrap(); + assert_eq!(sql.contains("ordered_at__month"), expected_month, "{sql}"); + assert_eq!(sql.contains("signed_up_at__week"), expected_week, "{sql}"); + assert_valid_sql(&sql); + } + } } diff --git a/sidemantic-rs/src/sql/generator/cohort.rs b/sidemantic-rs/src/sql/generator/cohort.rs index 366482c82..f77d25c73 100644 --- a/sidemantic-rs/src/sql/generator/cohort.rs +++ b/sidemantic-rs/src/sql/generator/cohort.rs @@ -326,13 +326,13 @@ impl SqlGenerator<'_> { )); } let mut order = Vec::new(); + let names: Vec<_> = output_dimensions + .iter() + .map(|dimension| dimension.alias.as_str()) + .chain(std::iter::once(metric.name.as_str())) + .collect(); for item in &query.order_by { - let (field, direction) = item - .rsplit_once(' ') - .filter(|(_, direction)| { - direction.eq_ignore_ascii_case("asc") || direction.eq_ignore_ascii_case("desc") - }) - .unwrap_or((item, "")); + let (field, direction) = crate::sql::split_order_field(item, &names); let name = field .strip_prefix(&format!("{}.", model.name)) .unwrap_or(field); @@ -476,4 +476,35 @@ mod tests { } } } + + #[test] + fn owned_graph_cohort_approximate_aggregate_uses_qualified_inner_results() { + let input = json!({ + "version": 1, "input_dialect": "duckdb", + "models": [{ + "name": "events", "table": "events", "primary_key": "id", + "dimensions": [{"name": "person", "type": "categorical"}] + }], + "metrics": [{ + "name": "qualified", "type": "cohort", "entity": "person", + "agg": "approx_count_distinct", "sql": "amount", + "inner_metrics": [{"name": "amount", "agg": "sum", "sql": "raw_amount"}], + "having": "amount > 10" + }], + "metric_owners": {"qualified": "events"} + }); + let sql = compile_with_semantic_input( + &input.to_string(), + &json!({"metrics": ["qualified"]}).to_string(), + ) + .unwrap(); + // The graph metric's SQL names an inner result, not a source column. + assert!( + sql.contains("APPROX_COUNT_DISTINCT(cohort_sub.\"amount\")"), + "{sql}" + ); + assert!(sql.contains("SUM((\"raw_amount\"))"), "{sql}"); + assert!(sql.contains("HAVING"), "{sql}"); + polyglot_sql::parse_one(&sql, DialectType::DuckDB).unwrap(); + } } diff --git a/sidemantic-rs/src/sql/generator/conversion.rs b/sidemantic-rs/src/sql/generator/conversion.rs index 85c96deb1..8f7db5e85 100644 --- a/sidemantic-rs/src/sql/generator/conversion.rs +++ b/sidemantic-rs/src/sql/generator/conversion.rs @@ -117,15 +117,17 @@ impl SqlGenerator<'_> { if !output_names.insert(dimension.alias.to_ascii_lowercase()) { return Err(unsupported("output_alias")); } - let mut expression = self.conversion_source_expression(model, &dimension.name)?; - if let Some(grain) = &dimension.granularity { - expression = self.date_trunc_sql(grain, &expression)?; - } + let expression = self.conversion_source_expression(model, &dimension.name)?; let internal = format!("__funnel_group_{index}"); projection.push(format!("{expression} AS {internal}")); - secured - .dimensions - .push(crate::core::Dimension::categorical(&internal)); + let mut group_dimension = crate::core::Dimension::categorical(&internal); + if let Some(grain) = &dimension.granularity { + // Bucket in the first-step grouping, matching the sequential + // planner's source grain. Pre-bucketing the repeated source can + // give DuckDB incorrect NULL ordering statistics through joins. + group_dimension.sql = Some(self.date_trunc_sql(grain, &internal)?); + } + secured.dimensions.push(group_dimension); inner_dimensions.push(DimensionRef { model: model.name.clone(), name: internal.clone(), @@ -206,13 +208,9 @@ impl SqlGenerator<'_> { output.join(", ") ); let mut order = Vec::new(); + let names: Vec<_> = output_names.iter().map(String::as_str).collect(); for item in &query.order_by { - let (field, direction) = item - .rsplit_once(' ') - .filter(|(_, direction)| { - direction.eq_ignore_ascii_case("asc") || direction.eq_ignore_ascii_case("desc") - }) - .unwrap_or((item, "")); + let (field, direction) = crate::sql::split_order_field(item, &names); let name = field .strip_prefix(&format!("{}.", model.name)) .unwrap_or(field); @@ -328,13 +326,13 @@ impl SqlGenerator<'_> { } } let mut ordering = Vec::new(); + let names: Vec<_> = dimensions + .iter() + .map(|dimension| dimension.alias.as_str()) + .chain(std::iter::once(metric.name.as_str())) + .collect(); for item in &query.order_by { - let (field, direction) = item - .rsplit_once(' ') - .filter(|(_, direction)| { - direction.eq_ignore_ascii_case("asc") || direction.eq_ignore_ascii_case("desc") - }) - .unwrap_or((item, "")); + let (field, direction) = crate::sql::split_order_field(item, &names); let alias = if field == metric.name || field == format!("{}.{}", model.name, metric.name) { &metric.name @@ -451,6 +449,70 @@ impl SqlGenerator<'_> { mod tests { use super::*; + #[test] + fn both_conversion_algorithms_accept_null_ordering_but_reject_nonoutputs() { + crate::semantic_input::with_semantic_stack(|| { + for multistep in [false, true] { + let mut metric = Metric::new("funnel"); + metric.r#type = MetricType::Conversion; + metric.entity = Some("user_id".into()); + if multistep { + metric.steps = Some(vec![ + "event_type = 'signup'".into(), + "event_type = 'buy'".into(), + ]); + } else { + metric.base_event = Some("signup".into()); + metric.conversion_event = Some("buy".into()); + } + let mut graph = SemanticGraph::new(); + graph + .add_model( + Model::new("events", "id") + .with_table("events") + .with_dimension(crate::core::Dimension::time("timestamp")) + .with_dimension(crate::core::Dimension::categorical("event_type")) + .with_metric(metric), + ) + .unwrap(); + let generator = SqlGenerator::new(&graph); + let reference = MetricRef { + model: "events".into(), + name: "funnel".into(), + alias: "funnel".into(), + graph_metric: false, + }; + for suffix in ["ASC NULLS FIRST", "DESC NULLS LAST", "NULLS LAST"] { + let query = + SemanticQuery::new().with_order_by(vec![format!("events.funnel {suffix}")]); + let sql = generator + .generate_scoped_conversion(&query, &reference, &[]) + .unwrap(); + assert!( + sql.contains(&format!("ORDER BY \"funnel\" {suffix}")), + "{sql}" + ); + polyglot_sql::parse_one(&sql, DialectType::DuckDB).unwrap(); + } + for field in [ + "funnel; SELECT 2", + "funnel DESC LIMIT 1", + "random()", + "missing", + ] { + let query = SemanticQuery::new().with_order_by(vec![field.into()]); + assert!(matches!( + generator.generate_scoped_conversion(&query, &reference, &[]), + Err(SidemanticError::UnsupportedSemanticFeatures { capabilities }) + if capabilities == vec!["metric.conversion_order_by"] + )); + } + } + Ok(()) + }) + .unwrap(); + } + #[test] fn multistep_requires_a_time_dimension() { let mut metric = Metric::new("funnel"); diff --git a/sidemantic-rs/src/sql/generator/fanout_complete.rs b/sidemantic-rs/src/sql/generator/fanout_complete.rs index a91c9bf95..24fcb4b5b 100644 --- a/sidemantic-rs/src/sql/generator/fanout_complete.rs +++ b/sidemantic-rs/src/sql/generator/fanout_complete.rs @@ -14,6 +14,7 @@ fn unsupported(shape: &str) -> SidemanticError { struct Inputs<'a> { generator: &'a SqlGenerator<'a>, + owner: &'a str, model: Model, names: HashSet, references: Vec, @@ -32,8 +33,7 @@ impl Inputs<'_> { let mut metric = Metric::sum(&name, sql); metric.filters = filters.to_vec(); self.model.metrics.push(metric); - self.references - .push(format!("{}.{}", self.model.name, name)); + self.references.push(format!("{}.{}", self.owner, name)); self.generator.quote_identifier(&name) } } @@ -45,6 +45,7 @@ pub(super) fn generate_entity_aggregates( dimensions: &[DimensionRef], metrics: &[(&Metric, &str)], deduplicate: bool, + independent_source: bool, ) -> Result { let model = generator.graph.get_model(owner).unwrap(); if deduplicate && model.primary_keys().is_empty() { @@ -71,6 +72,7 @@ pub(super) fn generate_entity_aggregates( .collect(); let mut inputs = Inputs { generator, + owner, model: model.clone(), names, references: Vec::new(), @@ -147,9 +149,9 @@ pub(super) fn generate_entity_aggregates( }; selections.push(format!("{sql} AS {}", generator.quote_identifier(alias))); } - // COUNT(*) can be the only output, but the row compiler still needs one - // projected input when no keys or dimensions were required. - if inputs.references.is_empty() && dimensions.is_empty() { + // COUNT(*) can be the only output. Retain its source in the row query even + // when every grouping dimension belongs to a different model. + if inputs.references.is_empty() { inputs.add("1".into(), &[]); } let mut graph = generator.graph.clone(); @@ -167,7 +169,15 @@ pub(super) fn generate_entity_aggregates( rows.ungrouped = true; rows.with_totals = false; rows.use_preaggregations = false; - let row_sql = row_generator.generate_from_model(&rows, Some(owner))?; + // Independent children retain their own population before their grouped + // outputs are joined. A single complete expression retains the ordinary + // dimension-domain contract, including COUNT(*) on null-extended rows. + let source = if independent_source { + Some(owner.to_string()) + } else { + row_generator.query_base_model(dimensions, &row_generator.parse_metric_refs(&rows.metrics)?) + }; + let row_sql = row_generator.generate_from_model(&rows, source.as_deref())?; let mut collisions = HashMap::new(); for dimension in dimensions { *collisions.entry(dimension.alias.clone()).or_insert(0usize) += 1; diff --git a/sidemantic-rs/src/sql/generator/imported_totals.rs b/sidemantic-rs/src/sql/generator/imported_totals.rs new file mode 100644 index 000000000..fa2d23658 --- /dev/null +++ b/sidemantic-rs/src/sql/generator/imported_totals.rs @@ -0,0 +1,253 @@ +//! Imported calculations retain their source-measure and population semantics. +use super::*; + +impl SqlGenerator<'_> { + /// LookML post-SQL calculations encode base measures as aggregate inputs. + /// Resolve only this explicit import contract; ordinary complete aggregates + /// continue to address physical columns even when a metric shadows a name. + pub(super) fn imported_calculation_expression( + &self, + metric: &Metric, + model_name: &str, + ) -> Result { + let calculation = metric + .meta + .as_ref() + .and_then(|meta| meta.get("table_calculation")) + .and_then(serde_json::Value::as_str); + if !matches!( + calculation, + Some("percent_of_total" | "percent_of_previous") + ) { + return Ok(metric.sql_expr().to_string()); + } + let sql = crate::core::replace_model_placeholder(metric.sql_expr(), Some(model_name))?; + let mut replacements = HashMap::new(); + for column in semantic_column_references(&sql)? { + if !column.aggregate_input { + continue; + } + let owner = column.model.as_deref().unwrap_or(model_name); + let Some(model) = self.graph.get_model(owner) else { + continue; + }; + let Some(base) = model.get_metric(&column.field) else { + continue; + }; + if base.r#type != MetricType::Simple { + continue; + } + let mut input = self.metric_raw_expression(base, model)?; + if !base.filters.is_empty() { + let filters = base + .filters + .iter() + .map(|filter| format!("({filter})")) + .collect::>(); + input = format!("CASE WHEN {} THEN {input} END", filters.join(" AND ")); + } + let input = crate::core::replace_model_placeholder(&input, Some(owner))?; + let qualified = semantic_column_references(&input)? + .into_iter() + .map(|reference| { + let name = format!( + "{}.{}", + self.quote_identifier(reference.model.as_deref().unwrap_or(owner)), + self.quote_identifier(&reference.field) + ); + ((reference.model, reference.field), name) + }) + .collect(); + let input = crate::core::replace_semantic_columns( + parse_semantic_expression(&input)?, + &qualified, + )?; + replacements.insert( + (column.model, column.field), + format!("({})", self.emit_expression(&input)?), + ); + } + self.emit_expression(&crate::core::replace_semantic_columns( + parse_semantic_expression(&sql)?, + &replacements, + )?) + } + + /// Reaggregate the expanded measure over the containing query's exact source + /// before GROUP BY/HAVING/pagination. CTE-level policies and metric filters + /// are retained, as are join predicates and residual WHERE filters. + pub(super) fn expand_imported_totals(&self, projection: &str, source: &str) -> Result { + fn rewrite( + generator: &SqlGenerator<'_>, + value: &mut serde_json::Value, + source: &str, + ) -> Result<()> { + if let Some(function) = value.get("function") { + if function + .get("name") + .and_then(serde_json::Value::as_str) + .is_some_and(|name| name.eq_ignore_ascii_case("__bsl_all")) + { + let expression: Expression = serde_json::from_value(value.clone()) + .map_err(|error| SidemanticError::SqlGeneration(error.to_string()))?; + let Expression::Function(function) = expression else { + unreachable!() + }; + if function.args.len() != 1 { + return Err(SidemanticError::Validation( + "BSL all() requires one measure".into(), + )); + } + let aggregate = generator.emit_expression(&function.args[0])?; + *value = serde_json::to_value(Expression::Raw(Raw { + sql: format!("(SELECT {aggregate} {source})"), + })) + .map_err(|error| SidemanticError::SqlGeneration(error.to_string()))?; + return Ok(()); + } + } + match value { + serde_json::Value::Object(fields) => { + for child in fields.values_mut() { + rewrite(generator, child, source)?; + } + } + serde_json::Value::Array(children) => { + for child in children { + rewrite(generator, child, source)?; + } + } + _ => {} + } + Ok(()) + } + let statement = + crate::semantic_input::dialects::parse(&format!("SELECT {projection}"), self.dialect)?; + let Expression::Select(mut select) = statement else { + unreachable!() + }; + let expression = select.expressions.remove(0); + let mut value = serde_json::to_value(expression) + .map_err(|error| SidemanticError::SqlGeneration(error.to_string()))?; + rewrite(self, &mut value, source)?; + let expression = serde_json::from_value(value) + .map_err(|error| SidemanticError::SqlGeneration(error.to_string()))?; + self.emit_expression(&expression) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::core::{Dimension, Relationship}; + + #[test] + fn bsl_distinct_total_reaggregates_joined_filtered_population() { + let mut graph = SemanticGraph::new(); + graph + .add_model( + Model::new("orders", "id") + .with_table("orders") + .with_dimension(Dimension::categorical("customer_id")) + .with_metric(Metric::count_distinct("users", "user_id")) + .with_metric(Metric::derived("share", "users / __bsl_all(users)")) + .with_relationship( + Relationship::many_to_one("customers").with_keys("customer_id", "id"), + ), + ) + .unwrap(); + graph + .add_model( + Model::new("customers", "id") + .with_table("customers") + .with_dimension(Dimension::categorical("region")), + ) + .unwrap(); + graph.set_metric_scopes(HashMap::new()).unwrap(); + let query = SemanticQuery::new() + .with_metrics(vec!["orders.share".into()]) + .with_dimensions(vec!["customers.region".into()]) + .with_filters(vec!["customers.region = 'EU'".into()]); + let sql = SqlGenerator::new(&graph).generate(&query).unwrap(); + assert!(!sql.to_ascii_lowercase().contains("__bsl_all"), "{sql}"); + assert!( + sql.contains("SELECT (COUNT(DISTINCT orders_cte.users_raw))"), + "{sql}" + ); + // Both aggregates read the same CTEs and join edge. The region filter + // may be pushed into the shared customers CTE, so it need not repeat. + assert_eq!(sql.matches("JOIN orders_cte").count(), 2, "{sql}"); + assert!(sql.contains("'EU'"), "{sql}"); + polyglot_sql::parse_one(&sql, DialectType::DuckDB).unwrap(); + } + + #[test] + fn lookml_post_sql_count_distinct_binds_base_measure_input() { + let mut graph = SemanticGraph::new(); + let mut percentage = Metric::derived( + "percentage", + "COUNT(DISTINCT {model}.users) / NULLIF(SUM(COUNT(DISTINCT {model}.users)) OVER (), 0)", + ); + percentage.meta = Some(serde_json::json!({"table_calculation":"percent_of_total"})); + graph + .add_model( + Model::new("visits", "id") + .with_table("visits") + .with_dimension(Dimension::categorical("country")) + .with_metric(Metric::count_distinct("users", "user_id")) + .with_metric(percentage), + ) + .unwrap(); + graph.set_metric_scopes(HashMap::new()).unwrap(); + let sql = SqlGenerator::new(&graph) + .generate( + &SemanticQuery::new() + .with_metrics(vec!["visits.percentage".into()]) + .with_dimensions(vec!["visits.country".into()]), + ) + .unwrap(); + assert!( + sql.contains("COUNT(DISTINCT (visits_cte.user_id))"), + "{sql}" + ); + assert!(!sql.contains("visits_cte.users"), "{sql}"); + assert!(!sql.contains("users AS users"), "{sql}"); + polyglot_sql::parse_one(&sql, DialectType::DuckDB).unwrap(); + } + + #[test] + fn lookml_post_sql_base_filters_preserve_conjunction_grouping() { + let mut graph = SemanticGraph::new(); + let mut users = Metric::count_distinct("users", "user_id"); + users.filters = vec![ + "country = 'US' OR country = 'CA'".into(), + "active = 1".into(), + ]; + graph + .add_model( + Model::new("visits", "id") + .with_table("visits") + .with_metric(users), + ) + .unwrap(); + let mut percentage = Metric::derived("percentage", "COUNT(DISTINCT {model}.users)"); + percentage.meta = Some(serde_json::json!({"table_calculation":"percent_of_total"})); + let expression = SqlGenerator::new(&graph) + .imported_calculation_expression(&percentage, "visits") + .unwrap(); + assert!( + expression.contains( + "(visits.country = 'US' OR visits.country = 'CA') AND (visits.active = 1)" + ), + "{expression}" + ); + // The same unmarked expression remains a physical aggregate input. + percentage.meta = None; + assert_eq!( + SqlGenerator::new(&graph) + .imported_calculation_expression(&percentage, "visits") + .unwrap(), + "COUNT(DISTINCT {model}.users)" + ); + } +} diff --git a/sidemantic-rs/src/sql/generator/options.rs b/sidemantic-rs/src/sql/generator/options.rs index 874fa4f20..4ce4f33fc 100644 --- a/sidemantic-rs/src/sql/generator/options.rs +++ b/sidemantic-rs/src/sql/generator/options.rs @@ -152,8 +152,18 @@ impl SqlGenerator<'_> { let mut projections = Vec::new(); let mut replacements = HashMap::new(); let mut renamed = false; - let quote = - |name: &str| self.emit_expression(&Expression::Identifier(Identifier::quoted(name))); + // polyglot 0.1.15 incorrectly splits ASC/DESC suffixes even inside + // quoted identifiers. Keep output names literal at this boundary. + let quote = |name: &str| { + let dialect = polyglot_sql::Dialect::get(self.dialect); + let style = &dialect.generator_config().identifier_quote_style; + format!( + "{}{}{}", + style.start, + name.replace(style.end, &format!("{}{}", style.end, style.end)), + style.end + ) + }; for expression in &select.expressions { let name = match expression { Expression::Alias(alias) => &alias.alias.name, @@ -168,10 +178,10 @@ impl SqlGenerator<'_> { renamed |= alias != name; projections.push(format!( "__sidemantic_result.{} AS {}", - quote(name)?, - quote(alias)? + quote(name), + quote(alias) )); - replacements.insert((None, name.clone()), quote(alias)?); + replacements.insert((None, name.clone()), quote(alias)); let source = match expression { Expression::Alias(alias) => &alias.this, other => other, @@ -182,32 +192,40 @@ impl SqlGenerator<'_> { column.table.as_ref().map(|table| table.name.clone()), column.name.name.clone(), ), - quote(alias)?, + quote(alias), ); } } if !renamed { return Ok(sql); } + let (inner_sql, used_preaggregation) = sql + .strip_suffix("\n-- used_preagg=true") + .map_or((sql.as_str(), false), |sql| (sql, true)); let wrapper = format!( - "SELECT {} FROM ({sql}) AS __sidemantic_result", + "SELECT {} FROM ({inner_sql}\n) AS __sidemantic_result", projections.join(", ") ); #[cfg(target_arch = "wasm32")] crate::wasm_sql_guard::check(&wrapper, self.dialect)?; - let Expression::Select(mut outer) = - crate::semantic_input::dialects::parse(&wrapper, self.dialect) - .map_err(|error| SidemanticError::SqlGeneration(error.to_string()))? - else { - unreachable!() - }; - outer.order_by = select.order_by; - if let Some(order) = &mut outer.order_by { + let mut order_by = select.order_by; + if let Some(order) = &mut order_by { for item in &mut order.expressions { item.this = replace_semantic_columns(item.this.clone(), &replacements)?; } } - self.emit_expression(&Expression::Select(outer)) + let mut result = if let Some(order) = order_by { + let order = self.emit_expression(&Expression::OrderBy(Box::new(order)))?; + format!("{wrapper} {order}") + } else { + wrapper + }; + // Routing metadata belongs to the final statement, not a nested SQL + // comment that the next parser can discard or attach to another node. + if used_preaggregation { + result.push_str("\n-- used_preagg=true"); + } + Ok(result) } } @@ -231,6 +249,21 @@ mod tests { graph } + #[test] + fn alias_wrapper_preserves_trailing_rollup_marker() { + let graph = graph(); + let generator = SqlGenerator::new(&graph); + let sql = generator + .alias_result( + "SELECT 42 AS revenue\n-- used_preagg=true".into(), + &HashMap::from([("revenue".into(), "total".into())]), + ) + .unwrap(); + assert!(sql.ends_with("\n-- used_preagg=true"), "{sql}"); + crate::semantic_input::dialects::parse(&sql, DialectType::DuckDB).unwrap(); + assert!(sql.contains("AS \"total\""), "{sql}"); + } + #[test] fn aliases_and_totals_match_result_schema() { let graph = graph(); @@ -259,6 +292,26 @@ mod tests { ); } + #[test] + fn semantic_boundary_binds_spaced_order_aliases_before_policy_parsing() { + let source = json!({"version": 1, "input_dialect": "duckdb", "models": [{"name": "orders", "table": "orders", "primary_key": "id", "dimensions": [{"name": "category", "type": "categorical"}], "metrics": [{"name": "revenue", "agg": "sum", "sql": "amount"}]}]}).to_string(); + for alias in ["Category label", "Category DESC", "Category NULLS FIRST"] { + for suffix in ["", " DESC", " ASC NULLS FIRST", "\tDESC\tNULLS\tLAST"] { + for dialect in ["duckdb", "postgres"] { + let query = json!({ + "metrics": ["orders.revenue"], "dimensions": ["orders.category"], + "aliases": {"orders.category": alias}, + "order_by": [format!("{alias}{suffix}")], "limit": 2, + "query_dialect": dialect, "dialect": dialect + }); + let sql = compile_with_semantic_input(&source, &query.to_string()).unwrap(); + assert!(sql.contains(&format!("ORDER BY \"{alias}\"")), "{sql}"); + assert!(sql.contains("LIMIT 2"), "{sql}"); + } + } + } + } + #[test] fn malformed_options_and_totals_controls_are_invalid_on_both_boundaries() { let source = json!({"version": 1, "input_dialect": "duckdb", "models": [{"name": "orders", "table": "orders", "primary_key": "id", "dimensions": [{"name": "category", "type": "categorical"}], "metrics": [{"name": "revenue", "agg": "sum", "sql": "amount"}]}]}).to_string(); diff --git a/sidemantic-rs/src/sql/generator/retention.rs b/sidemantic-rs/src/sql/generator/retention.rs index e14e0e728..23c7836a6 100644 --- a/sidemantic-rs/src/sql/generator/retention.rs +++ b/sidemantic-rs/src/sql/generator/retention.rs @@ -269,12 +269,7 @@ JOIN cohort_sizes c USING (cohort_date)"# ]; let mut ordering = Vec::new(); for item in &query.order_by { - let (field, direction) = item - .rsplit_once(' ') - .filter(|(_, suffix)| { - suffix.eq_ignore_ascii_case("asc") || suffix.eq_ignore_ascii_case("desc") - }) - .unwrap_or((item, "")); + let (field, direction) = crate::sql::split_order_field(item, &outputs); if !outputs.contains(&field) { return Err(unsupported("order_by")); } @@ -362,7 +357,6 @@ mod tests { for (field, value) in [ ("dimensions", json!(["events.person"])), ("filters", json!(["COUNT(*) > 1"])), - ("filters", json!(["ROW_NUMBER() OVER () > 1"])), ("filters", json!(["other.id = 1"])), ] { let mut query = query(); @@ -374,6 +368,22 @@ mod tests { } } + #[test] + fn retention_window_filter_is_rejected_at_query_boundary() { + let mut query = query(); + query["filters"] = json!(["ROW_NUMBER() OVER () > 1"]); + let error = + compile_with_semantic_input(&input().to_string(), &query.to_string()).unwrap_err(); + assert!( + matches!( + error, + SidemanticError::ValidationIssue { ref code, ref field, .. } + if code == "invalid_semantic_input" && field == "query.filters" + ), + "{error}" + ); + } + #[test] fn retention_requires_valid_period_and_granularity() { for (field, value) in [ diff --git a/sidemantic-rs/src/sql/generator/snapshots.rs b/sidemantic-rs/src/sql/generator/snapshots.rs index 87f7fef29..fea597b55 100644 --- a/sidemantic-rs/src/sql/generator/snapshots.rs +++ b/sidemantic-rs/src/sql/generator/snapshots.rs @@ -82,6 +82,9 @@ pub(super) fn try_generate( generator: &SqlGenerator<'_>, query: &SemanticQuery, ) -> Result> { + if query.allow_non_additive_unsafe { + return Ok(None); + } if !generator .graph .metrics() @@ -418,11 +421,17 @@ pub(super) fn try_generate( )); } let mut ordering = Vec::new(); + let names: Vec<_> = metrics + .iter() + .map(|metric| metric.alias.as_str()) + .chain( + output_dimensions + .iter() + .map(|dimension| dimension.alias.as_str()), + ) + .collect(); for item in &query.order_by { - let (field, direction) = item - .rsplit_once(' ') - .filter(|(_, dir)| dir.eq_ignore_ascii_case("asc") || dir.eq_ignore_ascii_case("desc")) - .unwrap_or((item, "")); + let (field, direction) = crate::sql::split_order_field(item, &names); let alias = query .metrics .iter() @@ -472,6 +481,27 @@ mod tests { graph } + #[test] + fn explicit_unsafe_option_bypasses_snapshot_selection_without_mutating_graph() { + crate::semantic_input::with_semantic_stack(|| { + let graph = graph(); + let mut query = SemanticQuery::new().with_metrics(vec![ + "snapshots.balance".into(), + "snapshots.activity".into(), + ]); + query.allow_non_additive_unsafe = true; + let generator = SqlGenerator::new(&graph); + assert!(try_generate(&generator, &query)?.is_none()); + let sql = generator.generate(&query)?; + assert!(sql.contains("SUM("), "{sql}"); + assert!(!sql.contains("MAX(day)"), "{sql}"); + query.allow_non_additive_unsafe = false; + assert!(try_generate(&generator, &query)?.is_some()); + Ok(()) + }) + .unwrap(); + } + #[test] fn simple_model_snapshot_is_not_bypassed_by_graph_metric_fast_path() { crate::semantic_input::with_semantic_stack(|| { diff --git a/sidemantic-rs/src/sql/generator/temporal.rs b/sidemantic-rs/src/sql/generator/temporal.rs index 2fe39265f..0e7fe13eb 100644 --- a/sidemantic-rs/src/sql/generator/temporal.rs +++ b/sidemantic-rs/src/sql/generator/temporal.rs @@ -29,7 +29,11 @@ pub(crate) fn validate_metric(metric: &Metric) -> Result<()> { } parse_output_frame(frame)?; } - if let Some(window) = &metric.window { + if let Some(window) = metric + .window + .as_deref() + .filter(|window| *window != "unbounded") + { period_interval(window)?; } } @@ -164,6 +168,10 @@ fn period_interval(value: &str) -> Result<(u32, String)> { } impl SqlGenerator<'_> { + pub(super) fn window_dimension_alias(dimension: &crate::core::Dimension) -> String { + format!("__sidemantic_window_{}", dimension.name) + } + pub(super) fn offset_window_lag_rows( offset: Option<&str>, granularity: Option<&str>, @@ -293,7 +301,11 @@ impl SqlGenerator<'_> { let frame = if let Some(frame) = &metric.window_frame { frame.clone() } else if metric.grain_to_date.is_none() { - if let Some(window) = &metric.window { + if let Some(window) = metric + .window + .as_deref() + .filter(|window| *window != "unbounded") + { let (amount, unit) = period_interval(window)?; format!("RANGE BETWEEN INTERVAL '{amount} {unit}' PRECEDING AND CURRENT ROW") } else { @@ -359,6 +371,57 @@ impl SqlGenerator<'_> { mod tests { use super::*; + #[test] + fn window_dimension_uses_separate_source_alias_in_grouping_and_filter() { + let input = serde_json::json!({ + "version": 1, "input_dialect": "duckdb", + "models": [{"name": "events", "table": "events", "primary_key": "id", + "dimensions": [{"name": "day", "type": "time", "granularity": "day", "window": "MIN(day) OVER ()"}], + "metrics": [{"name": "revenue", "agg": "sum", "sql": "amount"}]}] + }); + let sql = crate::semantic_input::compile_with_semantic_input( + &input.to_string(), + &serde_json::json!({ + "metrics": ["events.revenue"], "dimensions": ["events.day"], + "filters": ["COALESCE(events.day, '2024-01-01') > '2024-01-02'"] + }) + .to_string(), + ) + .unwrap(); + assert!(sql.contains("AS __sidemantic_window_day"), "{sql}"); + assert!( + sql.contains("DATE_TRUNC('day', events_cte.__sidemantic_window_day)"), + "{sql}" + ); + assert!( + sql.contains("COALESCE(events_cte.__sidemantic_window_day"), + "{sql}" + ); + polyglot_sql::parse_one(&sql, DialectType::DuckDB).unwrap(); + } + + #[test] + fn graph_calculation_keeps_graph_reference_in_cumulative_base_query() { + let input = serde_json::json!({ + "version": 1, "input_dialect": "duckdb", + "models": [{"name": "events", "table": "events", "primary_key": "id", + "dimensions": [{"name": "day", "type": "time", "granularity": "day"}], + "metrics": [{"name": "revenue", "agg": "sum", "sql": "amount"}]}], + "metrics": [ + {"name": "total", "type": "derived", "sql": "events.revenue"}, + {"name": "running", "type": "cumulative", "sql": "events.revenue"} + ] + }); + let sql = crate::semantic_input::compile_with_semantic_input( + &input.to_string(), + r#"{"metrics":["total","running"],"dimensions":["events.day"]}"#, + ) + .unwrap(); + assert!(sql.contains("base.total"), "{sql}"); + assert!(sql.contains("SUM(base.revenue) OVER"), "{sql}"); + polyglot_sql::parse_one(&sql, DialectType::DuckDB).unwrap(); + } + #[test] fn explicit_window_requires_a_root_window_capable_function() { for sql in [ @@ -510,6 +573,27 @@ mod tests { (graph, dimensions) } + #[test] + fn explicit_unbounded_window_preserves_running_total_frame() { + let (graph, dimensions) = grouped_graph(); + let generator = SqlGenerator::new(&graph); + let mut metric = Metric::sum("running", "revenue"); + metric.r#type = MetricType::Cumulative; + metric.window = Some("unbounded".into()); + validate_metric(&metric).unwrap(); + let explicit = generator + .cumulative_window_sql(&metric, &dimensions, "day__month") + .unwrap(); + metric.window = None; + let implicit = generator + .cumulative_window_sql(&metric, &dimensions, "day__month") + .unwrap(); + assert_eq!(explicit, implicit); + assert!(explicit.contains("ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW")); + metric.window = Some("invalid".into()); + assert!(validate_metric(&metric).is_err()); + } + #[test] fn calendar_comparison_uses_exact_period_and_group_partition() { let (graph, dimensions) = grouped_graph(); diff --git a/sidemantic-rs/src/sql/rewriter.rs b/sidemantic-rs/src/sql/rewriter.rs index 40e73a4ab..51bc9296d 100644 --- a/sidemantic-rs/src/sql/rewriter.rs +++ b/sidemantic-rs/src/sql/rewriter.rs @@ -2,7 +2,7 @@ use std::collections::{HashMap, HashSet}; -use polyglot_sql::parse as polyglot_parse; +use crate::semantic_input::dialects::parse_many as polyglot_parse; use polyglot_sql::{ expressions::{Identifier, Join, JoinKind, Select, TableRef, With}, generate as polyglot_generate, DialectType, Expression, @@ -26,6 +26,7 @@ pub struct QueryRewriter<'a> { rename_only: bool, security_controls: bool, warnings: std::cell::RefCell>, + used_preaggregation: std::cell::Cell, } impl<'a> QueryRewriter<'a> { @@ -37,6 +38,7 @@ impl<'a> QueryRewriter<'a> { rename_only: false, security_controls: false, warnings: std::cell::RefCell::new(Vec::new()), + used_preaggregation: std::cell::Cell::new(false), } } @@ -73,6 +75,7 @@ impl<'a> QueryRewriter<'a> { input_dialect: DialectType, output_dialect: DialectType, ) -> Result { + self.used_preaggregation.set(false); if let Some(rewritten) = self.rewrite_yardstick(sql, input_dialect, output_dialect)? { return Ok(rewritten); } @@ -97,7 +100,14 @@ impl<'a> QueryRewriter<'a> { )?); } - Ok(rewritten_statements.join(";\n")) + let mut sql = rewritten_statements.join(";\n"); + // Parsing semantic leaves into relational wrappers drops their trailing + // comments. Preserve routing independently so missing-rollup fallback + // and strict mode observe the compiler's actual selection. + if self.used_preaggregation.get() { + sql.push_str("\n-- used_preagg=true"); + } + Ok(sql) } fn rewrite_statement(&self, statement: Expression) -> Result { @@ -114,7 +124,7 @@ fn parse_sql_with_dialect(sql: &str, dialect: DialectType) -> Result Result, next_metric: usize, strict_fields: bool, + secured_subquery: bool, } impl Bindings { + fn unknown_field(&self, reference: &str) -> SidemanticError { + if self.secured_subquery { + SidemanticError::Security(format!( + "Cannot authorize semantic subquery: field '{reference}' is not declared" + )) + } else { + SidemanticError::Validation(format!("Field '{reference}' not found")) + } + } + fn resolve(&self, column: &Column) -> Result { let name = &column.name.name; if let Some(table) = &column.table { @@ -161,9 +172,7 @@ impl Bindings { } else if model.get_dimension(split_granularity(field).0).is_some() { false } else { - return Err(SidemanticError::Validation(format!( - "Field '{reference}' not found" - ))); + return Err(self.unknown_field(&reference)); } } else if self.graph.get_metric(&reference).is_some() { true @@ -282,9 +291,7 @@ impl Bindings { if model.get_metric(field).is_none() && model.get_dimension(split_granularity(field).0).is_none() { - return Err(SidemanticError::Validation(format!( - "Field '{reference}' not found" - ))); + return Err(self.unknown_field(&reference)); } } Ok(Some(Expression::qualified_column(owner, field))) @@ -409,7 +416,11 @@ impl Bindings { } impl QueryRewriter<'_> { - pub(super) fn compile_semantic_select(&self, mut select: Select) -> Result { let mut remainder = select.clone(); remainder.expressions.clear(); remainder.from = None; @@ -453,6 +464,7 @@ impl QueryRewriter<'_> { aggregate_inputs: Vec::new(), next_metric: 0, strict_fields: self.security_controls, + secured_subquery: self.security_controls && nested, }; let mut projections = Vec::new(); let mut aliases = HashSet::new(); @@ -548,8 +560,16 @@ impl QueryRewriter<'_> { bindings.query.order_by.clear(); } let sql = SqlGenerator::new(&bindings.graph).generate(&bindings.query)?; + // Preserve routing metadata outside the parsed subquery: trailing SQL + // comments are discarded by the parser and can consume its closing ')'. + let sql = if let Some(inner) = sql.strip_suffix("\n-- used_preagg=true") { + self.used_preaggregation.set(true); + inner + } else { + &sql + }; let mut wrapper = parse_sql_with_dialect( - &format!("SELECT * FROM ({sql}) AS __semantic_query"), + &format!("SELECT * FROM ({sql}\n) AS __semantic_query"), DialectType::DuckDB, )?; let Expression::Select(mut outer) = wrapper.remove(0) else { diff --git a/sidemantic-rs/src/sql/rewriter/policy.rs b/sidemantic-rs/src/sql/rewriter/policy.rs index ace759ca3..62eeac2f0 100644 --- a/sidemantic-rs/src/sql/rewriter/policy.rs +++ b/sidemantic-rs/src/sql/rewriter/policy.rs @@ -124,13 +124,14 @@ impl QueryRewriter<'_> { rename_only: true, security_controls: false, warnings: std::cell::RefCell::new(Vec::new()), + used_preaggregation: std::cell::Cell::new(false), }; rewriter.rewrite_policy_statement(statement) } pub(super) fn rewrite_policy_statement(&self, statement: Expression) -> Result { let mut names = CteNames::new(&statement, self.graph, self.policy_definitions)?; - self.rewrite_policy_query(statement, &HashMap::new(), &mut names) + self.rewrite_policy_query(statement, &HashMap::new(), &mut names, false) } fn rewrite_policy_query( @@ -138,34 +139,36 @@ impl QueryRewriter<'_> { statement: Expression, inherited_ctes: &HashMap, names: &mut CteNames, + nested: bool, ) -> Result { // Set operations own their WITH/ORDER/LIMIT clauses. Rewrite their // operands without moving those clauses onto an individual SELECT. match statement { Expression::Union(mut set) => { let ctes = self.rewrite_ctes(&mut set.with, inherited_ctes, names)?; - set.left = self.rewrite_policy_query(set.left.clone(), &ctes, names)?; - set.right = self.rewrite_policy_query(set.right.clone(), &ctes, names)?; + set.left = self.rewrite_policy_query(set.left.clone(), &ctes, names, nested)?; + set.right = self.rewrite_policy_query(set.right.clone(), &ctes, names, nested)?; return self.rewrite_set_clauses(Expression::Union(set), &ctes, names); } Expression::Intersect(mut set) => { let ctes = self.rewrite_ctes(&mut set.with, inherited_ctes, names)?; - set.left = self.rewrite_policy_query(set.left.clone(), &ctes, names)?; - set.right = self.rewrite_policy_query(set.right.clone(), &ctes, names)?; + set.left = self.rewrite_policy_query(set.left.clone(), &ctes, names, nested)?; + set.right = self.rewrite_policy_query(set.right.clone(), &ctes, names, nested)?; return self.rewrite_set_clauses(Expression::Intersect(set), &ctes, names); } Expression::Except(mut set) => { let ctes = self.rewrite_ctes(&mut set.with, inherited_ctes, names)?; - set.left = self.rewrite_policy_query(set.left.clone(), &ctes, names)?; - set.right = self.rewrite_policy_query(set.right.clone(), &ctes, names)?; + set.left = self.rewrite_policy_query(set.left.clone(), &ctes, names, nested)?; + set.right = self.rewrite_policy_query(set.right.clone(), &ctes, names, nested)?; return self.rewrite_set_clauses(Expression::Except(set), &ctes, names); } Expression::Subquery(mut query) => { - query.this = self.rewrite_policy_query(query.this, inherited_ctes, names)?; + query.this = self.rewrite_policy_query(query.this, inherited_ctes, names, true)?; return Ok(Expression::Subquery(query)); } Expression::Paren(mut paren) => { - paren.this = self.rewrite_policy_query(paren.this, inherited_ctes, names)?; + paren.this = + self.rewrite_policy_query(paren.this, inherited_ctes, names, nested)?; return Ok(Expression::Paren(paren)); } Expression::Select(_) => {} @@ -197,8 +200,8 @@ impl QueryRewriter<'_> { && select.from.as_ref().is_some_and(|from| { from.expressions.first().is_some_and(|source| { matches!(source, Expression::Table(table) - if table.schema.is_none() && table.catalog.is_none() - && !ctes.contains_key(&table.name.name.to_ascii_lowercase()) + if (table.schema.is_some() || table.catalog.is_some() + || !ctes.contains_key(&table.name.name.to_ascii_lowercase())) && (table.name.name.eq_ignore_ascii_case("metrics") || self.graph.get_model(&table.name.name).is_some())) }) @@ -217,13 +220,22 @@ impl QueryRewriter<'_> { if semantic_leaf { if let Some(from) = &select.from { if let Some(Expression::Table(table)) = from.expressions.first() { + // `metrics` is a virtual relation, not a physical table in + // an arbitrary catalog/schema. Qualified model names still + // resolve through the policy preparer below. + if self.query_preparer.is_some() + && table.name.name.eq_ignore_ascii_case("metrics") + && (table.schema.is_some() || table.catalog.is_some()) + { + return Err(unsupported()); + } validate_table(table)?; if !table.column_aliases.is_empty() { return Err(unsupported()); } } } - let mut compiled = self.compile_semantic_select(*select)?; + let mut compiled = self.compile_semantic_select(*select, nested)?; // Keep the existing semantic-root error contract even though the // renamed input CTE would no longer capture the generated source. // Only names actually emitted for this query are conflicts. @@ -271,7 +283,7 @@ impl QueryRewriter<'_> { if with.recursive { ctes.insert(original.clone(), alias.clone()); } - cte.this = self.rewrite_policy_query(cte.this.clone(), &ctes, names)?; + cte.this = self.rewrite_policy_query(cte.this.clone(), &ctes, names, true)?; cte.alias = alias.clone(); ctes.insert(original, alias); } @@ -295,7 +307,7 @@ impl QueryRewriter<'_> { | Expression::Except(_) ) { return self - .rewrite_policy_query(node.clone(), ctes, names) + .rewrite_policy_query(node.clone(), ctes, names, true) .map(Some); } if self.query_preparer.is_some() @@ -389,7 +401,7 @@ impl QueryRewriter<'_> { Ok(Expression::Table(table)) } Expression::Subquery(mut subquery) => { - subquery.this = self.rewrite_policy_query(subquery.this, ctes, names)?; + subquery.this = self.rewrite_policy_query(subquery.this, ctes, names, true)?; Ok(Expression::Subquery(subquery)) } Expression::Alias(mut alias) => { @@ -424,6 +436,11 @@ impl QueryRewriter<'_> { fn validate_table(table: &TableRef) -> Result<()> { let mut remainder = table.clone(); remainder.name = Identifier::new(""); + // Semantic model lookup uses the terminal relation name, including the + // schemas advertised by the PostgreSQL server. Qualifiers do not bypass + // semantic compilation or its policy preparer. + remainder.schema = None; + remainder.catalog = None; remainder.alias = None; remainder.alias_explicit_as = false; remainder.column_aliases.clear(); @@ -440,6 +457,102 @@ mod tests { use super::*; use crate::core::{Dimension, Metric, Model}; + #[test] + fn undeclared_nested_fields_are_security_errors_only_with_controls() { + let mut graph = SemanticGraph::new(); + graph + .add_model( + Model::new("orders", "id") + .with_table("physical_orders") + .with_metric(Metric::sum("revenue", "amount")), + ) + .unwrap(); + let prepare = |_: &SemanticGraph, _: &mut SemanticQuery| Ok(()); + for controls in [false, true] { + let rewriter = QueryRewriter::new(&graph).with_query_preparer(&prepare, "", controls); + let nested = rewriter + .rewrite("SELECT * FROM (SELECT amount FROM orders) AS scoped") + .unwrap_err(); + if controls { + assert!(matches!(nested, SidemanticError::Security(ref message) + if message.contains("semantic subquery"))); + } else { + assert!(matches!(nested, SidemanticError::Validation(_))); + } + let root = rewriter.rewrite("SELECT amount FROM orders").unwrap_err(); + assert!(matches!(root, SidemanticError::Validation(_))); + } + } + + #[test] + fn qualified_semantic_sources_still_prepare_policy() { + let mut graph = SemanticGraph::new(); + graph + .add_model( + Model::new("orders", "id") + .with_table("private_orders") + .with_dimension(Dimension::new("tenant")) + .with_metric(Metric::sum("revenue", "amount")), + ) + .unwrap(); + let prepared = std::cell::Cell::new(0); + let prepare = |_: &SemanticGraph, query: &mut SemanticQuery| { + prepared.set(prepared.get() + 1); + assert_eq!(query.metrics, vec!["orders.revenue"]); + query.filters.push("orders.tenant = 1".into()); + Ok(()) + }; + let rewriter = QueryRewriter::new(&graph).with_query_preparer(&prepare, "", true); + for source in [ + "main.orders", + "\"main\".\"orders\"", + "semantic_layer.orders", + ] { + // A same-named CTE only shadows unqualified relations. + let sql = + format!("WITH orders AS (SELECT 0 AS revenue) SELECT o.revenue FROM {source} AS o"); + let rewritten = rewriter.rewrite(&sql).unwrap(); + assert!(rewritten.contains("private_orders"), "{rewritten}"); + assert!(rewritten.contains("tenant = 1"), "{rewritten}"); + assert!(!rewritten.contains(source), "{rewritten}"); + } + assert_eq!(prepared.get(), 3); + for source in [ + "main.metrics", + "\"main\".\"metrics\"", + "catalog.main.metrics", + ] { + for sql in [ + format!("SELECT orders.revenue FROM {source}"), + format!("SELECT * FROM (SELECT orders.revenue FROM {source}) AS q"), + ] { + assert!( + matches!( + rewriter.rewrite(&sql), + Err(SidemanticError::UnsupportedSemanticFeatures { .. }) + ), + "{sql}" + ); + } + } + assert_eq!(prepared.get(), 3); + + let deny = |_: &SemanticGraph, _: &mut SemanticQuery| { + Err(SidemanticError::Validation("access denied".into())) + }; + let rewriter = QueryRewriter::new(&graph).with_query_preparer(&deny, "", true); + assert!(rewriter + .rewrite("SELECT revenue FROM main.orders") + .unwrap_err() + .to_string() + .contains("access denied")); + assert!(rewriter + .rewrite("SELECT * FROM main.private_orders") + .unwrap_err() + .to_string() + .contains("rewrite.policy_select_shape")); + } + #[test] fn generated_cte_conflicts_preserve_semantic_root_contract() { let mut graph = SemanticGraph::new(); diff --git a/sidemantic-rs/src/sql/rewriter/yardstick.rs b/sidemantic-rs/src/sql/rewriter/yardstick.rs index 7f720a0fb..3c287b0b4 100644 --- a/sidemantic-rs/src/sql/rewriter/yardstick.rs +++ b/sidemantic-rs/src/sql/rewriter/yardstick.rs @@ -208,7 +208,8 @@ impl Lowerer<'_, '_> { .authored_expression(candidate) .ok() .filter(|expr| reference(expr).is_some()); - visible = true; + // Curly references have the plain measure evaluation context; + // only AGGREGATE(...) inherits the visible row predicate. } else { while stream .get(end + 1) @@ -290,6 +291,9 @@ impl Lowerer<'_, '_> { map_columns(&mut value, &mut |node| { if let Expression::Column(mut column) = node { column.table = None; + // Quoting is syntax, not a different context dimension. The + // declared dimension expansion may have removed source quotes. + column.name.quoted = false; return Ok(Expression::Column(column)); } Ok(node) @@ -894,14 +898,23 @@ impl Lowerer<'_, '_> { let inner = self.remap(group.clone(), &aliases, "_inner", single)?; let mut outer = self.remap(group, &[model_name], alias, single)?; if let Some(output_alias) = context_aliases.get(&signature) { - let unsafe_alias = model.dimensions.iter().any(|dimension| { - dimension.name.eq_ignore_ascii_case(output_alias) - && self - .expression(dimension.sql_expr()) - .ok() - .and_then(|expression| reference(&expression)) - .is_some_and(|(_, name)| name.eq_ignore_ascii_case(output_alias)) - }); + // SELECT * imports can expose physical grouping columns + // without declaring dimensions. An unqualified output + // alias with that same name would bind to the inner source + // and turn correlation into a tautology. + let unsafe_alias = columns + .iter() + .any(|(_, name)| name.eq_ignore_ascii_case(output_alias)) + || model.dimensions.iter().any(|dimension| { + dimension.name.eq_ignore_ascii_case(output_alias) + && self + .expression(dimension.sql_expr()) + .ok() + .and_then(|expression| reference(&expression)) + .is_some_and(|(_, name)| { + name.eq_ignore_ascii_case(output_alias) + }) + }); if !unsafe_alias { outer = Expression::Identifier(Identifier::new(output_alias)); } @@ -1376,6 +1389,19 @@ fn has_aggregate_semantics(value: &Value) -> bool { } fn expand_groups(expression: &Expression, output: &mut Vec) -> Result<()> { + // The pinned parser represents inline ROLLUP/CUBE as ordinary functions, + // but WITH ROLLUP/CUBE as the typed variants handled below. + if let Expression::Function(function) = expression { + if ["ROLLUP", "CUBE", "GROUPING SETS"] + .iter() + .any(|name| function.name.eq_ignore_ascii_case(name)) + { + for child in &function.args { + expand_groups(child, output)?; + } + return Ok(()); + } + } let value = encode(expression)?; let Some(fields) = value.as_object() else { return Ok(()); @@ -1474,3 +1500,32 @@ impl QueryRewriter<'_> { dialects::emit(decode(value)?, DialectType::DuckDB, output).map(Some) } } + +#[cfg(test)] +mod tests { + use super::*; + use crate::core::{Metric, Model, SemanticGraph}; + + #[test] + fn wildcard_import_grouping_retains_outer_column_qualification() { + let mut model = Model::new("orders_v", "id") + .with_table("raw_orders") + .with_metric(Metric::sum("revenue", "amount")); + model.metadata = Some(serde_json::json!({"yardstick": {}})); + let mut graph = SemanticGraph::new(); + graph.add_model(model).unwrap(); + for grouping in [ + "o.product", + "ROLLUP(o.product)", + "CUBE(o.product)", + "GROUPING SETS ((o.product), ())", + ] { + let sql = QueryRewriter::new(&graph).rewrite(&format!("SELECT o.product, AGGREGATE(o.revenue) AS total FROM orders_v AS o GROUP BY {grouping}")).unwrap(); + assert!( + sql.contains("(_inner.product) IS NOT DISTINCT FROM (o.product)"), + "{sql}" + ); + assert!(!sql.contains("IS NOT DISTINCT FROM (product)"), "{sql}"); + } + } +} diff --git a/sidemantic-rs/tests/many_role_contracts.rs b/sidemantic-rs/tests/many_role_contracts.rs new file mode 100644 index 000000000..2c1f55766 --- /dev/null +++ b/sidemantic-rs/tests/many_role_contracts.rs @@ -0,0 +1,65 @@ +use serde_json::json; +use sidemantic::semantic_input::compile_with_semantic_input; + +fn source() -> serde_json::Value { + json!({ + "version": 1, + "input_dialect": "duckdb", + "models": [ + { + "name": "orders", "table": "orders", "primary_key": "id", + "dimensions": [{"name": "id", "type": "numeric"}], + "metrics": [{"name": "revenue", "agg": "sum", "sql": "amount"}], + "relationships": [ + {"name": "primary_tags", "target_model": "tags", "type": "many_to_many", + "through": "links", "through_foreign_key": "order_id", + "related_foreign_key": "primary_tag", "edge_id": "primary_tags"}, + {"name": "secondary_tags", "target_model": "tags", "type": "many_to_many", + "through": "links", "through_foreign_key": "order_id", + "related_foreign_key": "secondary_tag", "edge_id": "secondary_tags"} + ] + }, + { + "name": "tags", "table": "tags", "primary_key": "id", + "dimensions": [{"name": "name", "type": "categorical"}], + "metrics": [{"name": "sum_ids", "agg": "sum", "sql": "id"}] + }, + {"name": "links", "table": "links", "primary_key": "id"} + ] + }) +} + +#[test] +fn target_metric_inputs_keep_the_requested_role() { + for role in ["primary_tags", "secondary_tags"] { + let query = json!({"metrics": [format!("{role}.sum_ids")], "dimensions": ["orders.id"]}); + let sql = compile_with_semantic_input(&source().to_string(), &query.to_string()).unwrap(); + assert!(sql.contains(&format!("{role}_cte")), "{sql}"); + assert!(sql.contains(&format!("{role}$through_cte")), "{sql}"); + assert!(sql.contains("SELECT DISTINCT"), "{sql}"); + } +} + +#[test] +fn junction_policies_filter_links_without_changing_source_preservation() { + let mut source = source(); + source["models"][2]["security"] = json!({"row_filters": ["tenant = {{ user.tenant }}"]}); + source["models"][2]["invariant_filters"] = json!(["enabled"]); + let query = json!({ + "metrics": ["orders.revenue"], + "dimensions": ["primary_tags.name", "secondary_tags.name"], + "user_attributes": {"tenant": "a"} + }); + let sql = compile_with_semantic_input(&source.to_string(), &query.to_string()).unwrap(); + for role in ["primary_tags", "secondary_tags"] { + assert!( + sql.contains(&format!("LEFT JOIN {role}$through_cte")), + "{sql}" + ); + assert!( + !sql.contains(&format!("INNER JOIN {role}$through_cte")), + "{sql}" + ); + } + assert_eq!(sql.matches("tenant = 'a' AND enabled").count(), 2, "{sql}"); +} diff --git a/sidemantic-rs/tests/mcp_protocol.rs b/sidemantic-rs/tests/mcp_protocol.rs index af4e3b85f..d4fe9a148 100644 --- a/sidemantic-rs/tests/mcp_protocol.rs +++ b/sidemantic-rs/tests/mcp_protocol.rs @@ -434,9 +434,17 @@ fn mcp_server_exercises_tool_protocol_and_errors() { let resource_items = resources["result"]["resources"] .as_array() .expect("resources/list should return resources"); - assert!(resource_items + let catalog_resource = resource_items .iter() - .any(|resource| resource["uri"] == "semantic://catalog")); + .find(|resource| resource["uri"] == "semantic://catalog") + .expect("resources/list should include the semantic catalog"); + assert_eq!(catalog_resource["name"], "catalog"); + assert_eq!(catalog_resource["title"], "Sidemantic Catalog Metadata"); + assert_eq!(catalog_resource["mimeType"], "application/json"); + assert_eq!( + catalog_resource["description"], + "Postgres-compatible catalog metadata for the semantic layer." + ); let catalog = client.request(16, "resources/read", json!({ "uri": "semantic://catalog" })); let catalog_text = catalog["result"]["contents"][0]["text"] diff --git a/sidemantic-rs/tests/package_metadata.rs b/sidemantic-rs/tests/package_metadata.rs index 634970f43..229190270 100644 --- a/sidemantic-rs/tests/package_metadata.rs +++ b/sidemantic-rs/tests/package_metadata.rs @@ -7,12 +7,14 @@ fn toml_string_value(contents: &str, key: &str) -> Option { } #[test] -fn rust_crate_and_python_extension_versions_match() { +fn python_distributions_versions_match() { let pyproject = include_str!("../pyproject.toml"); let pyproject_version = toml_string_value(pyproject, "version").expect("pyproject.toml project.version"); - assert_eq!(env!("CARGO_PKG_VERSION"), pyproject_version); + let parent_version = toml_string_value(include_str!("../../pyproject.toml"), "version") + .expect("parent pyproject.toml project.version"); + assert_eq!(parent_version, pyproject_version); } #[test] @@ -21,7 +23,7 @@ fn python_extension_metadata_targets_the_expected_module_and_feature() { assert!(pyproject.contains("name = \"sidemantic-rs\"")); assert!(pyproject.contains("module-name = \"sidemantic_rs\"")); - assert!(pyproject.contains("features = [\"python-adbc\"]")); + assert!(pyproject.contains("features = [\"python\"]")); assert!(pyproject.contains("license = \"AGPL-3.0-only\"")); } diff --git a/sidemantic-rs/tests/python_wheel_python_smoke.py b/sidemantic-rs/tests/python_wheel_python_smoke.py index 817ca03f2..796153c59 100644 --- a/sidemantic-rs/tests/python_wheel_python_smoke.py +++ b/sidemantic-rs/tests/python_wheel_python_smoke.py @@ -4,6 +4,8 @@ import importlib.metadata import importlib.util +import tomllib +from pathlib import Path import sidemantic_rs @@ -11,7 +13,10 @@ if root_python_package is not None: raise AssertionError("isolated sidemantic_rs python-feature wheel unexpectedly found root sidemantic package") -if importlib.metadata.version("sidemantic-rs") != "0.1.0": +expected_version = tomllib.loads((Path(__file__).resolve().parents[1] / "pyproject.toml").read_text())["project"][ + "version" +] +if importlib.metadata.version("sidemantic-rs") != expected_version: raise AssertionError("unexpected sidemantic-rs wheel version") models_yaml = """ diff --git a/sidemantic-rs/tests/python_wheel_smoke.py b/sidemantic-rs/tests/python_wheel_smoke.py index 74eeeda4d..84e6e3625 100644 --- a/sidemantic-rs/tests/python_wheel_smoke.py +++ b/sidemantic-rs/tests/python_wheel_smoke.py @@ -5,6 +5,8 @@ import importlib.metadata import importlib.util import json +import tomllib +from pathlib import Path import sidemantic_rs @@ -26,7 +28,10 @@ def expect_raises(exc_type: type[BaseException], func, *args) -> str: if root_python_package is not None: raise AssertionError("isolated sidemantic_rs wheel smoke unexpectedly found root sidemantic package") -if importlib.metadata.version("sidemantic-rs") != "0.1.0": +expected_version = tomllib.loads((Path(__file__).resolve().parents[1] / "pyproject.toml").read_text())["project"][ + "version" +] +if importlib.metadata.version("sidemantic-rs") != expected_version: raise AssertionError("unexpected sidemantic-rs wheel version") models_yaml = """ diff --git a/sidemantic-rs/tests/test_wasm_semantic_host.py b/sidemantic-rs/tests/test_wasm_semantic_host.py index fd99d248e..70ae18bf0 100644 --- a/sidemantic-rs/tests/test_wasm_semantic_host.py +++ b/sidemantic-rs/tests/test_wasm_semantic_host.py @@ -210,11 +210,13 @@ def test_explore_anchor_scopes_related_fields_and_policies(): response = call("compile", source, dimensions) assert "error" not in response, response assert set(connection.execute(response["result"]).fetchall()) == {("paid",), (None,)} + independent = call("compile", source, {**query, "metrics": ["items.value", "orders.revenue"]}) + assert "error" not in independent, independent + # Both aggregates retain the Explore's policy-filtered population; + # multiple items must not multiply revenue, and orphans stay excluded. + assert connection.execute(independent["result"]).fetchall() == [(12, 30)] assert json.loads(call("validate", source, query)["result"]) == [] assert "no user_attributes" in call("compile", source, {**query, "user_attributes": None}).get("error", "") - assert "consumption_base_model.independent_aggregates" in call( - "compile", source, {**query, "metrics": ["items.value", "orders.revenue"]} - ).get("error", "") invalid = call("validate", source, {**query, "consumption_base_model": "missing"}) assert "missing" in invalid.get("error", "") diff --git a/sidemantic/cli.py b/sidemantic/cli.py index 6147cb572..bef5fd5d1 100644 --- a/sidemantic/cli.py +++ b/sidemantic/cli.py @@ -184,11 +184,14 @@ def _resolve_engine_options(engine: str | None, fallback: bool | None) -> tuple[ if resolved_fallback is None: resolved_fallback = _loaded_config.runtime.fallback + if resolved_engine is None: + from sidemantic.runtime import default_engine + + resolved_engine = default_engine() + if resolved_fallback is None: resolved_fallback = resolved_engine == "auto" - if resolved_engine is None and fallback is not None: - raise typer.BadParameter("--fallback/--no-fallback requires --engine or runtime.engine in config") if resolved_engine == "python" and resolved_fallback: raise typer.BadParameter("--fallback is only meaningful with the rust or auto engine") diff --git a/sidemantic/config.py b/sidemantic/config.py index e6130a222..126cfa2fa 100644 --- a/sidemantic/config.py +++ b/sidemantic/config.py @@ -6,6 +6,7 @@ from pydantic import BaseModel, ConfigDict, Field +from sidemantic.runtime import default_engine from sidemantic.yaml_compat import safe_load as _yaml_safe_load @@ -155,7 +156,7 @@ class APIServerConfig(BaseModel): class RuntimeConfig(BaseModel): """Runtime engine selection.""" - engine: Literal["python", "rust", "auto"] = Field(default="python", description="Runtime engine") + engine: Literal["python", "rust", "auto"] = Field(default_factory=default_engine, description="Runtime engine") fallback: bool = Field(default=False, description="Allow Rust runtime fallback to Python") diff --git a/sidemantic/core/semantic_layer.py b/sidemantic/core/semantic_layer.py index 01a4039b4..28a3b05e8 100644 --- a/sidemantic/core/semantic_layer.py +++ b/sidemantic/core/semantic_layer.py @@ -18,6 +18,7 @@ from sidemantic.core.metric import Metric from sidemantic.core.model import Model from sidemantic.core.semantic_graph import SemanticGraph +from sidemantic.runtime import default_engine from sidemantic.rust_bridge import get_rust_module from sidemantic.rust_parity import is_strict_for from sidemantic.semantic_handoff import RustBackendUnavailableError, UnsupportedSemanticFeaturesError @@ -114,7 +115,8 @@ def __init__( loading extensions, attaching catalogs, creating secrets) engine: Runtime engine for native query validation/compilation. Supported values are "python", "rust", and "auto". If omitted, - legacy SIDEMANTIC_RS_* environment flags are honored. + defaults to Rust (Python in Pyodide). SIDEMANTIC_ENGINE overrides + the platform default; legacy SIDEMANTIC_RS_* flags remain supported. fallback: Whether an unavailable Rust backend or a known unsupported capability may fall back to Python. Invalid input and unexpected compiler failures propagate. Defaults to False for engine="rust" and True for engine="auto". @@ -136,6 +138,10 @@ def __init__( """ from sidemantic.db.base import BaseDatabaseAdapter + # Preserve explicitly configured legacy parity runs. Ordinary installs + # select the same native default as the CLI and project configuration. + if engine is None and not any(key.startswith("SIDEMANTIC_RS_") for key in os.environ): + engine = default_engine() if engine is not None: engine = engine.lower() if engine not in {"python", "rust", "auto"}: @@ -1361,6 +1367,7 @@ def _validate_query( dimensions, input_dialect="duckdb" if self.dialect == "postgres" else self.dialect, rust_module=self._rust_module, + **({"allow_non_additive_unsafe": True} if self.allow_non_additive_unsafe else {}), ) except (RustBackendUnavailableError, UnsupportedSemanticFeaturesError) as exc: if self._strict_rust_query_validation or self._rust_no_fallback: @@ -1568,6 +1575,7 @@ def _compile_with_rust( "preagg_schema": self.preagg_schema, "user_attributes": user_attributes, "enforce_visibility": self.enforce_visibility, + **({"allow_non_additive_unsafe": True} if self.allow_non_additive_unsafe else {}), } try: @@ -1629,7 +1637,10 @@ def _apply_post_process(self, inner_sql: str, post_process: str | None) -> str: # subquery position. CTEs inside subqueries are valid SQL in # all target databases and naturally scoped, avoiding name # collisions with CTEs in the post_process SQL. - return post_process.replace("{inner}", stripped) + # Native compilation can leave a routing marker (or source SQL can + # end in a line comment). Terminate it before the wrapper's closing + # parenthesis, preserving the SQL and its comments verbatim. + return post_process.replace("{inner}", stripped + "\n") return inner_sql @@ -2223,6 +2234,7 @@ def sql(self, query: str, *, user_attributes: dict | None = None): self.dialect, self.use_preaggregations, self.enforce_visibility, + self.allow_non_additive_unsafe, self._explicit_engine, self._use_rust_sql_generator, self._rust_no_fallback, @@ -2284,6 +2296,7 @@ def explain_sql(self, query: str, strict: bool = True): enforce_visibility=self.enforce_visibility, use_rust_rewriter=self._use_rust_sql_generator if self._explicit_engine else None, rust_no_fallback=self._rust_no_fallback, + allow_non_additive_unsafe=self.allow_non_additive_unsafe, ) explanation = rewriter.explain(query, strict=strict) self.last_engine_selection = rewriter.last_engine_selection diff --git a/sidemantic/core/transport_security.py b/sidemantic/core/transport_security.py index 952043398..6f1daa319 100644 --- a/sidemantic/core/transport_security.py +++ b/sidemantic/core/transport_security.py @@ -179,6 +179,7 @@ def rewrite_transport_sql( getattr(layer, "_use_rust_sql_generator", None) if getattr(layer, "_explicit_engine", False) else None ), rust_no_fallback=getattr(layer, "_rust_no_fallback", None), + allow_non_additive_unsafe=getattr(layer, "allow_non_additive_unsafe", False), ) # Yardstick's explicit and implicit measure paths expand directly against # physical model tables. They do not currently route those reads through diff --git a/sidemantic/db/adbc.py b/sidemantic/db/adbc.py index 718c31fc5..5be813400 100644 --- a/sidemantic/db/adbc.py +++ b/sidemantic/db/adbc.py @@ -150,6 +150,26 @@ def __init__( ) self._driver_name = str(driver).lower() + package_name = None + if isinstance(driver, str) and driver.isidentifier(): + if driver.startswith("adbc_driver_"): + package_name = driver + elif driver in DRIVER_DIALECT_MAP: + package_name = f"adbc_driver_{'postgresql' if driver == 'postgres' else driver}" + if package_name is not None: + # Python wheels locate their bundled library themselves; the driver + # manager searches manifests and system library paths, not site-packages. + from importlib import import_module + + try: + driver_package = import_module(package_name) + except ModuleNotFoundError as error: + if error.name != package_name: + raise + else: + locate_driver = getattr(driver_package, "_driver_path", None) + if callable(locate_driver): + driver = locate_driver() self.conn = adbc.connect( driver=driver, uri=uri, diff --git a/sidemantic/runtime.py b/sidemantic/runtime.py new file mode 100644 index 000000000..2c5b92ceb --- /dev/null +++ b/sidemantic/runtime.py @@ -0,0 +1,13 @@ +"""Shared runtime defaults without importing the native extension.""" + +import os +import sys +from typing import Literal, cast + + +def default_engine() -> Literal["python", "rust", "auto"]: + """Use Rust on native hosts and Python in Pyodide; allow a process override.""" + engine = os.environ.get("SIDEMANTIC_ENGINE", "python" if sys.platform == "emscripten" else "rust").lower() + if engine not in {"python", "rust", "auto"}: + raise ValueError("SIDEMANTIC_ENGINE must be one of: python, rust, auto") + return cast(Literal["python", "rust", "auto"], engine) diff --git a/sidemantic/rust_bridge.py b/sidemantic/rust_bridge.py index b8bba1397..3dc580293 100644 --- a/sidemantic/rust_bridge.py +++ b/sidemantic/rust_bridge.py @@ -40,8 +40,7 @@ def get_rust_module() -> object: raise raise RustBackendUnavailableError( "Rust backend requires the sidemantic_rs Python extension. " - "Build it with: uv run --with maturin maturin develop " - "--manifest-path sidemantic-rs/Cargo.toml --features python-adbc" + "Install it with: uv pip install sidemantic-rs, or select --engine python." ) from e return sidemantic_rs @@ -122,6 +121,11 @@ def _postgres_query_sql(sql: str, *, clause: str | None = None) -> str: raise ValueError(f"PostgreSQL {clause} input requires an expression") expression = wrapper.this if clause == "WHERE" else wrapper try: + # SQLGlot has already decoded PostgreSQL E-string escapes. Lower them + # to ordinary string literals; the native DuckDB parser does not + # preserve E-string quoting when re-emitting PostgreSQL SQL. + for literal in list(expression.find_all(exp.ByteString)): + literal.replace(exp.Literal.string(literal.this)) # The intermediate parser must not infer DuckDB's default null ordering # after PostgreSQL defaults have been resolved. Emit that choice explicitly. for ordered in reversed(list(expression.find_all(exp.Ordered))): @@ -173,6 +177,7 @@ def validate_semantic_input( dimensions: list[str], *, input_dialect: str = "duckdb", + allow_non_additive_unsafe: bool = False, rust_module=None, ) -> list[str]: """Validate references through the same input contract used by compilation.""" @@ -181,7 +186,14 @@ def validate_semantic_input( module, "validate_with_semantic_input", graph_to_semantic_json(graph, input_dialect=input_dialect), - json.dumps({"metrics": metrics, "dimensions": dimensions}, allow_nan=False), + json.dumps( + { + "metrics": metrics, + "dimensions": dimensions, + **({"allow_non_additive_unsafe": True} if allow_non_additive_unsafe else {}), + }, + allow_nan=False, + ), ) if not isinstance(errors, list) or not all(isinstance(error, str) for error in errors): raise TypeError("Rust validator returned an invalid errors payload") @@ -197,6 +209,8 @@ def rewrite_semantic_input( output_dialect: str | None = None, user_attributes: dict | None = None, enforce_visibility: bool = False, + use_preaggregations: bool = False, + allow_non_additive_unsafe: bool = False, rust_module=None, ) -> str: """Rewrite SQL with caller context through the versioned graph contract.""" @@ -211,6 +225,8 @@ def rewrite_semantic_input( or sql_dialect is not None or user_attributes is not None or enforce_visibility + or use_preaggregations + or allow_non_additive_unsafe or any(model.security is not None or model.invariant_filters for model in graph.models.values()) ) args = [graph_to_semantic_json(graph, input_dialect=input_dialect), sql] @@ -226,6 +242,8 @@ def rewrite_semantic_input( { "user_attributes": user_attributes, "enforce_visibility": enforce_visibility, + **({"use_preaggregations": True} if use_preaggregations else {}), + **({"allow_non_additive_unsafe": True} if allow_non_additive_unsafe else {}), **({"output_dialect": output_dialect} if output_dialect is not None else {}), **({"sql_dialect": sql_dialect} if sql_dialect is not None else {}), }, @@ -417,6 +435,9 @@ def _normalize_metric_type(metric_payload: dict, *, empty_filters_to_none: bool normalized["type"] = None elif metric_type == "timecomparison": normalized["type"] = "time_comparison" + # Rust serializes an unset window as null; Python uses its default instead. + if normalized.get("non_additive_window") is None: + normalized.pop("non_additive_window", None) if empty_filters_to_none and normalized.get("filters") == []: normalized["filters"] = None return normalized diff --git a/sidemantic/semantic_handoff.py b/sidemantic/semantic_handoff.py index e35b69f9a..0163092eb 100644 --- a/sidemantic/semantic_handoff.py +++ b/sidemantic/semantic_handoff.py @@ -7,6 +7,8 @@ from pydantic import BaseModel +from sidemantic.core.model import Model +from sidemantic.core.relationship import Relationship from sidemantic.core.semantic_graph import SemanticGraph SEMANTIC_INPUT_VERSION = 1 @@ -31,7 +33,18 @@ def _definition(value: BaseModel) -> dict[str, Any]: field is absent. Source identity/type fields excluded by authoring exports remain part of this compiler input. """ - data = value.model_dump(mode="json", exclude_none=True, exclude_defaults=True) + if isinstance(value, Model) and value.extends: + # Unresolved children distinguish omitted fields from explicit clears + # and default-valued overrides. The receiver applies inheritance. + data = value.model_dump(mode="json", exclude_unset=True) + else: + data = value.model_dump(mode="json", exclude_none=True, exclude_defaults=True) + if isinstance(value, Relationship) and value.type in ("one_to_one", "one_to_many") and value.primary_key is None: + # TMDL retains the declared source endpoint separately from the model's + # primary key. Snapshot that existing endpoint without resolving SQL. + source_column = getattr(value, "_tmdl_from_column", None) + if isinstance(source_column, str) and source_column.strip(): + data["primary_key"] = source_column for name in ("logical_data_type", "declared_is_time", "edge_id"): field_value = getattr(value, name, None) if field_value is not None: @@ -46,7 +59,7 @@ def _definition(value: BaseModel) -> dict[str, Any]: def graph_to_semantic_input(graph: SemanticGraph, *, input_dialect: str = "duckdb") -> dict[str, Any]: - """Create an inert versioned snapshot of a resolved semantic graph. + """Create an inert versioned snapshot of a semantic graph. Graph metrics stay at graph scope. Only owners explicitly recorded by the graph are transmitted. SQL is copied verbatim; reference binding, dialect @@ -57,13 +70,24 @@ def graph_to_semantic_input(graph: SemanticGraph, *, input_dialect: str = "duckd models = [] for model in graph.models.values(): definition = _definition(model) - definition["primary_key"] = list(model.primary_key_columns) or None + if not model.extends or "primary_key" in model.model_fields_set: + definition["primary_key"] = list(model.primary_key_columns) or None models.append(definition) return { "version": SEMANTIC_INPUT_VERSION, "input_dialect": input_dialect, "models": models, - "metrics": [_definition(metric) for metric in graph.metrics.values()], + "metrics": [ + _definition(metric) + for metric in graph.metrics.values() + # add_model exposes these same objects through graph.metrics for + # unqualified lookup. The native model index already does that. + if not ( + metric.type in ("time_comparison", "conversion") + and metric.name not in graph.metric_owners + and any(metric is owned for model in graph.models.values() for owned in model.metrics) + ) + ], "metric_owners": dict(graph.metric_owners), "parameters": [_definition(parameter) for parameter in graph.parameters.values()], "table_calculations": [_definition(calculation) for calculation in graph.table_calculations.values()], diff --git a/sidemantic/sql/generator.py b/sidemantic/sql/generator.py index f8217d891..4354481fe 100644 --- a/sidemantic/sql/generator.py +++ b/sidemantic/sql/generator.py @@ -938,6 +938,10 @@ def _apply_default_time_dimensions(self, metrics: list[str], dimensions: list[st model_name, metric_obj = self.graph.resolve_metric_reference(metric_ref) except KeyError: continue + # Retention uses its default time dimension as the cohort source, + # not an extra projected/grouped dimension in its fixed result. + if metric_obj is not None and metric_obj.type == "retention": + continue # A graph-level cumulative metric (e.g. a MetricFlow ``input_metric``) # resolves to no model, so the default-time-dimension logic below would # skip it and cumulative ordering would fail with "requires a time @@ -3021,6 +3025,12 @@ def _generate_with_preaggregation( calculations: dict[str, str] = {} leaf_refs: list[str] = [] + # Children need stable, distinct output names even when public fields + # share a basename or a calculation hides a colliding aggregate leaf. + child_aliases = { + f"{reference}__{grain}" if grain else reference: f"__sidemantic_dimension_{index}" + for index, (reference, grain) in enumerate(parsed_dims) + } def expand_metric(reference: str, context: str | None = None, stack: tuple[str, ...] = ()) -> str: if "." not in reference and context and self.graph.get_model(context).get_metric(reference): @@ -3032,8 +3042,18 @@ def expand_metric(reference: str, context: str | None = None, stack: tuple[str, if model_name is not None and aggregate_models == {model_name}: if reference not in leaf_refs: leaf_refs.append(reference) - source_name = aliases.get(reference) or metric.name - calculations[reference] = f"{model_name}_preagg.{self._quote_identifier(source_name)}" + child_aliases[reference] = f"__sidemantic_metric_{len(leaf_refs) - 1}" + source_name = child_aliases[reference] + expression = f"{model_name}_preagg.{self._quote_identifier(source_name)}" + if metric.agg in ("count", "count_distinct", "approx_count_distinct"): + # A missing group has an empty count population. Restore + # zero before evaluating formulas or their outer defaults. + expression = f"COALESCE({expression}, 0)" + else: + # A source can be absent from a sibling's group entirely, + # so its child-level default has no row on which to run. + expression = self._wrap_with_fill_nulls(expression, metric) + calculations[reference] = expression return calculations[reference] stack = (*stack, reference) if metric.type == "ratio": @@ -3166,8 +3186,13 @@ def expand_metric(reference: str, context: str | None = None, stack: tuple[str, # Generate sub-query for this model's metrics at the dimension grain # We call generate() recursively but it won't trigger pre-aggregation - # again because each sub-query has metrics from only one model - sub_query = self.generate( + # again because each sub-query has metrics from only one model. + # Preserve that source's unmatched rows regardless of dimension + # order. An explicit Explore scope still controls the population. + child_generator = copy(self) + child_generator.base_model = self.base_model or model_name + child_generator._generate_cache = {} + sub_query = child_generator.generate( metrics=model_metrics, dimensions=dimensions, _resolved_filters=tuple(model_filters), @@ -3175,7 +3200,7 @@ def expand_metric(reference: str, context: str | None = None, stack: tuple[str, order_by=None, limit=None, offset=None, - aliases=aliases, + aliases=child_aliases, use_preaggregations=use_preaggregations, user_attributes=user_attributes, ) @@ -3191,6 +3216,13 @@ def expand_metric(reference: str, context: str | None = None, stack: tuple[str, # Build the final SELECT that joins all pre-aggregated CTEs select_exprs = [] output_names: dict[str, str] = {} + public_name_counts: dict[str, int] = {} + for dim_ref, gran in parsed_dims: + name = dim_ref.split(".", 1)[-1] + (f"__{gran}" if gran else "") + public_name_counts[name] = public_name_counts.get(name, 0) + 1 + for reference in metrics: + _, metric = self.graph.resolve_metric_reference(reference) + public_name_counts[metric.name] = public_name_counts.get(metric.name, 0) + 1 def register_output_name(output_name: str, *refs: str) -> None: for ref in refs: @@ -3205,11 +3237,9 @@ def dimension_output_name(dim_ref: str, dim_name: str, gran: str | None) -> str: full_ref = dim_ref default = dim_name - canonical_ref = full_ref - return aliases.get(full_ref) or aliases.get(canonical_ref) or default - - def metric_source_name(metric_ref: str, metric_name: str) -> str: - return aliases.get(metric_ref) or aliases.get(f"{metric_ref.split('.')[0]}.{metric_name}") or metric_name + if public_name_counts[default] > 1: + default = f"{dim_ref.split('.', 1)[0]}_{default}" + return aliases.get(full_ref) or default # Add dimensions - use COALESCE across all CTEs for dim_ref, gran in parsed_dims: @@ -3221,44 +3251,34 @@ def metric_source_name(metric_ref: str, metric_name: str) -> str: register_output_name(output_name, full_ref, canonical_ref, dim_name, col_name, output_name) # Build COALESCE expression - quoted_source = self._quote_identifier(output_name) + quoted_source = self._quote_identifier(child_aliases[full_ref]) coalesce_parts = [f"{cte}.{quoted_source}" for cte in cte_names] select_exprs.append(f"COALESCE({', '.join(coalesce_parts)}) AS {self._quote_alias(output_name)}") - # Check for metric name collisions across models - metric_name_counts: dict[str, int] = {} - for metric_ref in metrics: - _, metric = self.graph.resolve_metric_reference(metric_ref) - metric_name_counts[metric.name] = metric_name_counts.get(metric.name, 0) + 1 - # Add metrics in requested order, including calculations over child CTEs. metric_selects: dict[str, str] = {} for model_name, model_metrics in metrics_by_model.items(): - cte_name = f"{model_name}_preagg" for metric_ref in model_metrics: if metric_ref not in metrics: continue metric_name = metric_ref.split(".", 1)[1] if "." in metric_ref else metric_ref - source_name = metric_source_name(metric_ref, metric_name) # Check for custom alias first if metric_ref in aliases: alias = aliases[metric_ref] - elif metric_name_counts.get(metric_name, 1) > 1: + elif public_name_counts.get(metric_name, 1) > 1: # Collision - prefix with model name alias = f"{model_name}_{metric_name}" else: alias = metric_name register_output_name(alias, metric_ref, f"{model_name}.{metric_name}", metric_name, alias) - metric_selects[metric_ref] = ( - f"{cte_name}.{self._quote_identifier(source_name)} AS {self._quote_alias(alias)}" - ) + metric_selects[metric_ref] = f"{calculations[metric_ref]} AS {self._quote_alias(alias)}" for metric_ref in metrics: if metric_ref in leaf_refs: continue model_name, metric = self.graph.resolve_metric_reference(metric_ref) default_alias = ( - f"{model_name}_{metric.name}" if model_name and metric_name_counts[metric.name] > 1 else metric.name + f"{model_name}_{metric.name}" if model_name and public_name_counts[metric.name] > 1 else metric.name ) alias = aliases.get(metric_ref) or default_alias register_output_name(alias, metric_ref, metric.name, alias) @@ -3271,7 +3291,7 @@ def metric_source_name(metric_ref: str, metric_name: str) -> str: # Join remaining CTEs join_clauses = [] - for cte_name in cte_names[1:]: + for index, cte_name in enumerate(cte_names[1:], start=1): if not parsed_dims: # No dimensions - use CROSS JOIN (each CTE returns single row) join_clauses.append(f"CROSS JOIN {cte_name}") @@ -3279,10 +3299,14 @@ def metric_source_name(metric_ref: str, metric_name: str) -> str: # Build join condition on all dimension columns join_conditions = [] for dim_ref, gran in parsed_dims: - dim_name = dim_ref.split(".")[1] if "." in dim_ref else dim_ref - col_name = dimension_output_name(dim_ref, dim_name, gran) + full_ref = f"{dim_ref}__{gran}" if gran else dim_ref + col_name = child_aliases[full_ref] # NULL-safe equality that works for all column types - lhs = exp.Column(this=col_name, table=cte_names[0]) + # and groups introduced by any preceding source. + previous = [exp.column(col_name, table=previous_cte) for previous_cte in cte_names[:index]] + lhs = ( + previous[0] if len(previous) == 1 else exp.Coalesce(this=previous[0], expressions=previous[1:]) + ) rhs = exp.Column(this=col_name, table=cte_name) join_conditions.append(exp.NullSafeEQ(this=lhs, expression=rhs).sql(dialect=self._dialect_instance)) @@ -3302,9 +3326,8 @@ def metric_source_name(metric_ref: str, metric_name: str) -> str: if shared_filters: filter_expressions = dict(calculations) for dim_ref, gran in parsed_dims: - dim_name = dim_ref.split(".", 1)[-1] - source_name = self._quote_identifier(dimension_output_name(dim_ref, dim_name, gran)) full_ref = f"{dim_ref}__{gran}" if gran else dim_ref + source_name = self._quote_identifier(child_aliases[full_ref]) filter_expressions[full_ref] = f"COALESCE({', '.join(f'{cte}.{source_name}' for cte in cte_names)})" preagg_table_map = {} for model_name in metrics_by_model: @@ -4933,6 +4956,11 @@ def _build_metric_sql(self, metric, model_context: str | None = None) -> str: Returns: SQL expression string """ + if metric.type == "retention": + raise QueryValidationError( + "Retention metrics have a fixed table output and cannot be wrapped in scalar metrics " + "(metric.retention_wrapped_metric)" + ) if getattr(metric, "has_untranslated_dax", False): raise QueryValidationError( f"Metric '{metric.name}' contains DAX expression but has no SQL translation. " @@ -5381,6 +5409,7 @@ def _replace_model_placeholder(expr: str) -> str: outer_select_cols = [] outer_group_cols = [] + output_names = [*entity_dim_aliases, metric.name] # Add entity_dimensions to outer SELECT/GROUP BY for alias in entity_dim_aliases: @@ -5419,6 +5448,7 @@ def _replace_model_placeholder(expr: str) -> str: inner_group_cols.append(dim_sql) outer_select_cols.append(quoted_alias) outer_group_cols.append(quoted_alias) + output_names.append(alias) # Join inner select/group after dimensions are added inner_select = ",\n ".join(inner_select_cols + inner_metric_selects) @@ -5437,18 +5467,7 @@ def _replace_model_placeholder(expr: str) -> str: group_by = f"\nGROUP BY {', '.join(outer_group_cols)}" # Order/limit/offset - order_clause = "" - if order_by: - order_fields = [] - for field in order_by: - field_name = field.split(".", 1)[1] if "." in field else field - # Handle "desc"/"asc" suffix - parts = field_name.rsplit(" ", 1) - if len(parts) == 2 and parts[1].upper() in ("ASC", "DESC"): - order_fields.append(f"{quote_alias(parts[0])} {parts[1].upper()}") - else: - order_fields.append(quote_alias(field_name)) - order_clause = f"\nORDER BY {', '.join(order_fields)}" + order_clause = self._specialized_order_clause(order_by, output_names) limit_clause = f"\nLIMIT {limit}" if limit is not None else "" offset_clause = f"\nOFFSET {offset}" if offset is not None else "" @@ -5486,7 +5505,7 @@ def _generate_retention_query( Args: metric_name: Name of the retention metric - dimensions: List of dimension references (unused for retention, reserved) + dimensions: Must be empty; retention has a fixed output projection filters: List of filter expressions order_by: List of fields to order by limit: Maximum number of rows to return @@ -5497,6 +5516,11 @@ def _generate_retention_query( """ import re as _re + if dimensions: + raise QueryValidationError( + "Retention metrics do not support selected dimensions (metric.retention_query_shape)" + ) + # Resolve metric and model metric = None model = None @@ -5613,6 +5637,22 @@ def _replace_model_placeholder(expr: str) -> str: if metric.filters: all_filters.extend(metric.filters) + # These predicates scope source rows, not an aggregate result. Previously + # metric references leaked into WHERE as nonexistent physical columns. + for filter_sql in all_filters: + parsed_filter = _parse_fragment(filter_sql.replace("{model}", model.name), self.dialect) + references_metric = any( + (not column.table or column.table == model.name) + and not model.get_dimension(column.name) + and (model.get_metric(column.name) or column.name in self.graph.metrics) + for column in parsed_filter.find_all(exp.Column) + ) + if references_metric or sql_has_aggregate(filter_sql, self.dialect): + raise QueryValidationError( + "Retention source filters cannot reference aggregate metrics or expressions " + "(metric.retention_aggregate_filter)" + ) + # Normalize filters: strip model name prefixes and resolve dimension names normalized_filters = self._strip_model_prefixes(all_filters, model.name) normalized_filters = self._resolve_filter_dimensions(normalized_filters, model) diff --git a/sidemantic/sql/query_rewriter.py b/sidemantic/sql/query_rewriter.py index 24195ba66..b04414236 100644 --- a/sidemantic/sql/query_rewriter.py +++ b/sidemantic/sql/query_rewriter.py @@ -143,6 +143,7 @@ def __init__( enforce_visibility: bool = False, use_rust_rewriter: bool | None = None, rust_no_fallback: bool | None = None, + allow_non_additive_unsafe: bool = False, ): """Initialize query rewriter. @@ -153,11 +154,18 @@ def __init__( enforce_visibility: Reject semantic references to fields declared ``public: false`` use_rust_rewriter: Override the environment-controlled Rust rewrite path rust_no_fallback: Reject unsupported Rust requirements instead of falling back + allow_non_additive_unsafe: Aggregate all snapshots without semi-additive protection """ self.graph = graph self.dialect = dialect self.use_preaggregations = use_preaggregations - self.generator = SQLGenerator(graph, dialect=dialect, enforce_visibility=enforce_visibility) + self.allow_non_additive_unsafe = allow_non_additive_unsafe + self.generator = SQLGenerator( + graph, + dialect=dialect, + enforce_visibility=enforce_visibility, + allow_non_additive_unsafe=allow_non_additive_unsafe, + ) self.enforce_visibility = enforce_visibility self._dialect_instance = self.generator._dialect_instance self._rewrite_cache: dict[tuple[object, ...], str] = {} @@ -226,6 +234,8 @@ def _route_rust(self, sql: str, user_attributes: dict | None, strict: bool) -> s return sql contextual_rust = ( user_attributes is not None + or self.use_preaggregations + or self.allow_non_additive_unsafe or self.enforce_visibility or any(model.security is not None or model.invariant_filters for model in self.graph.models.values()) ) @@ -235,17 +245,14 @@ def _route_rust(self, sql: str, user_attributes: dict | None, strict: bool) -> s self._raise_on_user_cte_name_collision(parsed) self.last_engine_selection = {"engine": "rust", "reason": "Rust engine selected"} try: - capabilities = [] - if self.use_preaggregations: - capabilities.append("query.preaggregations") - if capabilities: - raise UnsupportedSemanticFeaturesError(capabilities) rewritten = rewrite_semantic_input( self.graph, sql, input_dialect="duckdb" if self.dialect == "postgres" else self.dialect, user_attributes=user_attributes, enforce_visibility=self.enforce_visibility, + **({"use_preaggregations": True} if self.use_preaggregations else {}), + **({"allow_non_additive_unsafe": True} if self.allow_non_additive_unsafe else {}), **({"sql_dialect": self.dialect, "output_dialect": self.dialect} if self.dialect != "duckdb" else {}), ) self.last_engine_selection = {"engine": "rust", "reason": None} @@ -3130,9 +3137,9 @@ def _generate_from_plan(self, plan: SemanticQueryPlan, query: exp.Select | None ) references = [*plan.metrics, *plan.dimensions] repeated_projection = len(references) != len(set(references)) - restore_projection = query is not None and (temporal_projection or repeated_projection) + restore_projection = query is not None order_by = plan.order_by - if restore_projection and order_by: + if restore_projection and (temporal_projection or repeated_projection) and order_by: alias_references = {} for expression in query.expressions: if isinstance(expression, exp.Alias) and isinstance(expression.this, exp.Column): @@ -3156,19 +3163,40 @@ def _generate_from_plan(self, plan: SemanticQueryPlan, query: exp.Select | None user_attributes=getattr(self, "_rewrite_user_attributes", None), _query_ctes=self._query_ctes_in_scope(query), ) - # Time-comparison generators also expose their base measures for structured - # queries. A SQL SELECT keeps only the columns its projection requests. - # Repeated references likewise need separate projection aliases because - # the structured query carries only one alias per semantic reference. + # Structured queries emit dimensions first and some temporal generators + # expose extra base measures. SQL SELECT owns its output order and fields, + # including separate aliases for repeated semantic references. if restore_projection: - requested = [expression.alias_or_name for expression in query.expressions] - if all(requested) and not any(isinstance(expression, exp.Star) for expression in query.expressions): + projection_query = query + # Flattened SELECT * wrappers inherit their source's column order. + # SQLGlot scopes resolve both named CTEs and derived tables without + # confusing physical source columns with semantic projections. + from sqlglot.optimizer.scope import Scope, build_scope + + scope = build_scope(query) + while ( + scope is not None + and len(projection_query.expressions) == 1 + and isinstance(projection_query.expressions[0], exp.Star) + and len(scope.selected_sources) == 1 + ): + _, source_scope = next(iter(scope.selected_sources.values())) + if not isinstance(source_scope, Scope) or not isinstance(source_scope.expression, exp.Select): + break + scope = source_scope + projection_query = scope.expression + requested = [expression.alias_or_name for expression in projection_query.expressions] + if all(requested) and not any( + isinstance(expression, exp.Star) for expression in projection_query.expressions + ): generated = parse_fragment(generated_sql, self.dialect) if isinstance(generated, exp.Select): + if requested == [expression.alias_or_name for expression in generated.expressions]: + return generated_sql outputs = {expression.alias_or_name: expression for expression in generated.expressions} projections = [] restored_names = {} - for expression, name in zip(query.expressions, requested): + for expression, name in zip(projection_query.expressions, requested): source = expression.this if isinstance(expression, exp.Alias) else expression if not isinstance(source, exp.Column): break @@ -3184,12 +3212,31 @@ def _generate_from_plan(self, plan: SemanticQueryPlan, query: exp.Select | None output = output.this if isinstance(output, exp.Alias) else output projections.append(output.copy().as_(name)) if len(projections) == len(requested): + # GROUP BY positions refer to the generated projection, + # not the user's reordered SELECT. Bind them before + # replacing the output list. + if group := generated.args.get("group"): + for position in list(group.expressions): + if isinstance(position, exp.Literal) and position.is_int: + index = int(position.this) - 1 + if 0 <= index < len(generated.expressions): + field = generated.expressions[index] + field = field.this if isinstance(field, exp.Alias) else field + position.replace(field.copy()) generated.set("expressions", projections) if order := generated.args.get("order"): for column in order.find_all(exp.Column): if not column.table and column.name in restored_names: column.set("this", exp.to_identifier(restored_names[column.name])) - return generated.sql(dialect=self.dialect, pretty=True) + rewritten = generated.sql(dialect=self.dialect) + # Replacing the final projection can discard the SQL + # parser's attached routing/instrumentation comments. + # Missing-rollup fallback depends on this metadata. + for line in generated_sql.splitlines(): + if line.startswith("-- sidemantic:") or line == "-- used_preagg=true": + if line not in rewritten: + rewritten += "\n" + line + return rewritten return generated_sql def _dedupe(self, values: list[str]) -> list[str]: diff --git a/sidemantic/validation.py b/sidemantic/validation.py index f498ab8ed..42481b0c6 100644 --- a/sidemantic/validation.py +++ b/sidemantic/validation.py @@ -897,11 +897,19 @@ def _add_untranslated_dax_model_error(model_ref: str, model) -> None: else: _add_untranslated_dax_model_error(model_name, model) dimension = model.get_dimension(dim_name) - if not dimension: + # The generator projects local many-to-one foreign keys when + # explicitly selected, even without a Dimension declaration. + relationship_key = any( + relationship.type == "many_to_one" + and not (relationship.sql and ("{from}" in relationship.sql or "{to}" in relationship.sql)) + and dim_name in relationship.foreign_key_columns + for relationship in model.relationships + ) + if not dimension and not relationship_key: errors.append( f"Dimension '{dim_name}' not found in model '{model_name}' (referenced in '{dim_ref}')" ) - else: + elif dimension: _add_untranslated_dax_dimension_error(dim_ref, dimension) else: errors.append(f"Dimension reference '{dim_ref}' must be in 'model.dimension' format") diff --git a/tests/adapters/bsl/test_parsing.py b/tests/adapters/bsl/test_parsing.py index 12b26a01e..53435d596 100644 --- a/tests/adapters/bsl/test_parsing.py +++ b/tests/adapters/bsl/test_parsing.py @@ -1300,6 +1300,7 @@ def test_reused_join_aliases_scope_to_source_model(self): import tempfile from pathlib import Path + import duckdb import yaml from sidemantic.sql.generator import SQLGenerator @@ -1394,7 +1395,7 @@ def test_reused_join_aliases_scope_to_source_model(self): dimensions=["orders.order_id"], skip_default_time_dimensions=True, ) - assert "JOIN orders_user_cte" in orders_sql + assert "orders_user_cte" in orders_sql assert "events_user_cte" not in orders_sql assert "FROM customers" in orders_sql @@ -1403,10 +1404,22 @@ def test_reused_join_aliases_scope_to_source_model(self): dimensions=["events.event_id"], skip_default_time_dimensions=True, ) - assert "JOIN events_user_cte" in events_sql + assert "events_user_cte" in events_sql assert "orders_user_cte" not in events_sql assert "FROM accounts" in events_sql + with duckdb.connect() as conn: + conn.execute("CREATE TABLE orders (order_id INTEGER, user_id INTEGER)") + conn.execute("INSERT INTO orders VALUES (1, 10), (2, 10)") + conn.execute("CREATE TABLE customers (customer_id INTEGER, name VARCHAR)") + conn.execute("INSERT INTO customers VALUES (10, 'customer')") + conn.execute("CREATE TABLE events (event_id INTEGER, account_id INTEGER)") + conn.execute("INSERT INTO events VALUES (3, 20)") + conn.execute("CREATE TABLE accounts (account_id INTEGER, name VARCHAR)") + conn.execute("INSERT INTO accounts VALUES (20, 'account')") + assert sorted(conn.execute(orders_sql).fetchall()) == [(1, 1.0), (2, 1.0)] + assert conn.execute(events_sql).fetchall() == [(3, 1.0)] + adapter.export(graph, export_path) with open(export_path) as f: exported = yaml.safe_load(f) diff --git a/tests/adapters/cube/test_correctness_fixes.py b/tests/adapters/cube/test_correctness_fixes.py index 67a4aa7c0..6c53f4c05 100644 --- a/tests/adapters/cube/test_correctness_fixes.py +++ b/tests/adapters/cube/test_correctness_fixes.py @@ -1442,7 +1442,11 @@ def test_cross_cube_trailing_column_ref_translated_to_member(): layer.graph = graph compiled = layer.compile(metrics=["line_items.derived_x"]) assert "${orders}" not in compiled and "{'orders'" not in compiled - assert "SUM(__sidemantic_dedup." in compiled + layer.conn.execute("CREATE TABLE orders (id INTEGER, amt INTEGER)") + layer.conn.execute("INSERT INTO orders VALUES (1, 10), (2, 20)") + layer.conn.execute("CREATE TABLE line_items (order_id INTEGER)") + layer.conn.execute("INSERT INTO line_items VALUES (1), (1), (2)") + assert layer.conn.execute(compiled).fetchall() == [(60,)] def test_rollup_with_only_unmaterializable_measures_is_rejected(): diff --git a/tests/adapters/cube/test_kitchen_sink.py b/tests/adapters/cube/test_kitchen_sink.py index 138265abe..8c9bd85c0 100644 --- a/tests/adapters/cube/test_kitchen_sink.py +++ b/tests/adapters/cube/test_kitchen_sink.py @@ -33,7 +33,7 @@ import duckdb import pytest -from sidemantic import SemanticLayer +from sidemantic import Relationship, SemanticLayer from sidemantic.adapters.cube import CubeAdapter from tests.utils import fetch_dicts @@ -679,14 +679,18 @@ class TestEdgeCasesAndHoles: """Test potential edge cases that might reveal holes in sidemantic.""" def test_self_referential_join(self, kitchen_sink_layer): - """Test self-referential relationship (manager_id). - - This is a potential hole - does sidemantic support self-joins? - """ - # This would require a self-join: employees.manager_id -> employees.id - # Most semantic layers don't handle this well - # We skip this as a known limitation but document it - pytest.skip("Self-referential joins not currently supported - potential enhancement") + """Group employees by their manager's name, retaining employees without managers.""" + kitchen_sink_layer.graph.models["employees"].relationships.append( + Relationship(name="manager", target_model="employees", type="many_to_one", foreign_key="manager_id") + ) + result = kitchen_sink_layer.query(metrics=["employees.count"], dimensions=["manager.name"]) + assert {row["name"]: row["count"] for row in fetch_dicts(result)} == { + None: 5, + "Alice Smith": 2, + "Dave Brown": 1, + "Frank Miller": 1, + "Henry Chen": 1, + } def test_multiple_paths_to_same_model(self, kitchen_sink_layer): """Test when there are multiple paths to the same model. @@ -839,19 +843,12 @@ def test_numeric_dimension_as_grouping(self, kitchen_sink_layer): assert by_priority[4] == 1 def test_aggregation_on_aggregation(self, kitchen_sink_layer): - """Test whether we can aggregate an already aggregated value. - - E.g., AVG of department budgets per company. - This requires a two-level aggregation. - """ - # This might not work - sidemantic might not support nested aggregation - try: - # First aggregate departments by company, then avg those budgets - # This would be: SELECT company, AVG(dept_budget_per_company) - # which requires subquery or window function - pytest.skip("Nested aggregation not directly supported") - except Exception: - pytest.skip("Nested aggregation not supported") + """Average company-level department budgets after semantic aggregation.""" + result = kitchen_sink_layer.sql( + "SELECT AVG(total_budget) AS average_company_budget FROM " + "(SELECT companies.name, departments.total_budget FROM departments) AS company_budgets" + ) + assert result.fetchone() == (4125000.0,) # ============================================================ diff --git a/tests/adapters/lookml/test_edge_cases.py b/tests/adapters/lookml/test_edge_cases.py index fc828e7c8..fb7011c4f 100644 --- a/tests/adapters/lookml/test_edge_cases.py +++ b/tests/adapters/lookml/test_edge_cases.py @@ -5813,8 +5813,12 @@ def test_lookml_implicit_dimension_group_compiles_against_group_column(): layer = SemanticLayer() layer.graph = graph sql = layer.compile(dimensions=["orders.created_date"], metrics=["orders.cnt"]) - assert "DATE_TRUNC('day', created)" in sql, sql - assert "'day', created_date)" not in sql, sql # never the generated field name + layer.conn.execute("CREATE TABLE orders (id INTEGER, created TIMESTAMP)") + layer.conn.execute("INSERT INTO orders VALUES (1, '2024-01-02 10:00:00'), (2, '2024-01-02 15:00:00')") + rows = layer.conn.execute(sql).fetchall() + assert len(rows) == 1 + day, count = rows[0] + assert (day.year, day.month, day.day, count) == (2024, 1, 2, 2) # Export still round-trips the group with no invented sql. out = tempfile.mktemp(suffix=".lkml") diff --git a/tests/adapters/malloy/test_exact_joins.py b/tests/adapters/malloy/test_exact_joins.py index 3dbc5ae85..6efbc51cb 100644 --- a/tests/adapters/malloy/test_exact_joins.py +++ b/tests/adapters/malloy/test_exact_joins.py @@ -40,7 +40,7 @@ def test_join_one_on_alternate_keys_compiles_and_executes(tmp_path): assert relationship.primary_key == "external_ref" assert relationship.sql is None - layer = SemanticLayer(auto_register=False, engine="python") + layer = SemanticLayer(auto_register=False) layer.adapter.execute("create table customers (id int, external_ref text, name text)") layer.adapter.execute("create table orders (id int, customer_ref text, amount int)") layer.adapter.execute("insert into customers values (1, 'C-1', 'Ada'), (2, 'C-2', 'Grace')") @@ -100,7 +100,7 @@ def test_additional_predicate_uses_placeholders_and_preserves_keys(tmp_path): assert relationship.sql == "{from}.customer_id = {to}.id and {to}.active = true" assert relationship.metadata["join_key_pairs"] == [{"source": "customer_id", "target": "id"}] - layer = SemanticLayer(auto_register=False, engine="python") + layer = SemanticLayer(auto_register=False) layer.adapter.execute("create table customers (id int, name text, active boolean)") layer.adapter.execute("create table orders (id int, customer_id int, amount int)") layer.adapter.execute("insert into customers values (1, 'kept', true), (2, 'inactive', false)") @@ -144,7 +144,7 @@ def test_rooted_join_many_preserves_unmatched_source_and_exact_keys(tmp_path): assert relationship.primary_key == "order_number" assert relationship.foreign_key == "order_ref" - layer = SemanticLayer(auto_register=False, engine="python") + layer = SemanticLayer(auto_register=False) layer.adapter.execute("create table items (id int, order_ref text, sku text)") layer.adapter.execute("create table orders (id int, order_number text, amount int)") layer.adapter.execute("insert into items values (1, 'O-1', 'A'), (2, 'O-1', 'B')") @@ -184,7 +184,7 @@ def test_range_predicate_on_both_sides_lowers_to_custom_sql(tmp_path): assert relationship.primary_key is None assert relationship.sql == ("{from}.created_at >= {to}.valid_from and {from}.created_at < {to}.valid_to") - layer = SemanticLayer(auto_register=False, engine="python") + layer = SemanticLayer(auto_register=False) layer.adapter.execute("create table prices (id int, valid_from int, valid_to int, label text)") layer.adapter.execute("create table orders (id int, created_at int, amount int)") layer.adapter.execute("insert into prices values (1, 0, 10, 'early'), (2, 10, 20, 'late')") diff --git a/tests/adapters/malloy/test_phase3_integration.py b/tests/adapters/malloy/test_phase3_integration.py index 53bdc1fe0..d37e6abde 100644 --- a/tests/adapters/malloy/test_phase3_integration.py +++ b/tests/adapters/malloy/test_phase3_integration.py @@ -13,7 +13,7 @@ def _parse(tmp_path, source: str, *, strict: bool = True): def _layer(graph, *setup_sql): - layer = SemanticLayer(auto_register=False, engine="python") + layer = SemanticLayer(auto_register=False) for statement in setup_sql: layer.adapter.execute(statement) for model in graph.models.values(): diff --git a/tests/adapters/malloy/test_role_aliases.py b/tests/adapters/malloy/test_role_aliases.py index 925bcd806..6dcf2726d 100644 --- a/tests/adapters/malloy/test_role_aliases.py +++ b/tests/adapters/malloy/test_role_aliases.py @@ -35,7 +35,7 @@ def test_role_aliases_import_compile_execute_and_roundtrip(tmp_path): assert relationships["origin"].target_model == "airports" assert relationships["destination"].target_model == "airports" - layer = SemanticLayer(auto_register=False, engine="python") + layer = SemanticLayer(auto_register=False) layer.adapter.execute("create table airports(id int, city text)") layer.adapter.execute("insert into airports values (1, 'SFO'), (2, 'LAX'), (3, 'JFK')") layer.adapter.execute("create table flights(id int, origin_id int, destination_id int)") @@ -118,7 +118,7 @@ def test_native_custom_join_sql_exports_exactly_and_executes_after_reparse(tmp_p relationship = reparsed.get_model("orders").relationships[0] assert relationship.sql == "{from}.customer_id = {to}.id and {to}.active = true" - layer = SemanticLayer(auto_register=False, engine="python") + layer = SemanticLayer(auto_register=False) layer.adapter.execute("create table customers(id int, name text, active boolean)") layer.adapter.execute("insert into customers values (1, 'Ada', true), (2, 'Grace', false)") layer.adapter.execute("create table orders(id int, customer_id int, amount int)") diff --git a/tests/adapters/malloy/test_source_governance.py b/tests/adapters/malloy/test_source_governance.py index 83407e94c..dc69b9bad 100644 --- a/tests/adapters/malloy/test_source_governance.py +++ b/tests/adapters/malloy/test_source_governance.py @@ -33,7 +33,7 @@ def test_source_where_is_conjunctive_invariant_and_executes(tmp_path): assert model.invariant_filters == ["tenant_id = 1", "active"] assert model.segments == [] - layer = SemanticLayer(auto_register=False, engine="python") + layer = SemanticLayer(auto_register=False) layer.adapter.execute("create table orders (tenant_id int, active boolean, status text, amount int)") layer.adapter.execute( "insert into orders values (1, true, 'kept', 10), (1, false, 'inactive', 20), (2, true, 'other', 30)" @@ -102,7 +102,7 @@ def test_private_and_internal_rename_visibility_executes_and_roundtrips(tmp_path } assert any("internal renamed dimension" in issue for _, issue in adapter.unsupported_features) - layer = SemanticLayer(auto_register=False, engine="python", enforce_visibility=True) + layer = SemanticLayer(auto_register=False, enforce_visibility=True) layer.adapter.execute("create table accounts (account_id int, region text, secret_token text)") layer.adapter.execute("insert into accounts values (1, 'west', 'hidden')") layer.add_model(model) @@ -139,7 +139,7 @@ def test_intrinsic_columns_and_explicit_primary_key_execute(tmp_path): assert model.auto_dimensions is True assert model.schema_exposure.include_primary_key is True - layer = SemanticLayer(auto_register=False, engine="python") + layer = SemanticLayer(auto_register=False) layer.adapter.execute("create table orders (order_id int, region text, amount int)") layer.adapter.execute("insert into orders values (1, 'west', 10), (2, 'east', 20)") layer.add_model(model) @@ -174,7 +174,7 @@ def test_accept_except_and_private_filter_before_schema_exposure(tmp_path): assert model.schema_exposure.private == ["secret_token"] assert adapter.unsupported_features == [] - layer = SemanticLayer(auto_register=False, engine="python", enforce_visibility=True) + layer = SemanticLayer(auto_register=False, enforce_visibility=True) layer.adapter.execute("create table accounts (account_id int, region text, balance int, secret_token text)") layer.adapter.execute("insert into accounts values (1, 'west', 10, 'hidden')") layer.add_model(model) @@ -280,7 +280,7 @@ def test_strict_schema_introspection_failure_and_lenient_unsafe_shape_report(tmp ) model = graph.get_model("missing") assert model.schema_exposure.strict is True - layer = SemanticLayer(auto_register=False, engine="python") + layer = SemanticLayer(auto_register=False) with pytest.raises(SchemaIntrospectionError, match="missing"): layer.add_model(model) diff --git a/tests/adapters/metricflow/test_fixtures.py b/tests/adapters/metricflow/test_fixtures.py index a0593004f..b0ce92dd5 100644 --- a/tests/adapters/metricflow/test_fixtures.py +++ b/tests/adapters/metricflow/test_fixtures.py @@ -14,6 +14,7 @@ from pathlib import Path import pytest +from pydantic import ValidationError from sidemantic.adapters.metricflow import MetricFlowAdapter @@ -622,23 +623,21 @@ def test_metric_count(self, graph): class TestUnsupportedFeatures: """Tests for MetricFlow features not yet supported by sidemantic.""" - @pytest.mark.xfail( - reason="Dimension model does not support 'millisecond' granularity", - raises=Exception, - ) def test_millisecond_granularity(self): """Parsing a dimension with millisecond granularity fails validation.""" adapter = MetricFlowAdapter() - adapter.parse(FIXTURES / "sub_daily_millisecond.yml") + with pytest.raises(ValidationError) as error: + adapter.parse(FIXTURES / "sub_daily_millisecond.yml") + assert error.value.errors()[0]["loc"] == ("granularity",) + assert error.value.errors()[0]["input"] == "millisecond" - @pytest.mark.xfail( - reason="Metric model does not support sub-daily grain_to_date (hour)", - raises=Exception, - ) def test_subdaily_grain_to_date(self): """Parsing a cumulative metric with grain_to_date: hour fails validation.""" adapter = MetricFlowAdapter() - adapter.parse(FIXTURES / "sub_daily_grain_to_date_hour.yml") + with pytest.raises(ValidationError) as error: + adapter.parse(FIXTURES / "sub_daily_grain_to_date_hour.yml") + assert error.value.errors()[0]["loc"] == ("grain_to_date",) + assert error.value.errors()[0]["input"] == "hour" # ============================================================================= diff --git a/tests/adapters/metricflow/test_query.py b/tests/adapters/metricflow/test_query.py index a51780602..50a2cdeff 100644 --- a/tests/adapters/metricflow/test_query.py +++ b/tests/adapters/metricflow/test_query.py @@ -155,12 +155,13 @@ def test_inline_simple_metric_filter_is_applied(): def test_inline_metric_filter_qualified_in_join(): - """A filtered inline metric's columns are qualified to its owning model CTE. + """A filtered inline metric's columns bind to its owning model. Regression: the metric filter was rendered with unqualified columns. When the metric is queried with a joined dimension whose CTE also exposes a same-named column, the unqualified filter column was ambiguous and the query failed to - bind. The filter columns must be qualified with the owning model's CTE. + bind. Qualifying the columns or evaluating the filter before the join both + preserve the owning model's values. """ import tempfile import textwrap @@ -233,9 +234,8 @@ def test_inline_metric_filter_qualified_in_join(): layer.conn = conn layer.graph = graph - # The filter column is qualified to the orders CTE (not the ambiguous bare name). + # Compiling and executing the join must preserve the owning filter scope. sql = layer.compile(metrics=["completed_revenue"], dimensions=["customer.status"]) - assert "orders_cte.status" in sql assert "JOIN" in sql.upper() assert "customers" in sql.lower() diff --git a/tests/adapters/sidemantic_adapter/test_parsing.py b/tests/adapters/sidemantic_adapter/test_parsing.py index 6a2386f65..2dde1fa21 100644 --- a/tests/adapters/sidemantic_adapter/test_parsing.py +++ b/tests/adapters/sidemantic_adapter/test_parsing.py @@ -852,8 +852,16 @@ def test_parse_native_yaml_explicit_key_columns(tmp_path): layer.add_model(model) sql = layer.compile(metrics=["order_items.count"], dimensions=["shipments.carrier"]) - assert "shipments_cte.order_id = order_items_cte.order_id" in sql - assert "shipments_cte.item_id = order_items_cte.item_id" in sql + import sqlglot + from sqlglot import exp + + comparisons = { + frozenset((comparison.left.sql(), comparison.right.sql())) + for join in sqlglot.parse_one(sql).find_all(exp.Join) + for comparison in join.find_all(exp.EQ) + } + assert frozenset(("shipments_cte.order_id", "order_items_cte.order_id")) in comparisons + assert frozenset(("shipments_cte.item_id", "order_items_cte.item_id")) in comparisons def test_parse_native_yaml_resolves_model_and_metric_inheritance(tmp_path): diff --git a/tests/adapters/snowflake/test_conversion.py b/tests/adapters/snowflake/test_conversion.py index 69da51386..c2db56b9d 100644 --- a/tests/adapters/snowflake/test_conversion.py +++ b/tests/adapters/snowflake/test_conversion.py @@ -114,16 +114,10 @@ class TestCubeToSnowflake: def test_convert_cube_to_snowflake(self, snowflake_adapter, cube_adapter, tmp_path): """Test converting a Cube model to Snowflake format.""" - cube_examples = Path(__file__).parent.parent.parent.parent / "examples" / "cube" - - if not cube_examples.exists(): - pytest.skip("Cube examples not found") - # Parse Cube - graph = cube_adapter.parse(cube_examples) - - if not graph.models: - pytest.skip("No Cube models found") + fixture = Path(__file__).parents[2] / "fixtures" / "cube" / "orders.yml" + graph = cube_adapter.parse(fixture) + assert graph.models # Export to Snowflake output_file = tmp_path / "snowflake_output.yaml" @@ -141,16 +135,10 @@ class TestMetricFlowToSnowflake: def test_convert_metricflow_to_snowflake(self, snowflake_adapter, metricflow_adapter, tmp_path): """Test converting a MetricFlow model to Snowflake format.""" - mf_examples = Path(__file__).parent.parent.parent.parent / "examples" / "metricflow" - - if not mf_examples.exists(): - pytest.skip("MetricFlow examples not found") - # Parse MetricFlow - graph = metricflow_adapter.parse(mf_examples) - - if not graph.models: - pytest.skip("No MetricFlow models found") + fixture = Path(__file__).parents[2] / "fixtures" / "metricflow" / "semantic_models.yml" + graph = metricflow_adapter.parse(fixture) + assert graph.models # Export to Snowflake output_file = tmp_path / "snowflake_output.yaml" diff --git a/tests/adapters/tmdl/test_parsing.py b/tests/adapters/tmdl/test_parsing.py index 3bc361684..56aaf2e0d 100644 --- a/tests/adapters/tmdl/test_parsing.py +++ b/tests/adapters/tmdl/test_parsing.py @@ -21,6 +21,7 @@ from sidemantic.core.relationship import Relationship from sidemantic.core.semantic_graph import SemanticGraph from sidemantic.loaders import load_from_directory +from sidemantic.semantic_handoff import UnsupportedSemanticFeaturesError from sidemantic.sql.generator import SQLGenerator from sidemantic.validation import QueryValidationError @@ -119,8 +120,12 @@ def test_tmdl_untranslated_dax_metric_is_not_compiled_as_sql(): layer = SemanticLayer() load_from_directory(layer, "tests/fixtures/tmdl") - with pytest.raises(QueryValidationError, match="DAX expression but has no SQL translation"): + with pytest.raises((QueryValidationError, UnsupportedSemanticFeaturesError)) as exc_info: layer.compile(metrics=["Sales.Sales LY"]) + if isinstance(exc_info.value, UnsupportedSemanticFeaturesError): + assert "metric.dax" in exc_info.value.capabilities + else: + assert "DAX expression but has no SQL translation" in str(exc_info.value) def test_tmdl_untranslated_dax_dimension_is_not_compiled_as_sql(): @@ -131,8 +136,12 @@ def test_tmdl_untranslated_dax_dimension_is_not_compiled_as_sql(): assert amount_x2.sql is None assert amount_x2.has_untranslated_dax - with pytest.raises(QueryValidationError, match="DAX expression but has no SQL translation"): + with pytest.raises((QueryValidationError, UnsupportedSemanticFeaturesError)) as exc_info: layer.compile(metrics=["Sales.Total Sales"], dimensions=["Sales.Amount x2"]) + if isinstance(exc_info.value, UnsupportedSemanticFeaturesError): + assert "dimension.dax" in exc_info.value.capabilities + else: + assert "DAX expression but has no SQL translation" in str(exc_info.value) with pytest.raises(ValueError, match="DAX expression but has no SQL translation"): SQLGenerator(layer.graph).generate(metrics=["Sales.Total Sales"], dimensions=["Sales.Amount x2"]) @@ -161,8 +170,12 @@ def test_tmdl_dax_only_calculated_table_is_not_compiled_as_sql(): layer = SemanticLayer() layer.graph = graph - with pytest.raises(QueryValidationError, match="DAX table expression but has no SQL/table translation"): + with pytest.raises((QueryValidationError, UnsupportedSemanticFeaturesError)) as exc_info: layer.compile(metrics=["SalesByCategory.Revenue"], dimensions=["SalesByCategory.Category"]) + if isinstance(exc_info.value, UnsupportedSemanticFeaturesError): + assert "model.dax" in exc_info.value.capabilities + else: + assert "DAX table expression but has no SQL/table translation" in str(exc_info.value) with pytest.raises(ValueError, match="DAX table expression but has no SQL/table translation"): SQLGenerator(graph).generate(metrics=["SalesByCategory.Revenue"], dimensions=["SalesByCategory.Category"]) @@ -3026,8 +3039,16 @@ def test_tmdl_keyless_many_to_many_joins_without_keying_off_endpoints(): sql = layer.compile(dimensions=["Authors.region", "Books.genre"]) assert "Authors_cte.author_id" in sql assert "Books_cte.book_author_id" in sql - assert "author_id AS author_id" in sql - assert "book_author_id AS book_author_id" in sql + layer.conn.execute("CREATE TABLE Authors (author_id VARCHAR, region VARCHAR)") + layer.conn.execute("CREATE TABLE Books (book_author_id VARCHAR, genre VARCHAR)") + layer.conn.execute("INSERT INTO Authors VALUES ('a', 'US'), ('a', 'EU')") + layer.conn.execute("INSERT INTO Books VALUES ('a', 'fiction'), ('a', 'history')") + assert set(layer.conn.execute(sql).fetchall()) == { + ("US", "fiction"), + ("US", "history"), + ("EU", "fiction"), + ("EU", "history"), + } def test_tmdl_keyless_many_to_many_metric_raises_clear_error(): @@ -3134,8 +3155,11 @@ def test_tmdl_many_to_many_on_non_primary_key_column(): assert re.search(r"ON\s+B_cte\.b_alt\s*=\s*A_cte\.a_alt", sql) or re.search( r"ON\s+A_cte\.a_alt\s*=\s*B_cte\.b_alt", sql ), sql - assert "a_alt AS a_alt" in sql - assert "b_alt AS b_alt" in sql + layer.adapter.execute("create table A(id varchar, a_alt varchar)") + layer.adapter.execute("create table B(id varchar, b_alt varchar, b_label varchar)") + layer.adapter.execute("insert into A values ('a1', 'shared'), ('a2', 'shared')") + layer.adapter.execute("insert into B values ('b1', 'shared', 'matched')") + assert layer.adapter.execute(sql).fetchall() == [("matched", 2)] def test_tmdl_one_to_one_recovers_keyless_source_key(): @@ -3186,8 +3210,12 @@ def test_tmdl_one_to_one_recovers_keyless_source_key(): layer = SemanticLayer() layer.graph = graph sql = layer.compile(metrics=["A.a_count"], dimensions=["B.b_label"]) - assert "a_key AS a_key" in sql assert "A_cte.a_key" in sql + layer.adapter.execute("create table A(a_key varchar)") + layer.adapter.execute("create table B(b_key varchar, b_label varchar)") + layer.adapter.execute("insert into A values ('joined')") + layer.adapter.execute("insert into B values ('joined', 'matched')") + assert layer.adapter.execute(sql).fetchall() == [("matched", 1)] def test_tmdl_one_to_one_alternate_key_does_not_shadow_real_key(): @@ -3252,7 +3280,11 @@ def test_tmdl_one_to_one_alternate_key_does_not_shadow_real_key(): layer.graph = graph sql = layer.compile(metrics=["Orders.cnt"], dimensions=["OrderMeta.channel"]) assert "Orders_cte.alt_key" in sql - assert "alt_key AS alt_key" in sql + layer.adapter.execute("create table Orders(order_id integer, alt_key varchar)") + layer.adapter.execute("create table OrderMeta(meta_key varchar, channel varchar)") + layer.adapter.execute("insert into Orders values (1, 'alternate')") + layer.adapter.execute("insert into OrderMeta values ('alternate', 'web')") + assert layer.adapter.execute(sql).fetchall() == [("web", 1)] def test_tmdl_ambiguous_one_side_target_not_recovered_from_endpoint(): diff --git a/tests/conftest.py b/tests/conftest.py index f66c4d4a0..8135031cf 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -18,6 +18,21 @@ def isolate_cli_color_environment(monkeypatch: pytest.MonkeyPatch): monkeypatch.setattr(typer_rich_utils, "FORCE_TERMINAL", None) +def pytest_addoption(parser): + parser.addoption( + "--test-engine", + choices=("python", "rust"), + default="python", + help="Default engine for the shared suite; explicit engine contract tests retain their overrides", + ) + + +@pytest.fixture(autouse=True) +def selected_test_engine(monkeypatch, request): + """Run shared contracts with the selected engine without enabling fallback.""" + monkeypatch.setenv("SIDEMANTIC_ENGINE", request.config.getoption("--test-engine")) + + @pytest.fixture(autouse=True) def reset_registry(): """Clear the global registry before and after each test. diff --git a/tests/core/test_consumption_compiler.py b/tests/core/test_consumption_compiler.py index 19250453e..94c379f09 100644 --- a/tests/core/test_consumption_compiler.py +++ b/tests/core/test_consumption_compiler.py @@ -102,8 +102,10 @@ def test_explore_qualifies_relative_filter_and_order_expressions(): assert "status <> 'deleted'" in sql assert "status = 'paid'" in sql - assert "status AS status" in sql assert "revenue DESC" in sql + layer.conn.execute("CREATE TABLE orders (order_id INTEGER, status VARCHAR, amount INTEGER)") + layer.conn.execute("INSERT INTO orders VALUES (1, 'paid', 10), (2, 'deleted', 20), (3, 'pending', 30)") + assert layer.conn.execute(sql).fetchall() == [(10,)] def test_explore_filter_qualification_skips_subquery_columns(): diff --git a/tests/core/test_cross_surface_equivalence.py b/tests/core/test_cross_surface_equivalence.py index c61da73f6..59c48cdce 100644 --- a/tests/core/test_cross_surface_equivalence.py +++ b/tests/core/test_cross_surface_equivalence.py @@ -8,17 +8,9 @@ Surfaces covered here (the ones that compile/rewrite SQL from the same graph): SQL-first family (semantic SQL string -> rewritten SQL): - 1. Direct ``QueryRewriter`` -- the exact construction used by - ``sidemantic rewrite`` and ``sidemantic query --dry-run`` (see - ``sidemantic/cli.py``: ``QueryRewriter(layer.graph, - dialect=layer.adapter.dialect, use_preaggregations=layer.use_preaggregations)``). - 2. ``SemanticLayer.sql()`` -- the ``sidemantic query`` execution path. It - rewrites through the identical ``QueryRewriter(self.graph, - dialect=self.dialect, use_preaggregations=self.use_preaggregations)`` call - before executing (``sidemantic/core/semantic_layer.py``). Covered as a - regression tripwire so a future divergence in that construction is caught. - 3. HTTP API ``POST /sql/compile`` -- ``sidemantic/api_server.py`` calls - ``QueryRewriter(current_layer.graph, dialect=current_layer.dialect).rewrite(query)``. + 1. Shared policy-aware transport rewrite, used by CLI rewrite and dry-run. + 2. ``SemanticLayer.sql()`` uses the same transport rewrite before execution. + 3. HTTP ``POST /sql/compile`` uses the same transport rewrite for preview. Structured family (dimensions/metrics -> compiled SQL): 4. ``SemanticLayer.compile(...)`` -- the library/CLI compile entry point. @@ -46,7 +38,7 @@ from sidemantic import Dimension, Metric, Model, Relationship, SemanticLayer from sidemantic.api_server import create_app -from sidemantic.sql.query_rewriter import QueryRewriter +from sidemantic.core.transport_security import rewrite_transport_sql # Representative semantic-SQL queries exercised across the SQL-first surfaces. # Chosen to hit: simple metric aggregation; metric + categorical dimension + @@ -199,32 +191,13 @@ def client(layer: SemanticLayer) -> TestClient: def _rewrite_cli(layer: SemanticLayer, sql: str) -> str: - """Reproduce the exact QueryRewriter construction used by the CLI. - - Mirrors ``sidemantic rewrite`` / ``sidemantic query --dry-run`` in - ``sidemantic/cli.py``. - """ - return QueryRewriter( - layer.graph, - dialect=layer.adapter.dialect, - use_preaggregations=layer.use_preaggregations, - ).rewrite(sql) + """Use the CLI's actual policy-aware, engine-aware rewrite entry point.""" + return rewrite_transport_sql(layer, sql, user_attributes=None, transport="CLI rewrite") def _rewrite_layer_sql_path(layer: SemanticLayer, sql: str) -> str: - """Reproduce the rewrite step inside ``SemanticLayer.sql()``. - - ``SemanticLayer.sql()`` executes rather than returning SQL, so it cannot be - asserted on directly. This mirrors the exact construction it uses so a future - divergence between the ``sql()`` construction and the CLI construction fails - here (regression tripwire). The end-to-end ``sql()`` execution is separately - checked for result equality in ``test_execution_results_match_across_surfaces``. - """ - return QueryRewriter( - layer.graph, - dialect=layer.dialect, - use_preaggregations=layer.use_preaggregations, - ).rewrite(sql) + """Use the rewrite entry point called by SemanticLayer.sql().""" + return rewrite_transport_sql(layer, sql, user_attributes=None, transport="SemanticLayer.sql()") def _fetch_sorted(layer: SemanticLayer, sql: str) -> list[tuple]: @@ -248,7 +221,7 @@ def test_sql_first_rewrite_is_byte_identical(name: str, layer: SemanticLayer, cl assert response.status_code == 200, response.text api_sql = response.json()["sql"] - # Direct QueryRewriter (CLI) == SemanticLayer.sql() rewrite step == HTTP /sql/compile. + # CLI transport rewrite == SemanticLayer.sql() rewrite step == HTTP /sql/compile. assert cli_sql == layer_sql_path_sql, f"{name}: CLI rewrite diverged from SemanticLayer.sql() rewrite" assert cli_sql == api_sql, f"{name}: CLI rewrite diverged from HTTP /sql/compile" diff --git a/tests/core/test_invariant_filters.py b/tests/core/test_invariant_filters.py index 00723207d..849b153b5 100644 --- a/tests/core/test_invariant_filters.py +++ b/tests/core/test_invariant_filters.py @@ -26,8 +26,8 @@ def _orders() -> Model: ) -def _layer() -> SemanticLayer: - layer = SemanticLayer(auto_register=False, engine="python") +def _layer(*, engine=None) -> SemanticLayer: + layer = SemanticLayer(auto_register=False, engine=engine) layer.adapter.execute("CREATE TABLE orders (id INT, customer_id INT, tenant_id INT, status TEXT, amount INT)") layer.adapter.execute( "INSERT INTO orders VALUES (1, 10, 1, 'new', 10), (2, 20, 2, 'new', 100), (3, 10, 1, 'paid', 20)" @@ -51,9 +51,11 @@ def test_structured_compile_and_query_apply_invariant_before_aggregation(): def test_semantic_sql_applies_invariant(): layer = _layer() - assert sorted(layer.sql("SELECT orders.revenue, orders.status FROM orders").fetchall()) == [ - ("new", 10), - ("paid", 20), + result = layer.sql("SELECT orders.revenue, orders.status FROM orders") + assert [column[0] for column in result.description] == ["revenue", "status"] + assert sorted(result.fetchall()) == [ + (10, "new"), + (20, "paid"), ] @@ -126,7 +128,7 @@ def test_preaggregation_bakes_invariant_but_query_keeps_live_scope(): def test_partitioned_preaggregation_scopes_bucket_discovery_and_materialization(): - layer = SemanticLayer(auto_register=False, engine="python") + layer = SemanticLayer(auto_register=False) layer.adapter.execute("CREATE TABLE scoped_orders (created_at TIMESTAMP, tenant_id INT, amount INT)") layer.adapter.execute( "INSERT INTO scoped_orders VALUES ('2024-01-01', 1, 10), ('2024-02-01', 2, 200), ('2024-03-01', 1, 30)" @@ -215,8 +217,8 @@ def test_rust_payload_excludes_unsupported_invariants(): assert "invariant_filters" not in payload -def test_rust_sql_generator_falls_back_for_invariants(monkeypatch): - layer = _layer() +def test_legacy_python_sql_generator_falls_back_for_invariants(monkeypatch): + layer = _layer(engine="python") layer._use_rust_sql_generator = True def unexpected_rust_compile(**_kwargs): diff --git a/tests/core/test_pure_rust_python_test_parity.py b/tests/core/test_pure_rust_python_test_parity.py index 577ab9b50..6a4f2ac8a 100644 --- a/tests/core/test_pure_rust_python_test_parity.py +++ b/tests/core/test_pure_rust_python_test_parity.py @@ -1,4 +1,4 @@ -"""Run existing Python tests against the pure Rust test adapter.""" +"""Run shared contracts with fallback forbidden and direct Rust helper probes.""" from __future__ import annotations @@ -48,6 +48,7 @@ import tests.widget.test_widget_examples as widget_example_tests from tests.rust_layer_adapter import ( RustQueryRewriterAdapter, + RustRuntimeSemanticLayer, RustSemanticGraphDirectAdapter, RustSemanticLayerAdapter, RustSQLGeneratorAdapter, @@ -100,140 +101,6 @@ class PythonTestParityCase: ] -PRE_RUN_XFAILS = { - "tests.optimizations.test_pre_aggregations::test_sql_generation_without_preagg": ( - "Pre-aggregation metadata is not serialized to Rust yet, so this would be a false pass" - ), - "tests.optimizations.test_predicate_pushdown::test_segment_filter_skips_subquery_columns": ( - "Test body asserts a private Python SQLGenerator segment-resolution helper, not Rust SQL behavior" - ), -} - - -EXPECTED_GAPS = { - "tests.test_catalog::test_foreign_key_catalog_uses_canonical_target_for_role_alias": ( - "Rust catalog metadata does not yet preserve a relationship's canonical target_model for role aliases" - ), - "tests.test_validation::test_query_validation_reports_ambiguous_join_routes": ( - "Rust query validation does not yet reject ambiguous join routes" - ), - "tests.core.test_auto_dimensions::test_auto_dimensions_from_table": ( - "Rust adapter does not yet support Python auto-dimension DB introspection" - ), - "tests.core.test_auto_dimensions::test_auto_dimensions_type_mapping": ( - "Rust adapter does not yet support Python auto-dimension DB introspection" - ), - "tests.core.test_auto_dimensions::test_explicit_dimensions_take_precedence": ( - "Rust adapter does not yet support Python auto-dimension DB introspection" - ), - "tests.core.test_auto_dimensions::test_auto_dimensions_sql_model": ( - "Rust adapter does not yet support Python auto-dimension DB introspection" - ), - "tests.core.test_auto_dimensions::test_auto_dimensions_composite_pk": ( - "Rust adapter does not yet support Python auto-dimension DB introspection" - ), - "tests.core.test_auto_dimensions::test_auto_dimensions_query_works": ( - "Rust adapter does not yet support query execution for auto-introspected dimensions" - ), - "tests.core.test_auto_dimensions::test_auto_dimensions_time_granularity_query": ( - "Rust adapter does not yet support query execution for auto-introspected dimensions" - ), - "tests.core.test_auto_dimensions::test_auto_dimensions_non_string_type_metadata": ( - "Rust adapter does not yet support Python auto-dimension DB introspection" - ), - "tests.optimizations.test_pre_aggregations::test_sql_generation_with_preagg": ( - "Rust adapter does not yet support Python pre-aggregation routing" - ), - "tests.optimizations.test_pre_aggregations::test_sql_generation_without_preagg": ( - "Pre-aggregation metadata is not serialized to Rust yet, so this would be a false pass" - ), - "tests.optimizations.test_pre_aggregations::test_preagg_with_filters": ( - "Rust adapter does not yet support Python pre-aggregation routing" - ), - "tests.optimizations.test_pre_aggregations::test_preagg_granularity_conversion": ( - "Rust adapter does not yet support Python pre-aggregation routing" - ), - "tests.optimizations.test_pre_aggregations::test_preagg_per_query_override": ( - "Rust adapter does not yet support Python pre-aggregation routing" - ), - "tests.optimizations.test_pre_aggregations::test_avg_preaggregation_rolls_up_with_sum_count_state": ( - "Rust adapter does not yet support Python pre-aggregation routing" - ), - "tests.optimizations.test_pre_aggregations::test_avg_preaggregation_rejects_missing_count_state": ( - "Rust adapter does not yet support Python pre-aggregation routing" - ), - "tests.optimizations.test_pre_aggregations::test_ratio_metric_preaggregation_rebuilds_from_additive_leaves": ( - "Rust adapter does not yet support Python pre-aggregation routing" - ), - "tests.optimizations.test_pre_aggregations::test_derived_metric_preaggregation_rebuilds_from_additive_leaves": ( - "Rust adapter does not yet support Python pre-aggregation routing" - ), - "tests.optimizations.test_pre_aggregations::test_ratio_metric_preaggregation_rejects_count_distinct_leaf": ( - "Rust adapter does not yet support Python pre-aggregation routing" - ), - "tests.optimizations.test_pre_aggregations::test_ungrouped_composite_pk_partial_rollup_falls_to_raw": ( - "Rust adapter does not yet support Python pre-aggregation routing (ungrouped drill-to-detail)" - ), - "tests.optimizations.test_pre_aggregations::test_ungrouped_avg_metric_bails_to_raw": ( - "Rust adapter does not yet support Python pre-aggregation routing (ungrouped drill-to-detail)" - ), - "tests.optimizations.test_pre_aggregations::test_ungrouped_strict_without_pk_rollup_raises": ( - "Rust adapter does not yet support Python pre-aggregation routing (strict rollup-only mode)" - ), - "tests.optimizations.test_pre_aggregations::test_ungrouped_routes_to_pk_carrying_rollup": ( - "Rust adapter does not yet support Python pre-aggregation routing (ungrouped drill-to-detail)" - ), - "tests.optimizations.test_pre_aggregations::test_ungrouped_preagg_sql_has_no_group_by": ( - "Rust adapter does not yet support Python pre-aggregation routing (ungrouped drill-to-detail)" - ), - "tests.optimizations.test_pre_aggregations::test_ungrouped_explain_reports_pk_rollup_match": ( - "Rust adapter does not yet support Python pre-aggregation routing (ungrouped drill-to-detail)" - ), - "tests.optimizations.test_pre_aggregations::test_ungrouped_rollup_without_pk_falls_to_raw": ( - "Rust adapter does not yet support Python pre-aggregation routing (ungrouped drill-to-detail)" - ), - "tests.optimizations.test_pre_aggregations::test_ungrouped_keyless_model_falls_to_raw": ( - "Rust adapter does not yet support Python pre-aggregation routing (ungrouped drill-to-detail)" - ), - "tests.optimizations.test_pre_aggregations::test_lambda_preaggregation_unions_batch_rollup_with_fresh_source": ( - "Rust adapter does not yet support Python pre-aggregation routing (lambda union)" - ), - "tests.optimizations.test_pre_aggregations::test_lambda_preaggregation_unions_with_granularity_rollup": ( - "Rust adapter does not yet support Python pre-aggregation routing (lambda union)" - ), - "tests.optimizations.test_pre_aggregations::test_lambda_preaggregation_without_build_range_end_is_plain_rollup": ( - "Rust adapter does not yet support Python pre-aggregation routing (lambda union)" - ), - "tests.optimizations.test_preagg_recommender::test_query_instrumentation": ( - "Rust SQL generator does not yet emit Python query instrumentation contracts" - ), - "tests.optimizations.test_preagg_recommender::test_end_to_end_with_semantic_layer": ( - "Rust SQL generator does not yet emit Python query instrumentation contracts" - ), - "tests.metrics.test_cumulative::test_cumulative_with_time_comparison": ( - "Rust strict model validation rejects this Python fixture's time dimension without granularity" - ), - "tests.test_preaggregation_bugs::test_avg_metric_with_filtered_count_fails": ( - "Rust adapter does not yet support Python pre-aggregation routing bug contracts" - ), - "tests.test_preaggregation_bugs::test_filter_on_unmaterialized_dimension": ( - "Rust adapter does not yet support Python pre-aggregation routing bug contracts" - ), - "tests.test_preaggregation_bugs::test_filter_on_unmaterialized_time_grain": ( - "Rust adapter does not yet support Python pre-aggregation routing bug contracts" - ), - "tests.test_preaggregation_bugs::test_week_to_month_granularity_wrong_results": ( - "Rust adapter does not yet support Python pre-aggregation routing bug contracts" - ), - "tests.test_preaggregation_bugs::test_avg_metric_needs_correct_count": ( - "Rust adapter does not yet support Python pre-aggregation routing bug contracts" - ), - "tests.test_sql_generation_security::test_model_ref_rewrite_matches_cte_identifier_quoting": ( - "Rust adapter does not yet support non-DuckDB dialect parity" - ), -} - - DIRECT_RUST_GRAPH_PARITY_NODEIDS = { "tests.joins.test_composite_key_joins::test_semantic_graph_composite_pk_adjacency", "tests.joins.test_composite_key_joins::test_semantic_graph_single_pk_still_works", @@ -359,35 +226,40 @@ def _is_direct_rust_sql_generator_test(nodeid: str, fixture_names: tuple[str, .. @pytest.mark.parametrize("case", PYTHON_LAYER_PARITY_CASES) def test_existing_python_layer_contract_matches_pure_rust(case: PythonTestParityCase, monkeypatch): - """Run the Python test function itself with a Rust-backed test layer.""" - if reason := PRE_RUN_XFAILS.get(case.nodeid): - pytest.xfail(reason) - - _patch_semantic_layer_constructors(monkeypatch, case.module) + """Exercise the public Rust engine; direct helper cases keep their Rust adapters.""" + _patch_semantic_layer_constructors(monkeypatch, case) fixtures: dict[str, object] = {} try: fixtures = _fixture_values(case) _call_python_test_case(case, fixtures) - except NotImplementedError as exc: - if reason := _expected_gap_reason(case.nodeid): - pytest.xfail(f"{reason}: {exc}") - raise - except (Exception, pytest.fail.Exception) as exc: - if reason := _expected_gap_reason(case.nodeid): - pytest.xfail(f"{reason}: {exc}") - raise finally: _close_fixture_values(fixtures) -def _patch_semantic_layer_constructors(monkeypatch, module: ModuleType) -> None: - monkeypatch.setattr(sidemantic, "SemanticLayer", RustSemanticLayerAdapter) +def _patch_semantic_layer_constructors(monkeypatch, case: PythonTestParityCase) -> None: + module = case.module + direct_helper = ( + _is_direct_rust_graph_test(case.nodeid, case.fixture_names) + or _is_direct_rust_sql_generator_test(case.nodeid, case.fixture_names) + or _is_direct_rust_function_test(case.nodeid, case.fixture_names) + ) + layer_class = RustSemanticLayerAdapter if direct_helper else RustRuntimeSemanticLayer + monkeypatch.setattr(sidemantic, "SemanticLayer", layer_class) + monkeypatch.setattr(semantic_layer_module, "SemanticLayer", layer_class) + if hasattr(module, "SemanticLayer"): + monkeypatch.setattr(module, "SemanticLayer", layer_class) + if not direct_helper: + # Some shared bodies instantiate QueryRewriter directly from the layer + # graph; keep those calls on Rust too, not its legacy Python default. + monkeypatch.setenv("SIDEMANTIC_RS_REWRITER", "1") + monkeypatch.setenv("SIDEMANTIC_RS_NO_FALLBACK", "1") + return + + # Keep the direct probes on their original pure-Rust graph/compiler harness. + # Public runtime host services are covered separately above. monkeypatch.setattr(semantic_graph_module, "SemanticGraph", RustSemanticGraphDirectAdapter) - monkeypatch.setattr(semantic_layer_module, "SemanticLayer", RustSemanticLayerAdapter, raising=False) monkeypatch.setattr(sql_generator_module, "SQLGenerator", RustSQLGeneratorAdapter) monkeypatch.setattr(query_rewriter_module, "QueryRewriter", RustQueryRewriterAdapter) - if hasattr(module, "SemanticLayer"): - monkeypatch.setattr(module, "SemanticLayer", RustSemanticLayerAdapter) if hasattr(module, "SemanticGraph"): monkeypatch.setattr(module, "SemanticGraph", RustSemanticGraphDirectAdapter) if hasattr(module, "SQLGenerator"): @@ -404,7 +276,11 @@ def _fixture_values(case: PythonTestParityCase) -> dict[str, object]: values: dict[str, object] = {} for fixture_name in case.fixture_names: if fixture_name == "layer": - values[fixture_name] = RustSemanticLayerAdapter() + if case.module is auto_dimension_tests: + values["db"] = _call_fixture(case.module, "db") + values[fixture_name] = _call_fixture(case.module, "layer", values["db"]) + else: + values[fixture_name] = RustRuntimeSemanticLayer(auto_register=False) elif fixture_name == "test_db": values[fixture_name] = _call_fixture(case.module, "test_db") elif fixture_name == "semantic_layer": @@ -426,11 +302,11 @@ def _fixture_values(case: PythonTestParityCase) -> dict[str, object]: def _call_python_test_case(case: PythonTestParityCase, fixtures: dict[str, object]) -> None: if case.owner is None: - case.func(**fixtures) + case.func(**{name: fixtures[name] for name in case.fixture_names}) return instance = case.owner() - case.func(instance, **fixtures) + case.func(instance, **{name: fixtures[name] for name in case.fixture_names}) def _call_fixture(module: ModuleType, name: str, *args: object) -> object: @@ -462,7 +338,3 @@ def _close_fixture_values(values: dict[str, object]) -> None: close = getattr(value, "close", None) if callable(close): close() - - -def _expected_gap_reason(nodeid: str) -> str | None: - return PRE_RUN_XFAILS.get(nodeid) or EXPECTED_GAPS.get(nodeid) diff --git a/tests/core/test_result_cache.py b/tests/core/test_result_cache.py index 3add9d8f9..afbeb5ce5 100644 --- a/tests/core/test_result_cache.py +++ b/tests/core/test_result_cache.py @@ -5,6 +5,7 @@ from __future__ import annotations import threading +import time import pytest @@ -31,6 +32,15 @@ def _table(nrows: int = 1) -> pa.Table: return pa.table({"a": list(range(nrows))}) +def _wait_for_two_misses(cache: ResultCache) -> None: + # stats() takes the cache lock, so the second miss is visible only after + # that caller has joined the leader's in-flight generation. + deadline = time.monotonic() + 5.0 + while cache.stats()["misses"] < 2: + assert time.monotonic() < deadline, "second caller did not join the in-flight computation" + time.sleep(0.001) + + def test_hit_and_miss_counts(): cache = ResultCache(max_bytes=10 * 1024 * 1024) calls = {"n": 0} @@ -129,7 +139,7 @@ def compute(): calls["n"] += 1 entered.set() # Block so the second thread joins as a waiter before we finish. - hold.wait(timeout=5.0) + assert hold.wait(timeout=10.0) return _table() results: dict[str, pa.Table] = {} @@ -143,12 +153,14 @@ def worker(name: str): t2 = threading.Thread(target=worker, args=("t2",)) t2.start() - # Give t2 a moment to register as a waiter, then release the leader. - hold.set() - - t1.join(timeout=5.0) - t2.join(timeout=5.0) - + try: + _wait_for_two_misses(cache) + finally: + hold.set() + t1.join(timeout=5.0) + t2.join(timeout=5.0) + + assert not t1.is_alive() and not t2.is_alive() assert calls["n"] == 1 assert results["t1"] is results["t2"] @@ -166,7 +178,7 @@ class BoomError(RuntimeError): def compute(): calls["n"] += 1 entered.set() - hold.wait(timeout=5.0) + assert hold.wait(timeout=10.0) raise BoomError("compute failed") errors: dict[str, BaseException] = {} @@ -183,11 +195,14 @@ def worker(name: str): t2 = threading.Thread(target=worker, args=("t2",)) t2.start() - hold.set() - - t1.join(timeout=5.0) - t2.join(timeout=5.0) - + try: + _wait_for_two_misses(cache) + finally: + hold.set() + t1.join(timeout=5.0) + t2.join(timeout=5.0) + + assert not t1.is_alive() and not t2.is_alive() # The in-flight failure propagates to all waiters of that generation. assert calls["n"] == 1 assert isinstance(errors.get("t1"), BoomError) diff --git a/tests/core/test_runtime_defaults.py b/tests/core/test_runtime_defaults.py new file mode 100644 index 000000000..42680af6b --- /dev/null +++ b/tests/core/test_runtime_defaults.py @@ -0,0 +1,89 @@ +"""Default runtime dispatch, with the native boundary supplied by a test double.""" + +import pytest + +import sidemantic.core.semantic_layer as semantic_layer_module +import sidemantic.runtime as runtime +import sidemantic.rust_bridge as rust_bridge +from sidemantic import Metric, Model, SemanticLayer +from sidemantic.config import RuntimeConfig +from sidemantic.semantic_handoff import RustBackendUnavailableError + + +@pytest.fixture(autouse=True) +def normal_runtime_environment(monkeypatch): + monkeypatch.delenv("SIDEMANTIC_ENGINE", raising=False) + for key in list(runtime.os.environ): + if key.startswith("SIDEMANTIC_RS_"): + monkeypatch.delenv(key) + + +class Runtime: + def validate_with_semantic_input(self, *_args): + return [] + + def compile_with_semantic_input(self, *_args): + return "SELECT 42 AS count" + + +def test_default_compiles_with_rust(monkeypatch): + native = Runtime() + monkeypatch.setattr(semantic_layer_module, "get_rust_module", lambda: native) + monkeypatch.setattr(rust_bridge, "get_rust_module", lambda: native) + layer = SemanticLayer() + layer.add_model(Model(name="orders", table="orders", metrics=[Metric(name="count", agg="count")])) + assert "42" in layer.compile(metrics=["orders.count"]) + assert layer.last_engine_selection == {"engine": "rust", "reason": None} + assert RuntimeConfig().engine == "rust" + + +def test_missing_default_runtime_does_not_silently_fall_back(monkeypatch): + def unavailable(): + raise RustBackendUnavailableError("missing native package") + + monkeypatch.setattr(semantic_layer_module, "get_rust_module", unavailable) + with pytest.raises(RustBackendUnavailableError, match="missing native package"): + SemanticLayer() + assert SemanticLayer(engine="python").engine == "python" + assert SemanticLayer(engine="auto")._rust_unavailable_reason == "missing native package" + + +def test_pyodide_defaults_to_python_without_native_import(monkeypatch): + monkeypatch.setattr(runtime.sys, "platform", "emscripten") + assert RuntimeConfig().engine == "python" + assert SemanticLayer().engine == "python" + + +def test_process_override_and_explicit_selection(monkeypatch): + monkeypatch.setenv("SIDEMANTIC_ENGINE", "python") + assert SemanticLayer().engine == "python" + monkeypatch.setattr(semantic_layer_module, "get_rust_module", Runtime) + assert SemanticLayer(engine="rust").engine == "rust" + + +def test_cli_default_and_fallback_override(monkeypatch): + import sidemantic.cli as cli + + monkeypatch.setattr(cli, "_loaded_config", None) + assert cli._resolve_engine_options(None, None) == ("rust", False) + assert cli._resolve_engine_options(None, True) == ("rust", True) + assert cli._resolve_engine_options("auto", None) == ("auto", True) + monkeypatch.setattr(runtime.sys, "platform", "emscripten") + assert cli._resolve_engine_options(None, None) == ("python", False) + + +def test_default_sql_rewrite_uses_rust(monkeypatch): + class RewritingRuntime(Runtime): + def rewrite_with_semantic_input(self, *_args): + return "SELECT 42 AS count" + + def rewrite_with_semantic_input_context(self, *_args): + return "SELECT 42 AS count" + + native = RewritingRuntime() + monkeypatch.setattr(semantic_layer_module, "get_rust_module", lambda: native) + monkeypatch.setattr(rust_bridge, "get_rust_module", lambda: native) + layer = SemanticLayer() + layer.add_model(Model(name="orders", table="orders", metrics=[Metric(name="count", agg="count")])) + assert layer.sql("select count from orders").fetchall() == [(42,)] + assert layer.last_engine_selection == {"engine": "rust", "reason": None} diff --git a/tests/core/test_rust_bridge_yaml_serialization.py b/tests/core/test_rust_bridge_yaml_serialization.py index ad35f7332..e6f181ff2 100644 --- a/tests/core/test_rust_bridge_yaml_serialization.py +++ b/tests/core/test_rust_bridge_yaml_serialization.py @@ -1,5 +1,6 @@ """Regression coverage for Python->Rust YAML bridge serialization fidelity.""" +import pytest import yaml from sidemantic.core.dimension import Dimension @@ -12,6 +13,20 @@ from tests.rust_layer_adapter import _dimension_to_rust_dict, _metric_to_rust_dict, _relationship_to_rust_dict +@pytest.mark.parametrize("window,expected", [(None, "max"), ("min", "min"), ("max", "max")]) +def test_loaded_native_metric_restores_unset_non_additive_window(window, expected): + from sidemantic.rust_bridge import _graph_from_loaded_payload + + graph = _graph_from_loaded_payload( + { + "top_level_metrics": [ + {"name": "revenue", "type": "simple", "agg": "sum", "sql": "amount", "non_additive_window": window} + ] + } + ) + assert graph.metrics["revenue"].non_additive_window == expected + + def test_models_to_rust_yaml_preserves_extended_core_metadata(): model = Model( name="orders", diff --git a/tests/core/test_security_advisor_regressions.py b/tests/core/test_security_advisor_regressions.py index 27fb3bc8a..fc9a53232 100644 --- a/tests/core/test_security_advisor_regressions.py +++ b/tests/core/test_security_advisor_regressions.py @@ -7,6 +7,8 @@ """ import pytest +import sqlglot +from sqlglot import exp from sidemantic import Dimension, Metric, Model, SemanticLayer from sidemantic.core.security import SecurityPolicy, render_row_filter @@ -106,7 +108,10 @@ def test_semi_additive_month_grain_uses_last_snapshot(): assert "CASE WHEN" in normalized_sql assert " = MAX(" in normalized_sql assert " OVER (PARTITION BY " in normalized_sql - assert " ELSE NULL END" in normalized_sql + snapshot_case = sqlglot.parse_one(sql, dialect="duckdb").find(exp.Case) + assert snapshot_case is not None + # An omitted ELSE has the same SQL NULL semantics as an explicit ELSE NULL. + assert snapshot_case.args.get("default") is None or isinstance(snapshot_case.args["default"], exp.Null) # Correct: last day-of-month per account, summed = 110 + 210 = 320 (NOT naive 620). rows = layer.query(metrics=["bal.total_balance"], dimensions=["bal.day__month"]).fetchall() assert len(rows) == 1 diff --git a/tests/core/test_security_enforcement.py b/tests/core/test_security_enforcement.py index 5018c8865..d239d5970 100644 --- a/tests/core/test_security_enforcement.py +++ b/tests/core/test_security_enforcement.py @@ -66,7 +66,7 @@ def _layer(db, **kwargs): adapter = DuckDBAdapter() adapter.conn = db - return SemanticLayer(connection=adapter, auto_register=False, engine="python", **kwargs) + return SemanticLayer(connection=adapter, auto_register=False, **kwargs) def _customers_model(**security_kwargs): @@ -144,7 +144,7 @@ def test_security_block_with_none_attributes_denies(db): # Empty dict is "provided but empty" -> not deny-by-default; here it triggers the # undefined-attribute guard from the row filter instead. - with pytest.raises(SecurityError, match="undefined user attribute"): + with pytest.raises(SecurityError, match="undefined (user attribute|value)"): layer.compile(metrics=["customers.customer_count"], user_attributes={}) diff --git a/tests/core/test_semantic_handoff.py b/tests/core/test_semantic_handoff.py index fa8d3cb50..493ec5a6d 100644 --- a/tests/core/test_semantic_handoff.py +++ b/tests/core/test_semantic_handoff.py @@ -6,6 +6,7 @@ import pytest from sidemantic import Dimension, Metric, Model, PreAggregation, Relationship, SecurityPolicy +from sidemantic.core.inheritance import merge_model from sidemantic.core.semantic_graph import SemanticGraph from sidemantic.core.semantic_layer import SecurityError from sidemantic.rust_bridge import ( @@ -22,6 +23,80 @@ from sidemantic.validation import QueryValidationError +@pytest.mark.parametrize( + "overrides", + [ + {}, + {"primary_key": None, "default_grain": None, "auto_dimensions": False, "metadata": {}, "meta": None}, + {"primary_key": "alternate_id", "unique_keys": []}, + ], +) +def test_unresolved_child_snapshot_preserves_inheritance_override_semantics(overrides): + parent = Model( + name="base", + table="orders", + primary_key="id", + default_grain="month", + auto_dimensions=True, + metadata={"source": "parent"}, + meta={"label": "parent"}, + unique_keys=[["alternate_id"]], + ) + child = Model(name="child", extends="base", **overrides) + graph = SemanticGraph() + graph.add_model(parent) + graph.add_model(child) + snapshot = graph_to_semantic_input(graph)["models"][1] + assert ("primary_key" in snapshot) == ("primary_key" in overrides) + transported = Model.model_validate(snapshot) + expected = merge_model(child, parent) + actual = merge_model(transported, parent) + assert actual.primary_key_columns == expected.primary_key_columns + assert actual.model_dump(exclude={"primary_key"}) == expected.model_dump(exclude={"primary_key"}) + assert child.model_fields_set == {"name", "extends", *overrides} + + +@pytest.mark.parametrize("kind", ["one_to_one", "one_to_many", "many_to_one"]) +@pytest.mark.parametrize("explicit", [None, "explicit_key"]) +def test_snapshot_preserves_tmdl_local_endpoint_without_changing_model_key(kind, explicit): + graph = SemanticGraph() + relationship = Relationship(name="metadata", type=kind, foreign_key="meta_key", primary_key=explicit) + relationship._tmdl_from_column = "alt_key" + graph.add_model(Model(name="orders", table="orders", primary_key="id", relationships=[relationship])) + payload = graph_to_semantic_input(graph) + expected = explicit or ("alt_key" if kind in ("one_to_one", "one_to_many") else None) + assert payload["models"][0]["relationships"][0].get("primary_key") == expected + assert payload["models"][0]["primary_key"] == ["id"] + assert relationship.primary_key == explicit + + +@pytest.mark.parametrize( + "definition", + [ + {"type": "conversion", "entity": "id", "base_event": "signup", "conversion_event": "purchase"}, + {"type": "time_comparison", "base_metric": "events.count"}, + ], +) +def test_snapshot_does_not_duplicate_automatically_indexed_model_metrics(definition): + graph = SemanticGraph() + metric = Metric(name="special", **definition) + graph.add_model(Model(name="events", table="events", metrics=[metric])) + payload = graph_to_semantic_input(graph) + assert payload["metrics"] == [] + assert [item["name"] for item in payload["models"][0]["metrics"]] == ["special"] + assert graph.metrics["special"] is metric + + +def test_snapshot_keeps_explicit_graph_metric_even_when_model_uses_same_object(): + graph = SemanticGraph() + metric = Metric(name="revenue", agg="sum", sql="amount") + graph.add_model(Model(name="orders", table="orders", metrics=[metric])) + graph.add_metric(metric, model_name="orders") + payload = graph_to_semantic_input(graph) + assert [item["name"] for item in payload["metrics"]] == ["revenue"] + assert payload["metric_owners"] == {"revenue": "orders"} + + @pytest.fixture def source_graph(): graph = SemanticGraph() @@ -354,3 +429,20 @@ def test_postgres_nested_window_null_order_survives_intermediate_dialect(): assert _postgres_query_sql(sql) == ( "SELECT x FROM events ORDER BY SUM(x) OVER (ORDER BY y DESC NULLS FIRST) ASC NULLS LAST" ) + + +@pytest.mark.parametrize( + "literal,expected", + [(r"E'O\'Brien'", "O'Brien"), (r"E'back\\slash'", "back\\slash"), (r"E'line\nbreak'", "line\nbreak")], +) +def test_postgres_escape_strings_preserve_values_as_ordinary_literals(literal, expected): + import duckdb + import sqlglot + from sqlglot import exp + + from sidemantic.rust_bridge import _postgres_query_sql + + sql = _postgres_query_sql(f"SELECT {literal} AS value") + assert not list(sqlglot.parse_one(sql, read="duckdb").find_all(exp.ByteString)) + with duckdb.connect() as connection: + assert connection.execute(sql).fetchone() == (expected,) diff --git a/tests/db/test_adbc_adapter.py b/tests/db/test_adbc_adapter.py index 405b9323a..02f237e1a 100644 --- a/tests/db/test_adbc_adapter.py +++ b/tests/db/test_adbc_adapter.py @@ -164,17 +164,13 @@ def test_adbc_adapter_close(sqlite_adapter): sqlite_adapter.execute("SELECT 1") -@pytest.mark.skipif(True, reason="Requires pyarrow (optional dependency)") def test_adbc_adapter_fetch_record_batch(sqlite_adapter): - """Test fetching Arrow RecordBatch. - - Skipped by default since pyarrow is optional. - """ + """Test fetching Arrow values when the optional dependencies are installed.""" pytest.importorskip("pyarrow") result = sqlite_adapter.execute("SELECT 1 as x, 2 as y") batch = sqlite_adapter.fetch_record_batch(result) # Should return Arrow RecordBatchReader - assert batch is not None + assert batch.read_all().to_pydict() == {"x": [1], "y": [2]} def test_adbc_adapter_injection_attempt_in_table_name_is_rejected(sqlite_adapter): @@ -227,6 +223,30 @@ def test_adbc_adapter_from_url(): adapter.close() +@pytest.mark.parametrize("url", ["sqlite:///:memory:", "adbc://sqlite/:memory:"]) +def test_adbc_url_resolves_packaged_driver_without_system_install(monkeypatch, url): + import adbc_driver_manager.dbapi as adbc + + sqlite_package = pytest.importorskip("adbc_driver_sqlite") + from sidemantic.db.adbc import ADBCAdapter + + connect = adbc.connect + + def connect_with_bundled_driver(**kwargs): + # Require the wheel's absolute library path even on hosts where a + # system-installed SQLite driver would mask broken URL resolution. + assert kwargs["driver"] == sqlite_package._driver_path() + return connect(**kwargs) + + monkeypatch.setattr(adbc, "connect", connect_with_bundled_driver) + adapter = ADBCAdapter.from_url(url) + try: + assert adapter.dialect == "sqlite" + assert adapter.execute("select 42").fetchone() == (42,) + finally: + adapter.close() + + def test_adbc_adapter_from_url_invalid_scheme(): """Test that invalid URL scheme raises ValueError.""" from sidemantic.db.adbc import ADBCAdapter diff --git a/tests/db/test_duckdb_adapter.py b/tests/db/test_duckdb_adapter.py index f4663bdff..36062243b 100644 --- a/tests/db/test_duckdb_adapter.py +++ b/tests/db/test_duckdb_adapter.py @@ -99,18 +99,14 @@ def test_duckdb_adapter_close(): adapter.execute("SELECT 1") -@pytest.mark.skipif(True, reason="Requires pyarrow (optional dependency)") def test_duckdb_adapter_fetch_record_batch(): - """Test fetching Arrow RecordBatch. - - Skipped by default since pyarrow is optional. - """ + """Test fetching Arrow values when the optional dependency is installed.""" pytest.importorskip("pyarrow") adapter = DuckDBAdapter() result = adapter.execute("SELECT 1 as x, 2 as y") batch = adapter.fetch_record_batch(result) # Should return Arrow RecordBatchReader - assert batch is not None + assert batch.read_all().to_pydict() == {"x": [1], "y": [2]} def test_duckdb_absolute_file_paths(): diff --git a/tests/malloy_differential/sidemantic_runner.py b/tests/malloy_differential/sidemantic_runner.py index 0c9be99eb..5c112ece6 100644 --- a/tests/malloy_differential/sidemantic_runner.py +++ b/tests/malloy_differential/sidemantic_runner.py @@ -59,7 +59,7 @@ def _run_query(workspace: Path, source_file: Path, query: dict[str, Any]) -> dic layer = SemanticLayer( connection=f"duckdb:///{workspace / 'fixture.duckdb'}", auto_register=False, - engine="python", + fallback=False, ) # Exercise the normal public registration path so intrinsic physical # dimensions are discovered before the differential query is validated. diff --git a/tests/metrics/test_filters.py b/tests/metrics/test_filters.py index 273006aa9..b35a62fb2 100644 --- a/tests/metrics/test_filters.py +++ b/tests/metrics/test_filters.py @@ -521,7 +521,7 @@ def test_structured_filters_resolve_grained_and_computed_dimensions_before_where where_sql = postgres_sql.split("WHERE", 1)[1] assert "created_at__month" not in where_sql assert "events.gross" not in where_sql - assert "DATE_TRUNC('MONTH', occurred_at)" in where_sql + assert "DATE_TRUNC('MONTH', OCCURRED_AT)" in where_sql.upper() assert "(unit_price * quantity) >= 20" in where_sql assert df_rows( diff --git a/tests/metrics/test_non_additive_guard.py b/tests/metrics/test_non_additive_guard.py index d24c85f57..03f1b6877 100644 --- a/tests/metrics/test_non_additive_guard.py +++ b/tests/metrics/test_non_additive_guard.py @@ -18,6 +18,7 @@ from sidemantic import Dimension, Metric, Model, Relationship, SemanticLayer from sidemantic.adapters.sidemantic import SidemanticAdapter from sidemantic.core.semantic_layer import UnsupportedMetricError +from sidemantic.semantic_handoff import UnsupportedSemanticFeaturesError from sidemantic.sql.generator import SQLGenerator @@ -92,7 +93,7 @@ def test_semi_additive_value_is_last_snapshot(): # Semi-additive: sum of the last snapshot per account (global last-date window # collapses to the single latest snapshot when no other grouping is requested). sql = layer.compile(metrics=["accounts.balance"]) - assert "__sidemantic_snapshot_field" in sql + assert "MAX(" in sql assert "OVER (" in sql # Grouped by account: last balance per account, summed -> 150 + 70 + 33 = 253. @@ -242,11 +243,15 @@ def test_semi_additive_plus_fanout_symmetric_aggregate_raises(): relationships=[Relationship(name="accounts", type="many_to_one", foreign_key="account_id")], ) ) - with pytest.raises(UnsupportedMetricError) as exc: + expected_error = UnsupportedSemanticFeaturesError if layer.engine == "rust" else UnsupportedMetricError + with pytest.raises(expected_error) as exc: layer.compile(metrics=["accounts.balance", "transactions.amount"], dimensions=["accounts.region"]) - msg = str(exc.value).lower() - assert "symmetric" in msg or "fan-out" in msg - assert "compose" in msg + if layer.engine == "rust": + assert exc.value.capabilities == ["metric.non_additive_metric_shape"] + else: + msg = str(exc.value).lower() + assert "symmetric" in msg or "fan-out" in msg + assert "compose" in msg def test_adapter_round_trips_non_additive_dimension(): @@ -362,7 +367,7 @@ def test_graph_metric_wrapping_semi_additive_measure_is_planned(): ) layer.add_metric(Metric(name="wrapped_balance", sql="bal.total_balance")) sql = layer.compile(metrics=["wrapped_balance"], dimensions=["bal.account"]) - assert "__sidemantic_snapshot_field" in sql + assert "MAX(" in sql assert dict(layer.query(metrics=["wrapped_balance"], dimensions=["bal.account"]).fetchall()) == {"A": 110, "B": 210} diff --git a/tests/optimizations/test_pre_aggregations.py b/tests/optimizations/test_pre_aggregations.py index 06d3a6d81..c2564a7dd 100644 --- a/tests/optimizations/test_pre_aggregations.py +++ b/tests/optimizations/test_pre_aggregations.py @@ -4,6 +4,8 @@ import duckdb import pytest +import sqlglot +from sqlglot import exp from sidemantic import Dimension, Metric, Model from sidemantic.core.pre_aggregation import Index, PreAggregation, RefreshKey, RefreshResult @@ -1208,7 +1210,10 @@ def test_avg_preaggregation_rolls_up_with_sum_count_state(layer): preagg_rows = layer.adapter.execute(preagg_sql).fetchall() assert "products_preagg_by_category" in preagg_sql - assert "SUM(avg_price_raw) / NULLIF(SUM(count_raw), 0)" in preagg_sql + expression = sqlglot.parse_one(preagg_sql) + assert expression.find(exp.Div) is not None + assert expression.find(exp.Nullif) is not None + assert {aggregate.this.name for aggregate in expression.find_all(exp.Sum)} == {"avg_price_raw", "count_raw"} assert preagg_rows == baseline_rows @@ -1296,7 +1301,11 @@ def test_ratio_metric_preaggregation_rebuilds_from_additive_leaves(layer): preagg_rows = layer.adapter.execute(preagg_sql).fetchall() assert "orders_preagg_by_status" in preagg_sql - assert "SUM(revenue_raw) / NULLIF(COALESCE(SUM(count_raw), 0), 0)" in preagg_sql + expression = sqlglot.parse_one(preagg_sql) + assert expression.find(exp.Div) is not None + assert expression.find(exp.Nullif) is not None + assert expression.find(exp.Coalesce) is not None + assert {aggregate.this.name for aggregate in expression.find_all(exp.Sum)} == {"revenue_raw", "count_raw"} assert preagg_rows == baseline_rows @@ -1426,7 +1435,8 @@ def test_lambda_preaggregation_unions_with_granularity_rollup(layer): preagg_rows = layer.adapter.execute(preagg_sql).fetchall() assert "UNION ALL" in preagg_sql - assert "DATE_TRUNC('MONTH', created_at_day)" in preagg_sql + month_bucket = sqlglot.parse_one("DATE_TRUNC('month', created_at_day)", read="duckdb") + assert any(expression == month_bucket for expression in sqlglot.parse_one(preagg_sql, read="duckdb").walk()) assert preagg_rows == baseline_rows @@ -1511,7 +1521,9 @@ def test_derived_metric_preaggregation_rebuilds_from_additive_leaves(layer): preagg_rows = layer.adapter.execute(preagg_sql).fetchall() assert "orders_preagg_by_status" in preagg_sql - assert "SUM(revenue_raw) - SUM(discounts_raw)" in preagg_sql + expression = sqlglot.parse_one(preagg_sql) + assert expression.find(exp.Sub) is not None + assert {aggregate.this.name for aggregate in expression.find_all(exp.Sum)} == {"revenue_raw", "discounts_raw"} assert preagg_rows == baseline_rows @@ -2148,16 +2160,23 @@ def test_preagg_strict_raises_when_table_missing(): ) -def test_sql_path_falls_back_to_raw_when_rollup_missing(): +@pytest.mark.parametrize( + "projection,columns,rows", + [ + ("orders.revenue, orders.status", ["revenue", "status"], {(120, 0), (90, 1)}), + ("orders.status, orders.revenue", ["status", "revenue"], {(0, 120), (1, 90)}), + ], +) +def test_sql_path_falls_back_to_raw_when_rollup_missing(projection, columns, rows): """layer.sql() (the SQL/CLI path) also falls back to raw when the rollup table is missing.""" layer = _layer_with_unbuilt_rollup() layer.use_preaggregations = True - result = layer.sql("SELECT orders.revenue, orders.status FROM orders") + result = layer.sql(f"SELECT {projection} FROM orders") # The rollup table is absent, so rows come back only if it fell back to raw. - # The rewriter orders columns dimensions-first, so each row is (status, revenue). - assert set(result.fetchall()) == {(0, 120), (1, 90)} + assert [column[0] for column in result.description] == columns + assert set(result.fetchall()) == rows def test_sql_path_strict_raises_when_rollup_missing(): diff --git a/tests/optimizations/test_predicate_pushdown.py b/tests/optimizations/test_predicate_pushdown.py index d6e144f67..9896c3ecd 100644 --- a/tests/optimizations/test_predicate_pushdown.py +++ b/tests/optimizations/test_predicate_pushdown.py @@ -4,7 +4,6 @@ from sqlglot import exp from sidemantic import Dimension, Metric, Model, Segment -from sidemantic.sql.generator import SQLGenerator def test_single_model_filter_pushdown(layer): @@ -300,14 +299,12 @@ def test_segment_filter_skips_subquery_columns(layer): layer.add_model(model) - generator = SQLGenerator(layer.graph) - filters = generator._resolve_segments(["orders.in_other"]) - assert len(filters) == 1 - - filter_sql = filters[0] - parsed = sqlglot.parse_one(filter_sql) + sql = layer.compile(metrics=["orders.count"], segments=["orders.in_other"]) + parsed = sqlglot.parse_one(sql) - assert any(col.table == "orders_cte" for col in parsed.find_all(exp.Column)) + membership = parsed.find(exp.In) + assert membership is not None + assert membership.this.name == "id" subquery = None for subquery_def in parsed.find_all(exp.Subquery): @@ -319,6 +316,14 @@ def test_segment_filter_skips_subquery_columns(layer): for col in subquery.find_all(exp.Column): assert not col.table + # Executing the compiled query also catches accidental correlation of the + # subquery's id to the outer orders table, which changes which rows match. + layer.conn.execute("create table orders_table (id integer)") + layer.conn.execute("insert into orders_table values (1), (2)") + layer.conn.execute("create table other_table (id integer, flag varchar)") + layer.conn.execute("insert into other_table values (1, 'y'), (2, 'n')") + assert layer.conn.execute(sql).fetchall() == [(1,)] + def test_metric_level_filters_not_pushed(layer): """Test that metric-level filters are applied via CASE WHEN, not pushed to CTE. @@ -740,6 +745,18 @@ def test_mixed_metric_and_window_dim_filter_pushed_into_model_subquery_in_preagg layer.add_model(order_items) # Filter references BOTH a window dim (next_status) and a metric (revenue) + layer.conn.execute(""" + create table orders_table ( + order_id integer, customer_id integer, status varchar, + created_at timestamp, order_date date, amount double + ); + insert into orders_table values + (1, 1, 'pending', '2025-01-01 09:00:00', '2025-01-01', 50), + (2, 1, 'complete', '2025-01-01 10:00:00', '2025-01-01', 200), + (3, 2, 'pending', '2025-01-01 11:00:00', '2025-01-01', 5); + create table order_items_table (item_id integer, order_id integer, qty integer); + insert into order_items_table values (1, 1, 1), (2, 2, 2), (3, 3, 3); + """) sql = layer.compile( metrics=["orders.revenue", "order_items.quantity"], dimensions=["orders.order_date"], @@ -759,3 +776,12 @@ def test_mixed_metric_and_window_dim_filter_pushed_into_model_subquery_in_preagg # The outer query should NOT have the window dim filter outer_query = sql[sql.rindex("SELECT") :] assert "next_status" not in outer_query, "Window dim filter should NOT be in outer WHERE" + + # Order 1 qualifies through its next status; order 2 qualifies through its + # own amount. Order 3 is excluded from revenue, while all item rows still + # contribute to the independent quantity population. + cursor = layer.conn.execute(sql) + assert [column[0] for column in cursor.description] == ["order_date", "revenue", "quantity"] + rows = cursor.fetchall() + assert len(rows) == 1 + assert rows[0][1:] == (250.0, 6) diff --git a/tests/queries/test_basic.py b/tests/queries/test_basic.py index 0dd26cb8f..cb6a5675d 100644 --- a/tests/queries/test_basic.py +++ b/tests/queries/test_basic.py @@ -511,7 +511,6 @@ def test_custom_join_sql_projects_extra_predicate_columns(): ) sql = layer.compile(metrics=["orders.revenue"], dimensions=["customers.country"], order_by=["customers.country"]) - assert "valid_to AS valid_to" in sql assert "customers_cte.valid_to IS NULL" in sql rows = df_rows( @@ -600,7 +599,6 @@ def test_dotted_graph_metric_projects_sql_column_and_orders_by_alias(layer): order_by=["events.p95.latency DESC"], ) - assert "latency AS latency" in sql assert "ORDER BY" in sql assert '"events.p95.latency" DESC' in sql diff --git a/tests/queries/test_non_additive_option_transport.py b/tests/queries/test_non_additive_option_transport.py new file mode 100644 index 000000000..f12e1f312 --- /dev/null +++ b/tests/queries/test_non_additive_option_transport.py @@ -0,0 +1,78 @@ +"""Explicit snapshot escape-hatch preferences reach every compilation entrypoint.""" + +import json + +import pytest + +from sidemantic import Dimension, Metric, Model, SemanticLayer +from sidemantic.rust_bridge import rewrite_semantic_input, validate_semantic_input + + +@pytest.fixture +def layer(monkeypatch): + monkeypatch.setattr("sidemantic.core.semantic_layer.get_rust_module", lambda: object()) + result = SemanticLayer(engine="rust", fallback=False, allow_non_additive_unsafe=True, auto_register=False) + result.add_model( + Model( + name="accounts", + table="accounts", + dimensions=[Dimension(name="day", type="time", granularity="day")], + metrics=[Metric(name="balance", agg="sum", sql="amount", non_additive_dimension="day")], + ) + ) + yield result + result.adapter.close() + + +@pytest.mark.parametrize("method", ["compile", "query", "sql", "explain_sql"]) +def test_layer_passes_live_snapshot_preference(monkeypatch, layer, method): + calls = [] + + def compile(graph, query, **kwargs): + calls.append(query.get("allow_non_additive_unsafe", False)) + return "SELECT 42 AS balance" + + def rewrite(graph, query, **kwargs): + calls.append(kwargs.get("allow_non_additive_unsafe", False)) + return "SELECT 42 AS balance" + + monkeypatch.setattr("sidemantic.rust_bridge.compile_semantic_input", compile) + monkeypatch.setattr("sidemantic.rust_bridge.validate_semantic_input", lambda *args, **kwargs: []) + monkeypatch.setattr("sidemantic.sql.query_rewriter.rewrite_semantic_input", rewrite) + for enabled in [True, False]: + layer.allow_non_additive_unsafe = enabled + if method in {"compile", "query"}: + result = getattr(layer, method)(metrics=["accounts.balance"]) + else: + result = getattr(layer, method)("SELECT balance FROM accounts") + if method in {"query", "sql"}: + assert result.fetchall() == [(42,)] + assert calls[-1] is enabled + assert calls == [True, False] + + +@pytest.mark.parametrize("method", ["validate", "rewrite"]) +def test_bridge_serializes_snapshot_preference(layer, method): + calls = [] + + class Extension: + def validate_with_semantic_input(self, source, query): + calls.append(json.loads(query)) + return [] + + def rewrite_with_semantic_input_context_diagnostics(self, source, sql, context): + calls.append(json.loads(context)) + return json.dumps({"sql": "SELECT 42", "warnings": []}) + + if method == "validate": + validate_semantic_input( + layer.graph, ["accounts.balance"], [], allow_non_additive_unsafe=True, rust_module=Extension() + ) + else: + rewrite_semantic_input( + layer.graph, + "SELECT balance FROM accounts", + allow_non_additive_unsafe=True, + rust_module=Extension(), + ) + assert calls[0]["allow_non_additive_unsafe"] is True diff --git a/tests/queries/test_post_process_comments.py b/tests/queries/test_post_process_comments.py new file mode 100644 index 000000000..fb5f3a17d --- /dev/null +++ b/tests/queries/test_post_process_comments.py @@ -0,0 +1,26 @@ +"""Embedding compiled SQL must terminate trailing line comments.""" + +import pytest + +from sidemantic import SemanticLayer + + +@pytest.mark.parametrize( + "suffix", + [ + "\n-- used_preagg=true", + "\n-- used_preagg=true\n-- sidemantic: models=orders", + " -- source comment", + "\r\n-- source comment", + ], +) +def test_post_process_preserves_query_before_trailing_comment(suffix): + layer = SemanticLayer(auto_register=False) + try: + sql = layer._apply_post_process( + "SELECT '-- literal text' AS label, 41 AS value" + suffix, + "SELECT label, value + 1 AS result FROM ({inner}) AS scoped", + ) + assert layer.adapter.execute(sql).fetchall() == [("-- literal text", 42)] + finally: + layer.adapter.close() diff --git a/tests/queries/test_relationship_key_validation.py b/tests/queries/test_relationship_key_validation.py new file mode 100644 index 000000000..f1bdb8350 --- /dev/null +++ b/tests/queries/test_relationship_key_validation.py @@ -0,0 +1,34 @@ +"""Declared local relationship keys remain usable as grouping fields.""" + +import pytest + +from sidemantic import Metric, Model, Relationship, SemanticLayer +from sidemantic.validation import QueryValidationError + + +@pytest.mark.parametrize("engine", ["python", "rust"]) +def test_declared_foreign_key_groups_without_dimension(engine): + if engine == "rust": + pytest.importorskip("sidemantic_rs") + layer = SemanticLayer(engine=engine, fallback=False, auto_register=False) + layer.add_model(Model(name="customers", table="customers", primary_key="id")) + layer.add_model( + Model( + name="orders", + table="orders", + primary_key="id", + metrics=[Metric(name="revenue", agg="sum", sql="amount")], + relationships=[Relationship(name="customers", type="many_to_one", foreign_key="customer_id")], + ) + ) + try: + layer.adapter.execute("create table orders(id integer, customer_id integer, amount integer)") + layer.adapter.execute("insert into orders values (1, 10, 2), (2, 10, 3), (3, 20, 7)") + sql = layer.compile( + metrics=["orders.revenue"], dimensions=["orders.customer_id"], order_by=["orders.customer_id"] + ) + assert layer.adapter.execute(sql).fetchall() == [(10, 5), (20, 7)] + with pytest.raises(QueryValidationError, match="unknown_key"): + layer.compile(metrics=["orders.revenue"], dimensions=["orders.unknown_key"]) + finally: + layer.adapter.close() diff --git a/tests/queries/test_rust_rewrite_preaggregation_options.py b/tests/queries/test_rust_rewrite_preaggregation_options.py new file mode 100644 index 000000000..310916b0a --- /dev/null +++ b/tests/queries/test_rust_rewrite_preaggregation_options.py @@ -0,0 +1,62 @@ +"""Preaggregation preferences cross the SQL rewrite boundary without fallback.""" + +import json + +import pytest + +from sidemantic import Dimension, Metric, Model +from sidemantic.core.semantic_graph import SemanticGraph +from sidemantic.rust_bridge import rewrite_semantic_input +from sidemantic.sql.query_rewriter import QueryRewriter + + +def graph(): + result = SemanticGraph() + result.add_model( + Model( + name="orders", + table="raw_orders", + dimensions=[Dimension(name="status", type="categorical")], + metrics=[Metric(name="revenue", agg="sum", sql="amount")], + ) + ) + return result + + +@pytest.mark.parametrize("method", ["rewrite", "explain"]) +@pytest.mark.parametrize( + "sql", + [ + "SELECT revenue FROM orders", + "WITH orders_cte AS (SELECT revenue FROM orders) SELECT * FROM orders_cte", + ], +) +def test_rollup_preference_reaches_native_compiler(monkeypatch, method, sql): + calls = [] + + def rewrite(graph, sql, **options): + calls.append(options) + return "SELECT 42 AS revenue" + + monkeypatch.setattr("sidemantic.sql.query_rewriter.rewrite_semantic_input", rewrite) + rewriter = QueryRewriter(graph(), use_rust_rewriter=True, rust_no_fallback=True, use_preaggregations=True) + result = getattr(rewriter, method)(sql) + assert (result if method == "rewrite" else result.rewritten_sql) == "SELECT 42 AS revenue" + assert calls[0]["use_preaggregations"] is True + assert rewriter.last_engine_selection == {"engine": "rust", "reason": None} + + +@pytest.mark.parametrize("enabled", [False, True]) +def test_bridge_encodes_rollup_preference_in_context(enabled): + calls = [] + + class Extension: + def rewrite_with_semantic_input_context_diagnostics(self, source, sql, context): + calls.append(json.loads(context)) + return json.dumps({"sql": "SELECT 42 AS revenue", "warnings": []}) + + result = rewrite_semantic_input( + graph(), "SELECT revenue FROM orders", use_preaggregations=enabled, rust_module=Extension() + ) + assert result == "SELECT 42 AS revenue" + assert calls[0].get("use_preaggregations", False) is enabled diff --git a/tests/queries/test_semantic_sql_planner.py b/tests/queries/test_semantic_sql_planner.py index f63870dd0..e319be704 100644 --- a/tests/queries/test_semantic_sql_planner.py +++ b/tests/queries/test_semantic_sql_planner.py @@ -1,4 +1,9 @@ -"""Tests for semantic SQL rewrite planning and explanations.""" +"""Python optimizer explanations with engine-selected SQL execution checks. + +Plan names and rules describe the Python optimizer implementation. Queries sent +through SemanticLayer execute with the suite's selected engine, and their rows +are compared with the independently compiled Python baseline. +""" import pytest @@ -14,7 +19,7 @@ @pytest.fixture def semantic_layer(): - layer = SemanticLayer(auto_register=False) + layer = SemanticLayer(auto_register=False, fallback=False) orders = Model( name="orders", @@ -91,9 +96,22 @@ def _compiled_semantic_sql(layer: SemanticLayer, sql: str, use_preaggregations: layer.graph, dialect="duckdb", use_preaggregations=use_preaggregations, + use_rust_rewriter=False, ).rewrite(sql) +def _python_plan(layer: SemanticLayer, sql: str): + """Inspect Python optimizer internals separately from runtime execution.""" + return QueryRewriter( + layer.graph, + dialect=layer.dialect, + use_preaggregations=layer.use_preaggregations, + enforce_visibility=layer.enforce_visibility, + use_rust_rewriter=False, + allow_non_additive_unsafe=layer.allow_non_additive_unsafe, + ).explain(sql) + + def _sorted_rows(rows): return sorted(rows, key=repr) @@ -104,8 +122,9 @@ def _assert_query_matches_baseline( baseline_sql: str, ordered: bool = False, ): - explanation = layer.explain_sql(sql) + explanation = _python_plan(layer, sql) optimized_result = layer.sql(sql) + assert layer.last_engine_selection == {"engine": layer.engine, "reason": None} optimized_columns = fetch_columns(optimized_result) optimized_rows = fetch_rows(optimized_result) baseline_result = layer.conn.execute(baseline_sql) @@ -539,7 +558,7 @@ def test_bi_corpus_rejection_matrix_records_expected_reasons(semantic_layer, nam orders = semantic_layer.get_model("orders") orders.metrics.append(Metric(name="unique_customers", agg="count_distinct", sql="customer_id")) - explanation = semantic_layer.explain_sql(sql) + explanation = _python_plan(semantic_layer, sql) assert explanation.chosen_plan == "semantic_plus_postprocess" assert rejected_rule in explanation.rejected_rules @@ -634,7 +653,7 @@ def test_external_cte_chain_corpus_flattens_linear_steps(semantic_layer): ], ) def test_linear_cte_chain_rejects_unsafe_steps(semantic_layer, sql): - explanation = semantic_layer.explain_sql(sql) + explanation = _python_plan(semantic_layer, sql) assert explanation.chosen_plan == "semantic_plus_postprocess" assert explanation.post_process is not None @@ -997,7 +1016,7 @@ def test_semantic_island_with_outer_dependency_is_rejected(semantic_layer): ) """ - explanation = semantic_layer.explain_sql(sql) + explanation = _python_plan(semantic_layer, sql) assert explanation.chosen_plan == "semantic_plus_postprocess" assert explanation.semantic_islands == [] @@ -1048,7 +1067,7 @@ def test_root_semantic_query_allows_redundant_group_by_dimensions(semantic_layer """ direct_sql = "SELECT orders.revenue, orders.status FROM orders" - explanation = semantic_layer.explain_sql(sql) + explanation = _python_plan(semantic_layer, sql) assert explanation.chosen_plan == "direct_semantic" assert explanation.metrics == ["orders.revenue"] @@ -1063,7 +1082,7 @@ def test_root_semantic_query_preserves_having_metric_filter(semantic_layer): HAVING orders.revenue > 225 """ - explanation = semantic_layer.explain_sql(sql) + explanation = _python_plan(semantic_layer, sql) rows = _rows(semantic_layer.sql(sql)) assert explanation.chosen_plan == "direct_semantic" @@ -1085,7 +1104,7 @@ def test_root_unqualified_dimension_filter_is_qualified_for_pushdown(semantic_la WHERE status = 'completed' """ - explanation = semantic_layer.explain_sql(unqualified_sql) + explanation = _python_plan(semantic_layer, unqualified_sql) assert explanation.chosen_plan == "direct_semantic" assert explanation.filters == ["orders.status = 'completed'"] @@ -1458,8 +1477,8 @@ def test_aggregate_boundary_rejects_non_additive_subtotals(semantic_layer): GROUP BY status """ - ratio_explanation = semantic_layer.explain_sql(ratio_sql) - median_explanation = semantic_layer.explain_sql(median_sql) + ratio_explanation = _python_plan(semantic_layer, ratio_sql) + median_explanation = _python_plan(semantic_layer, median_sql) assert ratio_explanation.chosen_plan == "semantic_plus_postprocess" assert ratio_explanation.rejected_rules["aggregate_boundary_rollup"] == "outer_projection_computes_expression" @@ -1587,7 +1606,7 @@ def test_aggregate_boundary_time_grain_rollup_rejects_week_to_month(semantic_lay ], ) def test_aggregate_boundary_time_grain_rollup_rejects_unsafe_dimension_expressions(semantic_layer, sql, reason): - explanation = semantic_layer.explain_sql(sql) + explanation = _python_plan(semantic_layer, sql) assert explanation.chosen_plan == "semantic_plus_postprocess" assert explanation.rejected_rules["aggregate_boundary_rollup"] == reason @@ -1738,7 +1757,7 @@ def test_conditional_aggregate_pivot_with_outer_row_filter_uses_preaggregation(s ], ) def test_conditional_aggregate_pivot_rejects_unsafe_shapes(semantic_layer, sql, reason): - explanation = semantic_layer.explain_sql(sql) + explanation = _python_plan(semantic_layer, sql) assert explanation.chosen_plan == "semantic_plus_postprocess" assert "conditional_aggregate_wrapper" not in explanation.applied_rules @@ -2100,7 +2119,7 @@ def test_wrapped_preaggregation_executes_against_materialized_table(semantic_lay ORDER BY status """ - explanation = semantic_layer.explain_sql(wrapped_sql) + explanation = _python_plan(semantic_layer, wrapped_sql) preagg_rows = _rows(semantic_layer.sql(wrapped_sql)) base_rows = _rows( semantic_layer.query( @@ -2278,8 +2297,11 @@ def test_wrapped_fanout_preserves_aliases_and_executes(semantic_layer): "orders.revenue": "total_revenue", "customers.count": "customer_count", } - assert "orders_preagg.total_revenue AS total_revenue" in explanation.rewritten_sql - assert "customers_preagg.customer_count AS customer_count" in explanation.rewritten_sql + assert fetch_columns(semantic_layer.adapter.execute(explanation.rewritten_sql)) == [ + "total_revenue", + "customer_count", + ] + assert fetch_rows(semantic_layer.adapter.execute(explanation.rewritten_sql)) == [(450, 2)] def test_wrapped_fanout_uses_child_preaggregations(semantic_layer): @@ -2497,7 +2519,7 @@ def test_fanout_join_key_preaggregation_rejects_missing_join_key_rollup(semantic semantic_layer.use_preaggregations = True sql = "SELECT orders.revenue, customers.region FROM orders" - explanation = semantic_layer.explain_sql(sql) + explanation = _python_plan(semantic_layer, sql) candidates = _candidate_by_name(explanation) assert explanation.chosen_plan == "direct_semantic" @@ -2517,7 +2539,7 @@ def test_fanout_join_key_preaggregation_rejects_one_to_many_remote_dimension(sem semantic_layer.use_preaggregations = True sql = "SELECT customers.count, orders.status FROM customers" - explanation = semantic_layer.explain_sql(sql) + explanation = _python_plan(semantic_layer, sql) candidates = _candidate_by_name(explanation) assert explanation.chosen_plan == "direct_semantic" @@ -2923,7 +2945,7 @@ def test_global_topn_rejects_rank_tie_semantics(semantic_layer, window_fn, expec ORDER BY revenue DESC """ - explanation = semantic_layer.explain_sql(sql) + explanation = _python_plan(semantic_layer, sql) assert explanation.chosen_plan == "semantic_plus_postprocess" assert explanation.rejected_rules["global_row_number_topn"] == expected_reason @@ -2941,7 +2963,7 @@ def test_global_topn_rejects_outer_projection_of_rank_column(semantic_layer): ORDER BY revenue DESC """ - explanation = semantic_layer.explain_sql(sql) + explanation = _python_plan(semantic_layer, sql) assert explanation.chosen_plan == "semantic_plus_postprocess" assert ( @@ -3093,7 +3115,7 @@ def test_set_operation_branch_order_by_is_rejected(semantic_layer): SELECT orders.revenue, orders.status FROM orders """ - explanation = semantic_layer.explain_sql(sql) + explanation = _python_plan(semantic_layer, sql) assert explanation.chosen_plan == "passthrough_plain_sql" assert explanation.semantic_islands == [] @@ -3125,7 +3147,7 @@ def test_global_row_number_topn_rejects_partitioned_rank(semantic_layer): def test_projection_width_reduction_omits_unused_primary_key(semantic_layer): sql = "SELECT orders.revenue, orders.status FROM orders" - explanation = semantic_layer.explain_sql(sql) + explanation = _python_plan(semantic_layer, sql) assert explanation.chosen_plan == "direct_semantic" assert "id AS id" not in explanation.rewritten_sql @@ -3137,7 +3159,7 @@ def test_projection_width_reduction_omits_unused_primary_key(semantic_layer): def test_projection_width_reduction_keeps_join_keys_for_joined_dimensions(semantic_layer): sql = "SELECT orders.revenue, customers.region FROM orders" - explanation = semantic_layer.explain_sql(sql) + explanation = _python_plan(semantic_layer, sql) assert explanation.chosen_plan == "direct_semantic" assert "customer_id AS customer_id" in explanation.rewritten_sql @@ -3152,7 +3174,7 @@ def test_projection_width_reduction_keeps_count_distinct_key_state(semantic_laye orders.metrics.append(Metric(name="unique_orders", agg="count_distinct")) sql = "SELECT orders.unique_orders, orders.status FROM orders ORDER BY orders.status" - explanation = semantic_layer.explain_sql(sql) + explanation = _python_plan(semantic_layer, sql) rows = _rows(semantic_layer.sql(sql)) assert explanation.chosen_plan == "direct_semantic" @@ -3208,9 +3230,13 @@ def test_explain_passthrough_plain_sql_non_strict(semantic_layer): def test_semantic_layer_explain_sql(semantic_layer): explanation = semantic_layer.explain_sql("SELECT orders.revenue FROM orders") - assert explanation.chosen_plan == "direct_semantic" - assert explanation.metrics == ["orders.revenue"] - assert "orders_cte" in explanation.rewritten_sql + assert semantic_layer.last_engine_selection == {"engine": semantic_layer.engine, "reason": None} + assert semantic_layer.conn.execute(explanation.rewritten_sql).fetchall() == [(450,)] + if semantic_layer.engine == "rust": + assert explanation.chosen_plan == "rust_semantic_rewriter" + else: + assert explanation.chosen_plan == "direct_semantic" + assert explanation.metrics == ["orders.revenue"] def test_rewrite_simple_query_still_generates_same_sql(semantic_layer): diff --git a/tests/queries/test_sql_projection_order.py b/tests/queries/test_sql_projection_order.py new file mode 100644 index 000000000..f781c4844 --- /dev/null +++ b/tests/queries/test_sql_projection_order.py @@ -0,0 +1,35 @@ +"""SQL projections retain their requested order across raw and rollup plans.""" + +import pytest + +from sidemantic import Dimension, Metric, Model, PreAggregation, SemanticLayer + + +@pytest.mark.parametrize("engine", ["python", "rust"]) +@pytest.mark.parametrize("rollup", [False, True]) +@pytest.mark.parametrize( + "wrapper", ["{query}", "SELECT * FROM ({query}) AS totals", "WITH totals AS ({query}) SELECT * FROM totals"] +) +def test_metric_first_projection_keeps_aliases_grouping_and_order(engine, rollup, wrapper): + layer = SemanticLayer(engine=engine, fallback=False, auto_register=False, use_preaggregations=rollup) + try: + layer.adapter.execute("CREATE TABLE orders(status VARCHAR, amount INTEGER)") + layer.adapter.execute("INSERT INTO orders VALUES ('b', 2), ('a', 3), ('a', 4)") + layer.add_model( + Model( + name="orders", + table="orders", + dimensions=[Dimension(name="status", type="categorical")], + metrics=[Metric(name="revenue", agg="sum", sql="amount")], + pre_aggregations=[PreAggregation(name="status", dimensions=["status"], measures=["revenue"])], + ) + ) + layer.adapter.execute( + "CREATE TABLE orders_preagg_status AS SELECT status, SUM(amount) AS revenue_raw FROM orders GROUP BY status" + ) + query = wrapper.format(query='SELECT revenue AS "Total", status AS "Status" FROM orders') + ' ORDER BY "Status"' + result = layer.sql(query) + assert [column[0] for column in result.description] == ["Total", "Status"] + assert result.fetchall() == [(7, "a"), (2, "b")] + finally: + layer.adapter.close() diff --git a/tests/queries/test_sql_rewriter.py b/tests/queries/test_sql_rewriter.py index 2bd2ac005..520fd0fb5 100644 --- a/tests/queries/test_sql_rewriter.py +++ b/tests/queries/test_sql_rewriter.py @@ -1,6 +1,7 @@ """Tests for SQL query rewriter.""" import pytest +import sqlglot from sidemantic.core.dimension import Dimension from sidemantic.core.metric import Metric @@ -185,8 +186,9 @@ def test_zero_limit_and_offset_are_preserved(semantic_layer): sql = "SELECT orders.revenue, orders.status FROM orders ORDER BY orders.status LIMIT 0 OFFSET 0" rewritten = QueryRewriter(semantic_layer.graph).rewrite(sql) - assert "\nLIMIT 0" in rewritten - assert "\nOFFSET 0" in rewritten + parsed = sqlglot.parse_one(rewritten) + assert parsed.args["limit"].expression.this == "0" + assert parsed.args["offset"].expression.this == "0" result = semantic_layer.sql(sql) rows = _rows(result) @@ -1478,8 +1480,9 @@ def test_postprocess_zero_limit_and_offset_in_outer(semantic_layer): """ rewritten = QueryRewriter(semantic_layer.graph).rewrite(sql) - assert "\nLIMIT 0" in rewritten - assert "\nOFFSET 0" in rewritten + parsed = sqlglot.parse_one(rewritten) + assert parsed.args["limit"].expression.this == "0" + assert parsed.args["offset"].expression.this == "0" result = semantic_layer.sql(sql) rows = _rows(result) diff --git a/tests/queries/test_ungrouped_queries.py b/tests/queries/test_ungrouped_queries.py index d4f512f19..25bb9b640 100644 --- a/tests/queries/test_ungrouped_queries.py +++ b/tests/queries/test_ungrouped_queries.py @@ -5,6 +5,7 @@ import pytest from sidemantic import Dimension, Metric, Model, SemanticLayer +from sidemantic.semantic_handoff import UnsupportedSemanticFeaturesError from tests.utils import fetch_dicts @@ -360,7 +361,7 @@ def test_with_totals_ignores_configured_default_limit(): def test_with_totals_unsupported_window_path_raises(layer): - """with_totals on a window-function (cumulative) metric raises NotImplementedError.""" + """with_totals on a window-function metric explicitly rejects the unsupported path.""" orders = Model( name="orders", table="orders", @@ -371,5 +372,9 @@ def test_with_totals_unsupported_window_path_raises(layer): layer.add_model(orders) layer.graph.add_metric(Metric(name="cumulative_revenue", type="cumulative", sql="orders.revenue")) - with pytest.raises(NotImplementedError, match="with_totals is not yet supported"): + with pytest.raises((NotImplementedError, UnsupportedSemanticFeaturesError)) as exc_info: layer.compile(metrics=["cumulative_revenue"], dimensions=["orders.order_date"], with_totals=True) + if isinstance(exc_info.value, UnsupportedSemanticFeaturesError): + assert exc_info.value.capabilities == ["query.totals.window"] + else: + assert "with_totals is not yet supported" in str(exc_info.value) diff --git a/tests/queries/test_yardstick_query_rewriter.py b/tests/queries/test_yardstick_query_rewriter.py index 6e7986cf9..a352d8b8d 100644 --- a/tests/queries/test_yardstick_query_rewriter.py +++ b/tests/queries/test_yardstick_query_rewriter.py @@ -413,13 +413,14 @@ def test_yardstick_plain_measure_reference_with_where_context(yardstick_layer): ] -def test_yardstick_curly_measure_reference_without_semantic_prefix(yardstick_layer): +@pytest.mark.parametrize("measure", ["{revenue}", "{sales_v.revenue}"]) +def test_yardstick_curly_measure_reference_without_semantic_prefix(yardstick_layer, measure): rows = fetch_dicts( yardstick_layer.sql( - """ + f""" SELECT year, - {revenue} AS revenue + {measure} AS revenue FROM sales_v WHERE region = 'US' GROUP BY year @@ -457,10 +458,11 @@ def test_yardstick_mixed_non_semantic_at_routing(yardstick_layer): ] -def test_yardstick_listing8_rollup_parity(yardstick_paper_layer): +@pytest.mark.parametrize("grouping", ["ROLLUP(o.prodName)", "CUBE(o.prodName)", "GROUPING SETS ((o.prodName), ())"]) +def test_yardstick_listing8_rollup_parity(yardstick_paper_layer, grouping): rows = fetch_dicts( yardstick_paper_layer.sql( - """ + f""" SELECT o.prodName, COUNT(*) AS c, @@ -469,7 +471,7 @@ def test_yardstick_listing8_rollup_parity(yardstick_paper_layer): o.sumRevenue AS r FROM paper_orders_v o WHERE o.custName <> 'Var Bob' -GROUP BY ROLLUP(o.prodName) +GROUP BY {grouping} ORDER BY o.prodName """ ) @@ -1903,6 +1905,8 @@ def test_yardstick_ordered_set_aggregates(tmp_path): category, PERCENTILE_CONT(0.5) WITHIN GROUP (ORDER BY value) AS MEASURE p50, PERCENTILE_DISC(0.5) WITHIN GROUP (ORDER BY value) AS MEASURE p50d, + PERCENTILE_CONT(0.25) WITHIN GROUP (ORDER BY value DESC) FILTER (WHERE value > 1) AS MEASURE p25_desc, + PERCENTILE_DISC(0.25) WITHIN GROUP (ORDER BY value DESC) FILTER (WHERE value > 1) AS MEASURE p25d_desc, QUANTILE_CONT(value, 0.5) AS MEASURE q50, QUANTILE_DISC(value, 0.5) AS MEASURE q50d, MODE(value) AS MEASURE mode_value @@ -1927,6 +1931,12 @@ def test_yardstick_ordered_set_aggregates(tmp_path): p50d = fetch_dicts(layer.sql("SEMANTIC SELECT category, AGGREGATE(p50d) AS p50d FROM ordered_set_v")) assert {(row["category"], int(row["p50d"])) for row in p50d} == {("A", 2), ("B", 10)} + p25_desc = fetch_dicts(layer.sql("SEMANTIC SELECT category, AGGREGATE(p25_desc) AS p25_desc FROM ordered_set_v")) + assert {(row["category"], float(row["p25_desc"])) for row in p25_desc} == {("A", 3.5), ("B", 15.0)} + + p25d_desc = fetch_dicts(layer.sql("SEMANTIC SELECT category, AGGREGATE(p25d_desc) AS p25d_desc FROM ordered_set_v")) + assert {(row["category"], int(row["p25d_desc"])) for row in p25d_desc} == {("A", 4), ("B", 20)} + q50 = fetch_dicts(layer.sql("SEMANTIC SELECT category, AGGREGATE(q50) AS q50 FROM ordered_set_v")) assert {(row["category"], float(row["q50"])) for row in q50} == {("A", 2.0), ("B", 10.0)} diff --git a/tests/rust_layer_adapter.py b/tests/rust_layer_adapter.py index 8f048c7af..a16673ade 100644 --- a/tests/rust_layer_adapter.py +++ b/tests/rust_layer_adapter.py @@ -25,6 +25,28 @@ RUST_ADAPTER_BIN = RUST_TARGET_DIR / "debug" / "examples" / f"parity_adapter{'.exe' if os.name == 'nt' else ''}" +class RustRuntimeSemanticLayer(ProductionSemanticLayer): + """Public runtime with an executable guard against Python compilation. + + Host responsibilities such as DB introspection and result execution use the + production layer. Query compilation and validation must use Rust, even + when the surrounding shared-suite run selects the Python engine. + """ + + def __init__(self, *args, **kwargs): + kwargs["engine"] = "rust" + kwargs["fallback"] = False + super().__init__(*args, **kwargs) + + def _compile_with_python(self, *args, **kwargs): + raise AssertionError("Rust parity contract attempted Python compilation") + + def compile(self, *args, **kwargs): + sql = super().compile(*args, **kwargs) + assert self.last_engine_selection == {"engine": "rust", "reason": None} + return sql + + class RustSemanticLayerAdapter: """Minimal SemanticLayer-compatible test adapter backed by sidemantic-rs. diff --git a/tests/semantic_conformance/README.md b/tests/semantic_conformance/README.md index dfff97733..3bb4af38c 100644 --- a/tests/semantic_conformance/README.md +++ b/tests/semantic_conformance/README.md @@ -32,6 +32,36 @@ It also covers metric filters, query filters, grouped NULL populations, and empty grouped and scalar aggregate populations. Rust cases require the real extension. Joined populations, derived and temporal calculations, cohort aggregates, snapshots, stored rollup routing, metric-result filters, ungrouped queries, and -other output dialects remain capability-gated. Existing rollup definitions can -be bypassed with `use_preaggregations=False`; stored scalar estimates are never -summed or treated as mergeable sketches. +other output dialects have their own result or rejection cases in the suite. +Stored scalar estimates must never be summed or treated as mergeable sketches. + +## Engine selection and acceptance + +The main CI workflow runs the shared suite with `--test-engine python` and +`--test-engine rust`. The Rust selection does not enable fallback. Security, +invariant, Malloy, and planner execution tests follow that selection. Python +optimizer rule names remain implementation tests; planner result comparisons +execute through the selected engine against an independent Python baseline. +The legacy shared-contract harness additionally forbids Python compilation +and checks Rust selection. Its direct Rust graph/compiler probes remain separate. + +A successful shared run is not evidence that every optional test executed: + +- PostgreSQL conformance cases require `SIDEMANTIC_TEST_POSTGRES_DSN`; the native + CI job supplies a PostgreSQL service and runs this suite with the built extension. +- Native binding and validation tests may skip their unused Python parameter. + The Rust parameter must execute. Such skips are not missing Rust coverage. +- SQLite ADBC tests use an installed driver; CI provides `adbc-driver-sqlite`. + Arrow tests execute when the optional Arrow dependency is present. +- Widget/workbench dependencies and online DAX/Yardstick services are separate + host or external-service requirements, not semantic-compiler substitutions. +- The default `not integration` selection excludes database/CLI and online + Power BI tests. The configured PostgreSQL, BigQuery, Snowflake, ClickHouse, + and ADBC integration jobs run both engines. Tests requiring other live + services still need their documented environment and credentials. + +Unsupported inputs must have explicit rejection assertions, not broad expected +failures. For example, MetricFlow millisecond dimensions and hourly +`grain_to_date` declarations are rejected during import. Retention grouping, +aggregate-result filters, and derived wrappers are not silently accepted as +working query shapes. diff --git a/tests/semantic_conformance/test_absent_counts.py b/tests/semantic_conformance/test_absent_counts.py index 6e4c29615..34d0c6048 100644 --- a/tests/semantic_conformance/test_absent_counts.py +++ b/tests/semantic_conformance/test_absent_counts.py @@ -105,9 +105,7 @@ def test_count_zero_flows_into_derived_and_ratio_before_outer_defaults(layer): assert cursor.fetchall() == [("empty", 1, 0, -2), ("matched", 3, 2, 0.5)] -def test_native_source_populations_preserve_orphans_with_zero_customer_count(layer): - if layer.engine != "rust": - pytest.skip("Native independent sources retain orphan groups omitted by Python's dimension-first joins") +def test_source_populations_preserve_orphans_with_zero_customer_count(layer): cursor = layer.adapter.execute( layer.compile( metrics=["orders.rows", "orders.people", "customers.customers", "combined", "fraction", "reverse_fraction"], @@ -134,6 +132,31 @@ def test_policy_does_not_restore_unauthorized_groups(layer): assert cursor.fetchall() == [("empty", 0, 1)] +def test_three_source_groups_merge_when_the_first_source_has_no_row(layer): + layer.add_model( + Model( + name="refunds", + table="count_refunds", + primary_key="id", + metrics=[Metric(name="refunds", agg="count")], + relationships=[Relationship(name="customers", type="many_to_one", foreign_key="customer_id")], + ) + ) + layer.adapter.execute( + "create table count_refunds(id integer, customer_id integer); insert into count_refunds values (1, 2)" + ) + cursor = layer.adapter.execute( + layer.compile( + metrics=["orders.rows", "customers.customers", "refunds.refunds"], + dimensions=["customers.region"], + order_by=["customers.region"], + ) + ) + # The empty group exists in the second and third sources, but not the first. + # Joining every later source only against the first would split it into two rows. + assert cursor.fetchall() == [("empty", 0, 1, 1), ("matched", 2, 1, 0), (None, 1, 0, 0)] + + def test_absent_count_zero_is_used_by_aggregate_filter_and_order(layer): cursor = layer.adapter.execute( layer.compile( diff --git a/tests/semantic_conformance/test_aggregate_aliases.py b/tests/semantic_conformance/test_aggregate_aliases.py index 6bfafaa28..3a69ec842 100644 --- a/tests/semantic_conformance/test_aggregate_aliases.py +++ b/tests/semantic_conformance/test_aggregate_aliases.py @@ -6,10 +6,11 @@ from sidemantic.semantic_handoff import UnsupportedSemanticFeaturesError -@pytest.fixture -def layer(): - pytest.importorskip("sidemantic_rs", reason="Alias acceptance requires the real Rust extension") - layer = SemanticLayer(engine="rust", auto_register=False) +@pytest.fixture(params=["python", "rust"]) +def layer(request): + if request.param == "rust": + pytest.importorskip("sidemantic_rs", reason="Alias acceptance requires the real Rust extension") + layer = SemanticLayer(engine=request.param, auto_register=False) layer.add_model( Model( name="customers", @@ -51,7 +52,7 @@ def layer(): def assert_result(layer, query, columns, expected): sql = layer.compile(**query) - assert layer.last_engine_selection["engine"] == "rust" + assert layer.last_engine_selection["engine"] == layer.engine result = layer.adapter.execute(sql) assert [column[0] for column in result.description] == columns assert result.fetchall() == expected diff --git a/tests/semantic_conformance/test_aggregate_breadth_parity.py b/tests/semantic_conformance/test_aggregate_breadth_parity.py index b535176cb..667dec1d3 100644 --- a/tests/semantic_conformance/test_aggregate_breadth_parity.py +++ b/tests/semantic_conformance/test_aggregate_breadth_parity.py @@ -80,6 +80,19 @@ def test_complete_sql_and_wrappers_run_after_entity_dedup(layer): assert layer.adapter.execute(sql).fetchall() == [("all", 30.0, 3, 40.0, 60.0, 30.0)] +def test_complete_aggregates_keep_groups_without_source_entities(layer): + layer.adapter.execute("insert into raw_items values (5, 99, 'orphan'), (6, 99, 'orphan')") + cursor = layer.query( + metrics=["orders.opaque_count", "orders.average"], + dimensions=["items.category"], + order_by=["items.category"], + ) + assert [field[0] for field in cursor.description] == ["category", "opaque_count", "average"] + # Authored COUNT(*) counts the deduplicated null-extended SQL row. Ordinary + # semantic counts instead count source entities and return zero here. + assert cursor.fetchall() == [("all", 3, 30.0), ("orphan", 1, None)] + + def test_complete_and_advanced_totals_deduplicate_overlapping_entity_groups(layer): layer.adapter.execute("insert into raw_items values (5, 1, 'overlap'), (6, 2, null), (7, 2, null)") cursor = layer.query( diff --git a/tests/semantic_conformance/test_approximate_distinct.py b/tests/semantic_conformance/test_approximate_distinct.py index d73c02800..da60a5110 100644 --- a/tests/semantic_conformance/test_approximate_distinct.py +++ b/tests/semantic_conformance/test_approximate_distinct.py @@ -98,7 +98,12 @@ def test_joined_populations(layer, relationship_type): Model(name="users", table="users", primary_key="id", dimensions=[Dimension(name="region", type="categorical")]) ) layer.graph.get_model("events").relationships.append( - Relationship(name="users", type=relationship_type, sql="user_id", foreign_key="id") + Relationship( + name="users", + type=relationship_type, + foreign_key="user_id" if relationship_type == "many_to_one" else "id", + primary_key="id" if relationship_type == "many_to_one" else "user_id", + ) ) sql = layer.compile(metrics=["events.users"], dimensions=["users.region"], filters=["events.user_id is not null"]) expected = layer.adapter.execute(""" diff --git a/tests/semantic_conformance/test_cohort_parity.py b/tests/semantic_conformance/test_cohort_parity.py index 21842a598..8af6dcf96 100644 --- a/tests/semantic_conformance/test_cohort_parity.py +++ b/tests/semantic_conformance/test_cohort_parity.py @@ -4,7 +4,7 @@ import pytest -from sidemantic import Dimension, Explore, Metric, Model, SecurityPolicy, SemanticLayer +from sidemantic import Dimension, Explore, Metric, Model, Relationship, SecurityPolicy, SemanticLayer from sidemantic.core.semantic_layer import SecurityError from sidemantic.semantic_handoff import graph_to_semantic_input @@ -116,6 +116,14 @@ def test_query_grouping_changes_inner_and_outer_grain(layer): ) +def test_cohort_order_preserves_explicit_null_placement(layer): + layer.adapter.execute("update events set region = null where user_id = 'u3'") + assert result(layer, dimensions=["events.region"], order_by=["events.region DESC NULLS FIRST"]) == ( + ["region", "qualified"], + [(None, 1), ("US", 1), ("EU", 1)], + ) + + def test_time_bucket_is_carried_through_to_outer_output(layer): assert result(layer, dimensions=["events.day__month"], order_by=["events.day__month"]) == ( ["day__month", "qualified"], @@ -304,18 +312,13 @@ def test_nonexistent_explicit_owner_does_not_fall_back_to_entity_match(layer): @pytest.mark.parametrize("layer", ["rust_owned"], indirect=True) -@pytest.mark.parametrize( - "mutation", ["unowned", "unknown_metric", "filled_ignored_offset", "wrapper", "joined_dimension"] -) +@pytest.mark.parametrize("mutation", ["unowned", "unknown_metric", "wrapper", "joined_dimension"]) def test_owned_graph_cohort_unsupported_shapes_remain_gated(layer, mutation): query = {} if mutation == "unowned": layer.graph.metric_owners.clear() elif mutation == "unknown_metric": layer.graph.metric_owners["ghost"] = "events" - elif mutation == "filled_ignored_offset": - cohort_metric(layer).fill_nulls_with = 0 - cohort_metric(layer).time_offset = "1 day" elif mutation == "wrapper": layer.graph.add_metric(Metric(name="wrapped", type="derived", sql="qualified * 2")) query["metrics"] = ["wrapped"] @@ -328,11 +331,29 @@ def test_owned_graph_cohort_unsupported_shapes_remain_gated(layer, mutation): dimensions=[Dimension(name="region", type="categorical")], ) ) + layer.graph.models["events"].relationships.append( + Relationship(name="other", type="many_to_one", foreign_key="user_id") + ) query["dimensions"] = ["other.region"] with pytest.raises(ValueError): result(layer, **query) +@pytest.mark.parametrize("layer", ["python_owned", "rust_owned"], indirect=True) +def test_owned_graph_cohort_default_and_ignored_offset_preserve_population(layer): + metric = cohort_metric(layer) + metric.agg = "sum" + metric.sql = "amount" + metric.fill_nulls_with = -9 + metric.time_offset = "1 day" + assert result(layer) == (["qualified"], [(103,)]) + assert result(layer, user_attributes={"tenant": 99}) == (["qualified"], [(-9,)]) + assert result(layer, dimensions=["events.region"], order_by=["events.region"]) == ( + ["region", "qualified"], + [("EU", 73), ("US", 30)], + ) + + @pytest.mark.parametrize("layer", ["python_owned", "rust_owned"], indirect=True) def test_owned_graph_cohort_visibility_is_enforced(layer): cohort_metric(layer).public = False @@ -397,10 +418,8 @@ def test_selected_calculations_preserve_implicit_entity_dimensions(layer): payload = {**query, "metrics": [cohort_reference(layer)], "table_calculations": [c.name for c in calculations]} assert ( - json.loads( - sidemantic_rs.validate_with_semantic_input( - json.dumps(graph_to_semantic_input(layer.graph)), json.dumps(payload) - ) + sidemantic_rs.validate_with_semantic_input( + json.dumps(graph_to_semantic_input(layer.graph)), json.dumps(payload) ) == [] ) diff --git a/tests/semantic_conformance/test_complete_filters.py b/tests/semantic_conformance/test_complete_filters.py index e00bfd873..e95d110b7 100644 --- a/tests/semantic_conformance/test_complete_filters.py +++ b/tests/semantic_conformance/test_complete_filters.py @@ -133,22 +133,11 @@ def test_filter_conjunction_preserves_disjunction_grouping(layer): "COUNT(CAST(NULL AS INTEGER))", "COUNT((SELECT 1))", "SUM(1)", - "SUM(CAST(amount AS DOUBLE))", - "SUM(amount / 2)", - "COUNT(COALESCE(amount, '{model}'))", - "COUNT(CASE WHEN amount > 0 THEN 'prefix{model}.value' ELSE 'other' END)", "SUM(COALESCE(other.amount, amount))", "SUM(CASE WHEN amount > 0 THEN other.amount ELSE 0 END)", - "SUM(COALESCE(SUM(amount), 0))", - "SUM(amount) + COUNT(amount)", - "SUM(amount) OVER ()", "SUM((SELECT amount))", "SUM(other.amount)", - "COUNT(DISTINCT ABS(amount))", - "AVG(DISTINCT amount)", - "AVG(amount) OVER ()", "AVG(other.amount)", - "COUNT(*) FILTER (WHERE amount > 0)", "COUNT(*) OVER ()", "COUNT(other.*)", ], @@ -172,6 +161,76 @@ def test_unproven_complete_filter_shapes_fail_explicitly(expression): layer.adapter.close() +@pytest.mark.parametrize("engine", ["python", "rust"]) +@pytest.mark.parametrize( + "expression, expected", + [ + ("SUM(CAST(amount AS DOUBLE))", [(8.0,)]), + ("SUM(amount / 2)", [(4.0,)]), + ("SUM(amount) + COUNT(amount)", [(11,)]), + ("COUNT(DISTINCT ABS(amount))", [(2,)]), + ("AVG(DISTINCT amount)", [(3.0,)]), + ("COUNT(*) FILTER (WHERE amount > 0)", [(3,)]), + # Complete expressions filter their physical inputs before evaluating + # the formula, including its CASE fallback and window cardinality. + ("COUNT(CASE WHEN amount > 0 THEN 'prefix{model}.value' ELSE 'other' END)", [(5,)]), + ("SUM(amount) OVER ()", [(8,)] * 5), + ("AVG(amount) OVER ()", [(pytest.approx(8 / 3),)] * 5), + ], +) +def test_complete_filter_formulas_execute_with_physical_inputs(engine, expression, expected): + if engine == "rust": + pytest.importorskip("sidemantic_rs") + layer = SemanticLayer(engine=engine, fallback=False, auto_register=False) + try: + layer.add_model( + Model( + name="orders", + table="orders", + primary_key="id", + metrics=[Metric(name="value", sql=expression, sql_is_complete=True, filters=["amount > 0"])], + ) + ) + layer.adapter.execute(""" + create table orders(id integer, amount integer); + insert into orders values (1, 2), (2, 2), (3, 4), (4, -9), (5, null); + """) + assert_result(layer, {"metrics": ["orders.value"]}, ["value"], expected) + finally: + layer.adapter.close() + + +@pytest.mark.parametrize("engine", ["python", "rust"]) +@pytest.mark.parametrize( + "expression, error", + [ + ("SUM(COALESCE(SUM(amount), 0))", "aggregate function calls cannot be nested"), + ("COUNT(COALESCE(amount, '{model}'))", "Could not convert string"), + ], +) +def test_invalid_complete_sql_keeps_database_validation(engine, expression, error): + import duckdb + + if engine == "rust": + pytest.importorskip("sidemantic_rs") + layer = SemanticLayer(engine=engine, fallback=False, auto_register=False) + try: + layer.add_model( + Model( + name="orders", + table="orders", + primary_key="id", + metrics=[Metric(name="value", sql=expression, sql_is_complete=True, filters=["amount > 0"])], + ) + ) + layer.adapter.execute("create table orders(id integer, amount integer); insert into orders values (1, null)") + sql = layer.compile(metrics=["orders.value"]) + with pytest.raises(duckdb.Error, match=error): + layer.adapter.execute(sql) + finally: + layer.adapter.close() + + def test_foreign_filter_population_fails_explicitly(): pytest.importorskip("sidemantic_rs", reason="Complete filter rejection requires the real Rust extension") layer = SemanticLayer(engine="rust", fallback=False, auto_register=False) diff --git a/tests/semantic_conformance/test_computed_keys.py b/tests/semantic_conformance/test_computed_keys.py index 02dce21ff..9338a0121 100644 --- a/tests/semantic_conformance/test_computed_keys.py +++ b/tests/semantic_conformance/test_computed_keys.py @@ -107,10 +107,11 @@ def test_from_metrics_sql_uses_structured_key_planning(layer): assert data.fetchall() == [(101, 7), (201, 17)] -def test_legacy_sql_shape_is_explicitly_unsupported(layer): - with pytest.raises(UnsupportedSemanticFeaturesError) as caught: - rewrite_semantic_input(layer.graph, "select accounts.id, accounts.budget from accounts") - assert "rewrite.computed_key_query_shape" in caught.value.capabilities +def test_model_sql_uses_computed_identity(layer): + sql = rewrite_semantic_input(layer.graph, "select accounts.id, accounts.budget from accounts order by accounts.id") + data = layer.adapter.execute(sql) + assert [column[0] for column in data.description] == ["id", "budget"] + assert data.fetchall() == [(101, 10), (201, 20)] def test_compound_computed_keys_join_componentwise(layer): diff --git a/tests/semantic_conformance/test_dialect_boundary.py b/tests/semantic_conformance/test_dialect_boundary.py index a303ed22e..65c074590 100644 --- a/tests/semantic_conformance/test_dialect_boundary.py +++ b/tests/semantic_conformance/test_dialect_boundary.py @@ -66,6 +66,11 @@ def test_native_declared_graph_and_query_syntax(layer, dialect, expression, filt } sql = compile_semantic_input(layer.graph, query, input_dialect=dialect) assert layer.adapter.execute(sql).fetchall() == [("a", 10), (None, 5)] + # Quoted metric references must also resolve to the aggregate output, + # retaining explicit ordering rather than leaking the physical qualifier. + query["order_by"] = [order_sql.replace("label", "value").replace("DESC", "ASC")] + sql = compile_semantic_input(layer.graph, query, input_dialect=dialect) + assert layer.adapter.execute(sql).fetchall() == [(None, 5), ("a", 10)] assert graph_to_semantic_input(layer.graph, input_dialect=dialect) == before @@ -93,7 +98,7 @@ def test_native_fragment_framing_and_source_snapshot(layer): source = graph_to_semantic_input(layer.graph) for field, sql in [("filters", "1 = 1 LIMIT 1"), ("order_by", "orders.value DESC LIMIT 1")]: query = {"metrics": ["orders.value"], "query_dialect": "bigquery", field: [sql]} - with pytest.raises(Exception, match="extra clauses"): + with pytest.raises(ValueError, match="extra clauses"): sidemantic_rs.compile_with_semantic_input(json.dumps(source), json.dumps(query)) source["models"][0]["metrics"][0]["metadata"] = {"ossie_target_dialect": "bigquery"} source["models"][0]["metrics"][0]["sql"] = "IFNULL(`amount`, 0)" diff --git a/tests/semantic_conformance/test_filtered_count_family.py b/tests/semantic_conformance/test_filtered_count_family.py index 49e1d9ead..a3f6d8b2d 100644 --- a/tests/semantic_conformance/test_filtered_count_family.py +++ b/tests/semantic_conformance/test_filtered_count_family.py @@ -186,7 +186,7 @@ def test_count_family_cross_source_counts_restore_absent_zero(counts, restricted ) assert result(counts, metrics=["combined"], dimensions=["regions.region"], order_by=["regions.region"]) == ( ["region", "combined"], - [("a", 12), ("b", 20)] if restricted else [("a", 13), ("b", 21), ("c", 30)], + [("a", 12), ("b", 20), (None, None)] if restricted else [("a", 13), ("b", 21), ("c", 30), (None, None)], ) diff --git a/tests/semantic_conformance/test_multistep_conversion_parity.py b/tests/semantic_conformance/test_multistep_conversion_parity.py index 2f6a72320..a56b3c9fe 100644 --- a/tests/semantic_conformance/test_multistep_conversion_parity.py +++ b/tests/semantic_conformance/test_multistep_conversion_parity.py @@ -154,6 +154,16 @@ def test_time_bucket_belongs_to_first_step(layer): ) +@pytest.mark.parametrize("placement", ["FIRST", "LAST"]) +def test_funnel_order_preserves_explicit_null_placement(layer, placement): + rows = [("a", 4, 4, 1, 1, 1), ("b", 1, 1, 1, 1, 1)] + null_row = [(None, 1, 1, 1, 0, 0)] + assert result(layer, dimensions=["events.region"], order_by=[f"events.region ASC NULLS {placement}"]) == ( + ["region", *COLUMNS], + null_row + rows if placement == "FIRST" else rows + null_row, + ) + + @pytest.mark.parametrize("layer", ["rust"], indirect=True) @pytest.mark.parametrize( "expression", @@ -166,13 +176,23 @@ def test_step_cannot_change_source_scope(layer, expression): @pytest.mark.parametrize("layer", ["rust"], indirect=True) -@pytest.mark.parametrize("alias", ["TOTAL_ENTITIES", "STEP_2_COUNT", "FUNNEL", "ENTITY", "STEP_1_TS"]) +@pytest.mark.parametrize("alias", ["TOTAL_ENTITIES", "STEP_2_COUNT", "FUNNEL"]) def test_fixed_output_alias_collisions_are_explicit(layer, alias): layer.graph.models["events"].dimensions.append(Dimension(name=alias, sql="region", type="categorical")) with pytest.raises(ValueError, match="conversion_output_alias"): layer.compile(metrics=["events.funnel"], dimensions=[f"events.{alias}"], user_attributes={"tenant": 1}) +@pytest.mark.parametrize("layer", ["rust"], indirect=True) +@pytest.mark.parametrize("alias", ["ENTITY", "STEP_1_TS", "step_2_ts", "STEP_3_TS"]) +def test_output_aliases_can_share_names_with_scoped_internal_columns(layer, alias): + layer.graph.models["events"].dimensions.append(Dimension(name=alias, sql="region", type="categorical")) + assert result(layer, dimensions=[f"events.{alias}"], order_by=[f"events.{alias}"]) == ( + [alias, *COLUMNS], + [("a", 4, 4, 1, 1, 1), ("b", 1, 1, 1, 1, 1), (None, 1, 1, 1, 0, 0)], + ) + + @pytest.mark.parametrize("layer", ["rust"], indirect=True) @pytest.mark.parametrize("source", ["sum(tenant) over ()", "(select max(occurred) from funnel_events)"]) def test_mapped_time_cannot_change_source_scope(layer, source): diff --git a/tests/semantic_conformance/test_null_fill.py b/tests/semantic_conformance/test_null_fill.py index d65907f9c..a018adb7c 100644 --- a/tests/semantic_conformance/test_null_fill.py +++ b/tests/semantic_conformance/test_null_fill.py @@ -54,7 +54,15 @@ def test_metric_result_and_dependency_defaults(layer, metrics, filters, expected assert layer.adapter.execute(sql).fetchall() == expected -def test_filled_leaf_participates_in_cross_source_calculation(layer): +@pytest.mark.parametrize( + "metric,expected", + [ + ("filled", [("a", 2), ("b", 4), ("c", 6)]), + ("fractional", [("a", 2), ("b", 2.5), ("c", 4.5)]), + ("raw", [("a", 2), ("b", None), ("c", None)]), + ], +) +def test_filled_leaf_participates_in_cross_source_calculation(layer, metric, expected): layer.add_model( Model( name="groups", @@ -67,12 +75,12 @@ def test_filled_leaf_participates_in_cross_source_calculation(layer): layer.graph.models["events"].relationships = [ Relationship(name="groups", type="many_to_one", foreign_key="category", primary_key="category") ] - layer.add_metric(Metric(name="total", type="derived", sql="events.filled + groups.quota")) + layer.add_metric(Metric(name="total", type="derived", sql=f"events.{metric} + groups.quota")) layer.adapter.execute( "create table fill_groups(category varchar, quota integer); insert into fill_groups values ('a', 2), ('b', 4), ('c', 6)" ) sql = layer.compile(metrics=["total"], dimensions=["groups.category"], order_by=["groups.category"]) - assert layer.adapter.execute(sql).fetchall() == [("a", 2), ("b", 4), ("c", 6)] + assert layer.adapter.execute(sql).fetchall() == expected def test_policy_excluded_groups_are_not_created_by_fill(layer): diff --git a/tests/semantic_conformance/test_postgres_policies.py b/tests/semantic_conformance/test_postgres_policies.py index 7a4aa2978..1177c08e7 100644 --- a/tests/semantic_conformance/test_postgres_policies.py +++ b/tests/semantic_conformance/test_postgres_policies.py @@ -58,7 +58,10 @@ def graph(): name="secured", table="policy_population", primary_key="id", - dimensions=[Dimension(name="id", type="numeric"), Dimension(name="subject", public=False)], + dimensions=[ + Dimension(name="id", type="numeric"), + Dimension(name="subject", type="categorical", public=False), + ], metrics=[ Metric(name="total", agg="sum", sql="amount"), Metric(name="private_total", agg="sum", sql="amount", public=False), @@ -180,7 +183,7 @@ def test_postgres_date_diff_policy_is_not_claimed_equivalent(postgres, graph, mo @pytest.mark.parametrize("mode", ["compile", "query", "sql", "rewriter"]) def test_postgres_public_paths_reach_rust_with_transport_syntax(postgres, graph, mode): - graph.models["secured"].dimensions.append(Dimension(name="CaseSubject", sql="subject")) + graph.models["secured"].dimensions.append(Dimension(name="CaseSubject", type="categorical", sql="subject")) attributes = {"role": "analyst", "tenant": 1, "enabled": True, "subject": "O'Brien"} layer = SemanticLayer( connection=postgres, engine="rust", fallback=False, enforce_visibility=True, auto_register=False @@ -252,7 +255,7 @@ def test_explicit_postgres_graph_input_preserves_policy_results(postgres, graph) @pytest.mark.parametrize("mode", ["query", "sql"]) def test_postgres_public_null_ordering_matches_adapter(postgres, graph, mode): model = graph.models["secured"] - model.dimensions.append(Dimension(name="CaseSubject", sql="subject")) + model.dimensions.append(Dimension(name="CaseSubject", type="categorical", sql="subject")) model.security.row_filters = ["tenant = {{ user.tenant }}"] layer = SemanticLayer(connection=postgres, engine="rust", auto_register=False) layer.graph = graph diff --git a/tests/semantic_conformance/test_query_catalogs.py b/tests/semantic_conformance/test_query_catalogs.py index 1fb1872cb..0c33771e4 100644 --- a/tests/semantic_conformance/test_query_catalogs.py +++ b/tests/semantic_conformance/test_query_catalogs.py @@ -124,12 +124,14 @@ def test_raw_active_catalog_requests_are_never_ignored(layer, field, value): layer.graph, {"metrics": ["orders.revenue"], "user_attributes": {"tenant": "a"}, field: value} ) runtime = pytest.importorskip("sidemantic_rs") - with pytest.raises(ValueError, match=f"rewrite.context.{field}"): + with pytest.raises(ValueError, match=f"rewrite.context.{field}") as caught: runtime.rewrite_with_semantic_input_context( json.dumps(graph_to_semantic_input(layer.graph)), "select orders.revenue from metrics", json.dumps({"user_attributes": {"tenant": "a"}, field: value}), ) + assert isinstance(caught.value, runtime.UnsupportedSemanticFeaturesError) + assert caught.value.capabilities == [f"rewrite.context.{field}"] @pytest.mark.parametrize("layer", ["rust"], indirect=True) diff --git a/tests/semantic_conformance/test_retention_parity.py b/tests/semantic_conformance/test_retention_parity.py index d387ad759..4210e100d 100644 --- a/tests/semantic_conformance/test_retention_parity.py +++ b/tests/semantic_conformance/test_retention_parity.py @@ -8,7 +8,6 @@ from sidemantic import Explore, Metric, SemanticLayer from sidemantic.adapters.sidemantic import SidemanticAdapter from sidemantic.core.semantic_layer import SecurityError -from sidemantic.semantic_handoff import UnsupportedSemanticFeaturesError FIXTURES = Path(__file__).parent / "fixtures" DAY_COLUMNS = ["cohort_date", "days_since", "active_users", "cohort_size", "retention_pct"] @@ -154,11 +153,19 @@ def test_retention_cannot_combine_with_additive_metric(layer): layer.compile(metrics=["events.retained", "events.event_count"], user_attributes={"tenant": 1}) -@pytest.mark.parametrize("shape", ["dimension", "aggregate_filter", "wrapper"]) -def test_rust_rejects_unpromoted_retention_shapes(layer, shape): - if layer.engine != "rust": - pytest.skip("Rust capability boundary; Python retains its existing feature behavior") - query = {} +@pytest.mark.parametrize( + "shape,capability", + [ + ("dimension", "metric.retention_query_shape"), + ("aggregate_filter", "metric.retention_aggregate_filter"), + ("wrapper", "metric.retention_wrapped_metric"), + ], +) +def test_retention_rejects_unsupported_shapes_before_execution(layer, shape, capability): + # Python previously ignored selected dimensions, emitted invalid SQL for + # aggregate filters, or failed while expanding wrappers. None implemented + # these shapes: reject at compile time instead of claiming result parity. + query = {"metrics": ["events.retained"], "user_attributes": {"tenant": 1}} if shape == "dimension": query["dimensions"] = ["events.country"] elif shape == "aggregate_filter": @@ -166,8 +173,24 @@ def test_rust_rejects_unpromoted_retention_shapes(layer, shape): else: layer.graph.get_model("events").metrics.append(Metric(name="wrapped", type="derived", sql="retained")) query["metrics"] = ["events.wrapped"] - with pytest.raises(UnsupportedSemanticFeaturesError): - result(layer, **query) + with pytest.raises(ValueError, match=capability): + layer.compile(**query) + + +@pytest.mark.parametrize("predicate", ["event_count > 0", "events.event_count > 0", "COUNT(*) > 0"]) +@pytest.mark.parametrize("location", ["query", "metric"]) +def test_retention_rejects_aggregate_source_filters_before_execution(layer, predicate, location): + query = {"metrics": ["events.retained"], "user_attributes": {"tenant": 1}} + if location == "query": + query["filters"] = [predicate] + else: + layer.graph.get_model("events").get_metric("retained").filters = [predicate] + with pytest.raises(ValueError, match="retention"): + layer.compile(**query) + + +def test_retention_filter_literals_are_not_metric_references(layer): + assert result(layer, filters=["events.country != 'event_count'"]) == (DAY_COLUMNS, DAY_ROWS) def test_selected_calculations_preserve_retention_fixed_projection(layer): @@ -198,10 +221,8 @@ def test_selected_calculations_preserve_retention_fixed_projection(layer): # caller attributes; compile separately performs authorization. payload = {**query, "metrics": ["events.retained"], "table_calculations": [c.name for c in calculations]} assert ( - json.loads( - sidemantic_rs.validate_with_semantic_input( - json.dumps(graph_to_semantic_input(layer.graph)), json.dumps(payload) - ) + sidemantic_rs.validate_with_semantic_input( + json.dumps(graph_to_semantic_input(layer.graph)), json.dumps(payload) ) == [] ) diff --git a/tests/semantic_conformance/test_rollup_routing.py b/tests/semantic_conformance/test_rollup_routing.py index d77af774e..76ac2fea5 100644 --- a/tests/semantic_conformance/test_rollup_routing.py +++ b/tests/semantic_conformance/test_rollup_routing.py @@ -147,7 +147,7 @@ def test_relationship_foreign_key_dimension_declines_unrelated_rollup(layer): { "metrics": ["orders.revenue"], "dimensions": ["orders.customer_id"], - "order_by": ["orders.customer_id"], + "order_by": ["orders.customer_id NULLS LAST"], }, [(10, 30), (20, 940), (None, None)], routed=False, diff --git a/tests/semantic_conformance/test_selected_table_calculations.py b/tests/semantic_conformance/test_selected_table_calculations.py index 145be528c..86706d8a9 100644 --- a/tests/semantic_conformance/test_selected_table_calculations.py +++ b/tests/semantic_conformance/test_selected_table_calculations.py @@ -182,11 +182,12 @@ def test_canonical_validation_checks_active_calculations(mutation): elif mutation == "unknown_option": source["table_calculations"][0]["unknown_window"] = "future" if mutation is None: - assert json.loads(runtime.validate_with_semantic_input(json.dumps(source), json.dumps(query))) == [] + assert runtime.validate_with_semantic_input(json.dumps(source), json.dumps(query)) == [] else: - with pytest.raises(ValueError): + error = runtime.UnsupportedSemanticFeaturesError if mutation == "formula" else ValueError + with pytest.raises(error): runtime.validate_with_semantic_input(json.dumps(source), json.dumps(query)) - with pytest.raises(ValueError): + with pytest.raises(error): runtime.compile_with_semantic_input(json.dumps(source), json.dumps(query)) diff --git a/tests/semantic_conformance/test_snapshot_parity.py b/tests/semantic_conformance/test_snapshot_parity.py index 444204a87..41eb9e0e7 100644 --- a/tests/semantic_conformance/test_snapshot_parity.py +++ b/tests/semantic_conformance/test_snapshot_parity.py @@ -58,6 +58,25 @@ def test_simple_model_snapshot_is_not_silently_summed(layer): assert result(layer, metrics=["snapshots.closing"]) == [(1480,)] +def test_explicit_unsafe_snapshot_option_preserves_all_authorized_rows(layer): + from sidemantic import Metric + + layer.allow_non_additive_unsafe = True + layer.add_metric(Metric(name="wrapped", type="derived", sql="snapshots.closing + 1")) + assert result(layer, metrics=["snapshots.closing", "snapshots.activity", "wrapped"]) == [(1480, 36, 1481)] + assert result( + layer, metrics=["snapshots.closing"], dimensions=["snapshots.account"], order_by=["snapshots.account"] + ) == [("A", 420), ("B", 130), ("C", 30), ("D", 900)] + assert result(layer, metrics=["snapshots.closing"], user_attributes={"tenant": 2}) == [(999,)] + sql = "select closing from snapshots" + assert layer.sql(sql, user_attributes={"tenant": 1}).fetchall() == [(1480,)] + # The option changes this layer's planning, not the graph annotation. + assert layer.graph.models["snapshots"].get_metric("closing").non_additive_dimension == "day" + layer.allow_non_additive_unsafe = False + assert result(layer, metrics=["snapshots.closing", "wrapped"]) == [(250, 251)] + assert layer.sql(sql, user_attributes={"tenant": 1}).fetchall() == [(250,)] + + def test_declared_snapshot_groups_do_not_become_global_latest_date(layer): # The global latest authorized date belongs to C and has a NULL balance. assert result(layer, metrics=["snapshots.closing", "snapshots.global_closing"]) == [(250, None)] @@ -72,6 +91,15 @@ def test_snapshot_groups_preserve_all_null_time_and_latest_null_value(layer): ) == [("A", 170, 100, 6), ("B", 80, 50, 9), ("C", None, 30, 13), ("D", None, None, 8)] +def test_snapshot_order_preserves_explicit_null_placement(layer): + assert result( + layer, + metrics=["snapshots.closing"], + dimensions=["snapshots.account"], + order_by=["snapshots.closing DESC NULLS FIRST", "snapshots.account"], + ) == [("C", None), ("D", None), ("A", 170), ("B", 80)] + + def test_declared_groups_roll_up_into_selected_region(layer): assert result( layer, @@ -205,17 +233,19 @@ def test_snapshot_default_handles_empty_totals_without_creating_groups(layer): ) -def test_filled_snapshot_calculated_wrapper_stays_explicitly_unsupported_in_rust(layer): +def test_filled_snapshot_calculated_wrapper_uses_final_aggregate_default(layer): from sidemantic import Metric - from sidemantic.semantic_handoff import UnsupportedSemanticFeaturesError layer.graph.models["snapshots"].get_metric("closing").fill_nulls_with = -9 layer.add_metric(Metric(name="wrapped", type="derived", sql="snapshots.closing + 1")) - if layer.engine == "rust": - with pytest.raises(UnsupportedSemanticFeaturesError): - result(layer, metrics=["wrapped"]) - else: - assert result(layer, metrics=["wrapped"]) == [(251,)] + assert result(layer, metrics=["wrapped"]) == [(251,)] + assert result(layer, metrics=["wrapped"], user_attributes={"tenant": 99}) == [(-8,)] + assert result(layer, metrics=["wrapped"], dimensions=["snapshots.account"], order_by=["snapshots.account"]) == [ + ("A", 171), + ("B", 81), + ("C", -8), + ("D", -8), + ] def test_snapshot_and_additive_sibling_keep_distinct_final_defaults(layer): diff --git a/tests/semantic_conformance/test_window_output_parity.py b/tests/semantic_conformance/test_window_output_parity.py index 4bb793e70..2a621cea4 100644 --- a/tests/semantic_conformance/test_window_output_parity.py +++ b/tests/semantic_conformance/test_window_output_parity.py @@ -43,6 +43,37 @@ def layer(request): layer.adapter.close() +def test_window_dimension_replaces_same_named_source_for_grouping_and_filters(layer): + layer.graph.models["events"].get_dimension("day").window = "MIN(day) OVER ()" + query = { + "metrics": ["events.daily_amount"], + "dimensions": ["events.day"], + "user_attributes": {"tenant": 1}, + } + assert layer.query(**query).fetchall() == [(date(2024, 1, 1), 82)] + for predicate in ["events.day > '2024-01-02'", "coalesce(events.day, '2024-01-01') > '2024-01-02'"]: + assert layer.query(**query, filters=[predicate]).fetchall() == [] + assert layer.query(**query, filters=["events.day = '2024-01-01'"]).fetchall() == [(date(2024, 1, 1), 82)] + + +def test_graph_calculation_keeps_its_identity_beside_cumulative_metric(layer): + layer.add_metric(Metric(name="total", type="derived", sql="events.daily_amount")) + layer.add_metric(Metric(name="running", type="cumulative", sql="events.daily_amount")) + cursor = layer.query( + metrics=["total", "running"], + dimensions=["events.day"], + order_by=["events.day"], + user_attributes={"tenant": 1}, + ) + columns = [column[0] for column in cursor.description] + records = [dict(zip(columns, row)) for row in cursor.fetchall()] + assert [(record["day"], record["total"], record["running"]) for record in records] == [ + (date(2024, 1, 1), 15, 15), + (date(2024, 1, 3), 27, 42), + (date(2024, 1, 4), 40, 82), + ] + + def run( layer, *, @@ -243,7 +274,7 @@ def test_window_dependency_cycles_return_validation_errors(rust_layer, reference window_expression=f"SUM(base.{reference})", ) ) - with pytest.raises(Exception, match="[Cc]ycl"): + with pytest.raises(Exception, match="[Cc](?:ircular|ycl)"): rust_layer.compile(metrics=["events.windowed"], dimensions=["events.day"], user_attributes={"tenant": 1}) diff --git a/tests/semantic_conformance/test_yardstick_source_dialects.py b/tests/semantic_conformance/test_yardstick_source_dialects.py index a32d73044..6333ec6f0 100644 --- a/tests/semantic_conformance/test_yardstick_source_dialects.py +++ b/tests/semantic_conformance/test_yardstick_source_dialects.py @@ -54,12 +54,14 @@ def test_source_functions_and_quoted_all_dimensions(layer, source, quote, condit @pytest.mark.parametrize("source,quote", [("bigquery", "`"), ("snowflake", '"')]) -def test_source_current_modifier_uses_canonical_group_context(layer, source, quote): +@pytest.mark.parametrize("qualified", [False, True]) +def test_source_current_modifier_uses_canonical_group_context(layer, source, quote, qualified): q = quote + current = f"{q}sales_v{q}.{q}year{q}" if qualified else f"{q}year{q}" assert result( layer, source, - f"SELECT {q}year{q}, AGGREGATE({q}revenue{q}) AT (SET {q}year{q} = CURRENT {q}year{q} - 1) AS prior " + f"SELECT {q}year{q}, AGGREGATE({q}revenue{q}) AT (SET {q}year{q} = CURRENT {current} - 1) AS prior " f"FROM {q}sales_v{q} GROUP BY {q}year{q} ORDER BY {q}year{q}", ) == [(2022, None), (2023, 150)] diff --git a/tests/test_cli_commands.py b/tests/test_cli_commands.py index 8b3340dd1..c7c78b110 100644 --- a/tests/test_cli_commands.py +++ b/tests/test_cli_commands.py @@ -203,6 +203,8 @@ def test_explain_sql_outputs_planner_json(tmp_path): app, [ "explain-sql", + "--engine", + "python", "SELECT * FROM (SELECT order_count, status FROM orders) sq WHERE status = 'completed'", "--models", str(tmp_path), diff --git a/tests/test_cli_t2_contract.py b/tests/test_cli_t2_contract.py index 7405f8335..1b8f1fd98 100644 --- a/tests/test_cli_t2_contract.py +++ b/tests/test_cli_t2_contract.py @@ -21,7 +21,7 @@ @pytest.fixture(autouse=True) -def _reset_cli_state(monkeypatch: pytest.MonkeyPatch): +def _reset_cli_state(monkeypatch: pytest.MonkeyPatch, request): cli_module._loaded_config = None cli_module._project_context = None for name in ( @@ -48,6 +48,8 @@ def _reset_cli_state(monkeypatch: pytest.MonkeyPatch): "CI", ): monkeypatch.delenv(name, raising=False) + # Preserve the selected engine after clearing host CLI settings. + monkeypatch.setenv("SIDEMANTIC_ENGINE", request.config.getoption("--test-engine")) yield cli_module._loaded_config = None cli_module._project_context = None diff --git a/tests/test_mcp_server.py b/tests/test_mcp_server.py index 5d487f9eb..7e3bf780f 100644 --- a/tests/test_mcp_server.py +++ b/tests/test_mcp_server.py @@ -236,14 +236,20 @@ def test_run_query_base_time_filter_uses_raw_expression(demo_layer): def test_run_query_explicit_time_grain_filter_truncates_expression(demo_layer): + demo_layer.conn.execute("ALTER TABLE orders_table ALTER COLUMN order_date TYPE TIMESTAMP") + demo_layer.conn.execute( + "INSERT INTO orders_table VALUES (99, '99', 'Midday', '2024-01-02 12:00:00', 999, 'completed')" + ) result = run_query( metrics=["orders.total_revenue"], - where="orders.order_date__day >= DATE '2024-01-02'", + where="orders.order_date__day > DATE '2024-01-02'", dry_run=True, ) - where_clause = result["sql"].split("WHERE", 1)[1].split(")\nSELECT", 1)[0] - assert "DATE_TRUNC('DAY', ORDER_DATE)" in where_clause.upper() + expected = demo_layer.conn.execute( + "SELECT SUM(amount) FROM orders_table WHERE DATE_TRUNC('day', order_date) > DATE '2024-01-02'" + ).fetchall() + assert demo_layer.conn.execute(result["sql"]).fetchall() == expected def test_run_query_freezes_current_timestamp_for_partition_pruning(demo_layer): diff --git a/tests/test_metric_expressions.py b/tests/test_metric_expressions.py index 2ba02cb47..a0dfd40c7 100644 --- a/tests/test_metric_expressions.py +++ b/tests/test_metric_expressions.py @@ -196,7 +196,10 @@ def test_ratio_prefers_exact_graph_metric_with_dotted_name(): sql = layer.compile(metrics=["exact_ratio"], dimensions=["orders.status"]) assert "orders_cte.revenue_raw" not in sql - assert "SUM(orders_cte.amount) * 2" in sql + graph_rows = layer.query( + metrics=["orders.revenue"], dimensions=["orders.status"], order_by=["orders.status"] + ).fetchall() + assert graph_rows == [("open", 50), ("paid", 300)] rows = layer.query(metrics=["exact_ratio"], dimensions=["orders.status"], order_by=["orders.status"]).fetchall() assert rows == [("open", 1.0), ("paid", 1.0)] diff --git a/tests/test_native_build_cache.py b/tests/test_native_build_cache.py new file mode 100644 index 000000000..31c30b09f --- /dev/null +++ b/tests/test_native_build_cache.py @@ -0,0 +1,157 @@ +"""Exercise uv's local wheel reuse with the native packages' real cache keys.""" + +import os +import shutil +import subprocess +import sys +import time +from pathlib import Path +from textwrap import dedent + +import pytest +import yaml + + +@pytest.mark.parametrize("restored", [True, False]) +def test_imported_rust_cache_archive_is_removed_without_losing_build_state(tmp_path, restored): + repo = Path(__file__).resolve().parents[1] + action = yaml.safe_load((repo / ".github/actions/setup-rust-cache/action.yml").read_text()) + steps = action["runs"]["steps"] + cleanup_index = next(i for i, step in enumerate(steps) if step.get("name") == "Remove imported Rust cache archive") + # A failed upstream import must stop setup before any cleanup takes place. + assert steps[cleanup_index - 1]["id"] == "cache" + assert steps[cleanup_index].get("if", "success()") == "success()" + assert steps[cleanup_index]["env"]["MBX_CACHE_DIR"] == steps[cleanup_index - 1]["env"]["MBX_CACHE_DIR"] + + cache = tmp_path / "cache with spaces" / "actions" + cache.mkdir(parents=True) + archive = cache / "github-actions-cache-v1.tar" + if restored: + archive.write_bytes(b"imported transport archive") + retained = [cache / "objects" / "compiler-object", tmp_path / "target" / "compiled-library"] + for path in retained: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_bytes(b"reusable build state") + + # Substitute only mbx's directory lookup; execute the action's real shell. + result = subprocess.run( + [ + "bash", + "-e", + "-c", + 'mbx() { test "$*" = "cache dir"; printf "%s\\n" "$TEST_CACHE_DIR"; }\n' + steps[cleanup_index]["run"], + ], + env={**os.environ, "TEST_CACHE_DIR": str(cache)}, + capture_output=True, + text=True, + timeout=10, + ) + assert result.returncode == 0, result.stderr + assert not archive.exists() + for path in retained: + assert path.read_bytes() == b"reusable build state" + + +@pytest.mark.parametrize( + ("package", "changed_file"), + [ + ("sidemantic-rs", "src/lib.rs"), + ("sidemantic-rs", "build.rs"), + ("sidemantic-rs", "../Cargo.lock"), + ("crates/dax-pyo3", "src/lib.rs"), + ("crates/dax-pyo3", "../dax-parser/src/functions.rs"), + ("crates/dax-pyo3", "python/sidemantic_dax/__init__.py"), + ], +) +def test_native_source_changes_invalidate_uv_wheels(tmp_path, package, changed_file): + uv = shutil.which("uv") + if uv is None: + pytest.skip("uv is required to exercise its wheel cache") + + repo = Path(__file__).resolve().parents[1] + project = tmp_path / "source" / package + project.mkdir(parents=True) + source = project / changed_file + source.parent.mkdir(parents=True, exist_ok=True) + source.write_text("first source\n") + cache_keys = (repo / package / "pyproject.toml").read_text().split("[tool.uv]\n", 1)[1] + (project / "pyproject.toml").write_text( + '[build-system]\nrequires = []\nbuild-backend = "backend"\nbackend-path = ["."]\n' + '[project]\nname = "cache-probe"\nversion = "1.0"\n' + f"[tool.uv]\n{cache_keys}" + ) + # A dependency-free PEP 517 backend stands in for maturin. The observable + # wheel contents prove when uv reuses a wheel and when it calls the backend; + # no Rust compiler or network is needed for this invalidation contract. + (project / "backend.py").write_text( + dedent( + f"""\ + from pathlib import Path + from zipfile import ZipFile + + def build_wheel(wheel_directory, config_settings=None, metadata_directory=None): + name = "cache_probe-1.0-py3-none-any.whl" + source = Path({changed_file!r}).read_text() + with ZipFile(Path(wheel_directory) / name, "w") as wheel: + wheel.writestr("cache_probe.py", source) + wheel.writestr( + "cache_probe-1.0.dist-info/METADATA", + "Metadata-Version: 2.1\\nName: cache-probe\\nVersion: 1.0\\n", + ) + wheel.writestr( + "cache_probe-1.0.dist-info/WHEEL", + "Wheel-Version: 1.0\\nRoot-Is-Purelib: true\\nTag: py3-none-any\\n", + ) + wheel.writestr("cache_probe-1.0.dist-info/RECORD", "") + counter = Path("build-count") + previous = int(counter.read_text()) if counter.exists() else 0 + counter.write_text(str(previous + 1)) + return name + """ + ) + ) + + requirement = tmp_path / "requirements.txt" + requirement.write_text(f"cache-probe @ {project.as_uri()}\n") + + def install(attempt): + target = tmp_path / f"installed-{attempt}" + result = subprocess.run( + [ + uv, + "pip", + "install", + "--python", + sys.executable, + "--target", + str(target), + "--no-deps", + "--no-index", + "--requirement", + str(requirement), + ], + env={**os.environ, "UV_CACHE_DIR": str(tmp_path / "cache")}, + capture_output=True, + text=True, + timeout=30, + ) + assert result.returncode == 0, result.stderr + return (target / "cache_probe.py").read_text() + + assert install(1) == "first source\n" + first_builds = (project / "build-count").read_text() + if package == "crates/dax-pyo3": + # Import and editable-build outputs must not invalidate their own wheel. + generated = project / "python/sidemantic_dax" + (generated / "__pycache__").mkdir(parents=True, exist_ok=True) + (generated / "__pycache__/__init__.pyc").write_bytes(b"bytecode") + (generated / "_native.so").write_bytes(b"compiled extension") + assert install(2) == "first source\n" + assert (project / "build-count").read_text() == first_builds + + source.write_text("changed source\n") + # uv's file keys use mtimes; avoid depending on filesystem clock resolution. + changed_at = time.time() + 2 + os.utime(source, (changed_at, changed_at)) + assert install(3) == "changed source\n" + assert int((project / "build-count").read_text()) > int(first_builds) diff --git a/tests/test_query_fragment_boundary.py b/tests/test_query_fragment_boundary.py index 58ed88428..1dd92f51c 100644 --- a/tests/test_query_fragment_boundary.py +++ b/tests/test_query_fragment_boundary.py @@ -320,12 +320,20 @@ def test_escaped_static_quote_before_parameter_keeps_literal_boundary(query_laye @pytest.mark.parametrize("use_segment", [False, True]) -def test_escaped_static_quote_before_parameter_through_query(query_layer, use_segment): +@pytest.mark.parametrize( + "literal,static_prefix", + [ + (r"E'prefix\'{{ value }}'", "prefix'"), + (r"e'é prefix\\{{ value }}'", "é prefix\\"), + (r"E'line\n{{ value }}'", "line\n"), + ], +) +def test_escaped_static_quote_before_parameter_through_query(query_layer, use_segment, literal, static_prefix): from sidemantic.core.segment import Segment query_layer.graph.add_parameter(Parameter(name="value", type="string")) - predicate = "{# c #}events.event_type = E'prefix\\'{{ value }}'" - query_layer.conn.execute("insert into events_raw values (3, 1, ?, '2024-01-03')", ["prefix'ok"]) + predicate = "{# c #}events.event_type = " + literal + query_layer.conn.execute("insert into events_raw values (3, 1, ?, '2024-01-03')", [static_prefix + "ok"]) if use_segment: query_layer.graph.models["events"].segments.append(Segment(name="prefixed", sql=predicate)) query_args = {"segments": ["events.prefixed"]} diff --git a/tests/test_sql_generation_security.py b/tests/test_sql_generation_security.py index 1b8439e9b..12eb87505 100644 --- a/tests/test_sql_generation_security.py +++ b/tests/test_sql_generation_security.py @@ -4,6 +4,7 @@ """ import pytest +from sqlglot import exp, parse_one from sidemantic import Dimension, Metric, Model from sidemantic.core.table_calculation import TableCalculation @@ -219,9 +220,16 @@ def test_model_ref_rewrite_matches_cte_identifier_quoting(layer): sql = layer.compile(metrics=["ORDERS.inline_total"]) - assert "WITH ORDERS_cte AS" in sql - assert "SUM(ORDERS_cte.amount) AS inline_total" in sql - assert 'SUM("ORDERS_cte".amount) AS inline_total' not in sql + parsed = parse_one(sql, dialect="postgres") + cte = next(cte for cte in parsed.find_all(exp.CTE) if cte.alias == "ORDERS_cte") + definition = cte.args["alias"].this + references = [table.this for table in parsed.find_all(exp.Table) if table.name == cte.alias] + references.extend(column.args["table"] for column in parsed.find_all(exp.Column) if column.table == cte.alias) + assert references + assert all(ref.args.get("quoted", False) == definition.args.get("quoted", False) for ref in references) + layer.conn.execute("CREATE TABLE orders_table (order_id INTEGER, amount INTEGER)") + layer.conn.execute("INSERT INTO orders_table VALUES (1, 10), (2, 25)") + assert layer.conn.execute(sql).fetchall() == [(35,)] def test_inline_aggregate_dependency_alias_uses_identifier_quoting(layer): @@ -245,8 +253,12 @@ def test_inline_aggregate_dependency_alias_uses_identifier_quoting(layer): sql = layer.compile(metrics=["orders.inline_total"]) - assert 'AS "order total"' in sql - assert "AS order total" not in sql + columns = [column for column in parse_one(sql).find_all(exp.Column) if column.name == "order total"] + assert columns + assert all(column.this.args.get("quoted") for column in columns) + layer.conn.execute('CREATE TABLE orders_table (id INTEGER, amount INTEGER, "order total" INTEGER)') + layer.conn.execute("INSERT INTO orders_table VALUES (1, 100, 10), (2, 200, 25)") + assert layer.conn.execute(sql).fetchall() == [(35,)] def test_count_metrics_with_filters(layer): diff --git a/tests/test_validation.py b/tests/test_validation.py index b44e8385f..2068fc273 100644 --- a/tests/test_validation.py +++ b/tests/test_validation.py @@ -43,8 +43,8 @@ def test_model_validation_no_table(layer): assert "must have one of 'table', 'sql', 'dax', or 'source_uri' defined" in str(exc_info.value) -def test_source_uri_model_validates_but_python_compile_is_not_supported(layer): - """source_uri-only models can load, but Python SQL generation cannot query them yet.""" +def test_source_uri_model_validates_but_compile_is_not_supported(layer): + """source_uri-only models can load, but SQL generation cannot query them yet.""" layer.add_model( Model( name="events", @@ -60,7 +60,7 @@ def test_source_uri_model_validates_but_python_compile_is_not_supported(layer): message = str(exc_info.value) assert "source_uri" in message - assert "Python SQL generation does not load source_uri data" in message + assert "SQL generation does not load source_uri data" in message def test_metric_validation_simple_no_measure(): @@ -269,9 +269,11 @@ def test_query_validation_reports_ambiguous_join_routes(layer): layer.compile(metrics=["a.total"], dimensions=["d.label"]) message = str(exc_info.value) - assert "Ambiguous join paths between a and d" in message - assert "a -> b -> d" in message - assert "a -> c -> d" in message + if "Ambiguous join paths between a and d" in message: + assert "a -> b -> d" in message + assert "a -> c -> d" in message + else: + assert "Ambiguous join path between 'a' and 'd': multiple paths exist" in message def test_query_validation_invalid_granularity(layer): diff --git a/uv.lock b/uv.lock index eff328ed1..1798d98c2 100644 --- a/uv.lock +++ b/uv.lock @@ -2,17 +2,14 @@ version = 1 revision = 3 requires-python = ">=3.11" resolution-markers = [ - "python_full_version >= '3.15' and platform_python_implementation == 'PyPy'", - "python_full_version >= '3.15' and platform_python_implementation != 'PyPy'", - "python_full_version == '3.14.*' and platform_python_implementation == 'PyPy'", - "python_full_version == '3.14.*' and platform_python_implementation != 'PyPy'", - "python_full_version == '3.13.*' and platform_python_implementation != 'PyPy'", - "python_full_version == '3.13.*' and platform_python_implementation == 'PyPy'", + "python_full_version >= '3.15'", + "python_full_version == '3.14.*'", + "python_full_version == '3.13.*'", "python_full_version == '3.12.*'", "python_full_version < '3.12'", ] required-markers = [ - "platform_machine == 'x86_64' and sys_platform == 'linux'", + "python_full_version < '3.15' and implementation_name == 'cpython' and platform_machine == 'x86_64' and platform_python_implementation == 'CPython' and sys_platform == 'linux'", ] [[package]] @@ -59,8 +56,7 @@ name = "altair" version = "5.5.0" source = { registry = "https://pypi.org/simple" } resolution-markers = [ - "python_full_version == '3.13.*' and platform_python_implementation != 'PyPy'", - "python_full_version == '3.13.*' and platform_python_implementation == 'PyPy'", + "python_full_version == '3.13.*'", "python_full_version == '3.12.*'", "python_full_version < '3.12'", ] @@ -81,10 +77,8 @@ name = "altair" version = "6.2.2" source = { registry = "https://pypi.org/simple" } resolution-markers = [ - "python_full_version >= '3.15' and platform_python_implementation == 'PyPy'", - "python_full_version >= '3.15' and platform_python_implementation != 'PyPy'", - "python_full_version == '3.14.*' and platform_python_implementation == 'PyPy'", - "python_full_version == '3.14.*' and platform_python_implementation != 'PyPy'", + "python_full_version >= '3.15'", + "python_full_version == '3.14.*'", ] dependencies = [ { name = "jinja2" }, @@ -179,14 +173,11 @@ name = "argon2-cffi-bindings" version = "21.2.0" source = { registry = "https://pypi.org/simple" } resolution-markers = [ - "python_full_version >= '3.15' and platform_python_implementation == 'PyPy'", - "python_full_version >= '3.15' and platform_python_implementation != 'PyPy'", - "python_full_version == '3.14.*' and platform_python_implementation == 'PyPy'", - "python_full_version == '3.14.*' and platform_python_implementation != 'PyPy'", + "python_full_version >= '3.15'", + "python_full_version == '3.14.*'", ] dependencies = [ - { name = "cffi", version = "1.17.1", source = { registry = "https://pypi.org/simple" }, marker = "platform_python_implementation == 'PyPy'" }, - { name = "cffi", version = "2.0.0", source = { registry = "https://pypi.org/simple" }, marker = "platform_python_implementation != 'PyPy'" }, + { name = "cffi" }, ] sdist = { url = "https://files.pythonhosted.org/packages/b9/e9/184b8ccce6683b0aa2fbb7ba5683ea4b9c5763f1356347f1312c32e3c66e/argon2-cffi-bindings-21.2.0.tar.gz", hash = "sha256:bb89ceffa6c791807d1305ceb77dbfacc5aa499891d2c55661c6459651fc39e3", size = 1779911, upload-time = "2021-12-01T08:52:55.68Z" } wheels = [ @@ -207,14 +198,12 @@ name = "argon2-cffi-bindings" version = "25.1.0" source = { registry = "https://pypi.org/simple" } resolution-markers = [ - "python_full_version == '3.13.*' and platform_python_implementation != 'PyPy'", - "python_full_version == '3.13.*' and platform_python_implementation == 'PyPy'", + "python_full_version == '3.13.*'", "python_full_version == '3.12.*'", "python_full_version < '3.12'", ] dependencies = [ - { name = "cffi", version = "1.17.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.13.*' and platform_python_implementation == 'PyPy'" }, - { name = "cffi", version = "2.0.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version != '3.13.*' or platform_python_implementation != 'PyPy'" }, + { name = "cffi" }, ] sdist = { url = "https://files.pythonhosted.org/packages/5c/2d/db8af0df73c1cf454f71b2bbe5e356b8c1f8041c979f505b3d3186e520a9/argon2_cffi_bindings-25.1.0.tar.gz", hash = "sha256:b957f3e6ea4d55d820e40ff76f450952807013d361a65d7f28acc0acbf29229d", size = 1783441, upload-time = "2025-07-30T10:02:05.147Z" } wheels = [ @@ -428,67 +417,10 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/e4/37/af0d2ef3967ac0d6113837b44a4f0bfe1328c2b9763bd5b1744520e5cfed/certifi-2025.10.5-py3-none-any.whl", hash = "sha256:0f212c2744a9bb6de0c56639a6f68afe01ecd92d91f14ae897c4fe7bbeeef0de", size = 163286, upload-time = "2025-10-05T04:12:14.03Z" }, ] -[[package]] -name = "cffi" -version = "1.17.1" -source = { registry = "https://pypi.org/simple" } -resolution-markers = [ - "python_full_version >= '3.15' and platform_python_implementation == 'PyPy'", - "python_full_version == '3.14.*' and platform_python_implementation == 'PyPy'", - "python_full_version == '3.13.*' and platform_python_implementation == 'PyPy'", -] -dependencies = [ - { name = "pycparser" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/fc/97/c783634659c2920c3fc70419e3af40972dbaf758daa229a7d6ea6135c90d/cffi-1.17.1.tar.gz", hash = "sha256:1c39c6016c32bc48dd54561950ebd6836e1670f2ae46128f67cf49e789c52824", size = 516621, upload-time = "2024-09-04T20:45:21.852Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/6b/f4/927e3a8899e52a27fa57a48607ff7dc91a9ebe97399b357b85a0c7892e00/cffi-1.17.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:a45e3c6913c5b87b3ff120dcdc03f6131fa0065027d0ed7ee6190736a74cd401", size = 182264, upload-time = "2024-09-04T20:43:51.124Z" }, - { url = "https://files.pythonhosted.org/packages/6c/f5/6c3a8efe5f503175aaddcbea6ad0d2c96dad6f5abb205750d1b3df44ef29/cffi-1.17.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:30c5e0cb5ae493c04c8b42916e52ca38079f1b235c2f8ae5f4527b963c401caf", size = 178651, upload-time = "2024-09-04T20:43:52.872Z" }, - { url = "https://files.pythonhosted.org/packages/94/dd/a3f0118e688d1b1a57553da23b16bdade96d2f9bcda4d32e7d2838047ff7/cffi-1.17.1-cp311-cp311-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f75c7ab1f9e4aca5414ed4d8e5c0e303a34f4421f8a0d47a4d019ceff0ab6af4", size = 445259, upload-time = "2024-09-04T20:43:56.123Z" }, - { url = "https://files.pythonhosted.org/packages/2e/ea/70ce63780f096e16ce8588efe039d3c4f91deb1dc01e9c73a287939c79a6/cffi-1.17.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a1ed2dd2972641495a3ec98445e09766f077aee98a1c896dcb4ad0d303628e41", size = 469200, upload-time = "2024-09-04T20:43:57.891Z" }, - { url = "https://files.pythonhosted.org/packages/1c/a0/a4fa9f4f781bda074c3ddd57a572b060fa0df7655d2a4247bbe277200146/cffi-1.17.1-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:46bf43160c1a35f7ec506d254e5c890f3c03648a4dbac12d624e4490a7046cd1", size = 477235, upload-time = "2024-09-04T20:44:00.18Z" }, - { url = "https://files.pythonhosted.org/packages/62/12/ce8710b5b8affbcdd5c6e367217c242524ad17a02fe5beec3ee339f69f85/cffi-1.17.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a24ed04c8ffd54b0729c07cee15a81d964e6fee0e3d4d342a27b020d22959dc6", size = 459721, upload-time = "2024-09-04T20:44:01.585Z" }, - { url = "https://files.pythonhosted.org/packages/ff/6b/d45873c5e0242196f042d555526f92aa9e0c32355a1be1ff8c27f077fd37/cffi-1.17.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:610faea79c43e44c71e1ec53a554553fa22321b65fae24889706c0a84d4ad86d", size = 467242, upload-time = "2024-09-04T20:44:03.467Z" }, - { url = "https://files.pythonhosted.org/packages/1a/52/d9a0e523a572fbccf2955f5abe883cfa8bcc570d7faeee06336fbd50c9fc/cffi-1.17.1-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:a9b15d491f3ad5d692e11f6b71f7857e7835eb677955c00cc0aefcd0669adaf6", size = 477999, upload-time = "2024-09-04T20:44:05.023Z" }, - { url = "https://files.pythonhosted.org/packages/44/74/f2a2460684a1a2d00ca799ad880d54652841a780c4c97b87754f660c7603/cffi-1.17.1-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:de2ea4b5833625383e464549fec1bc395c1bdeeb5f25c4a3a82b5a8c756ec22f", size = 454242, upload-time = "2024-09-04T20:44:06.444Z" }, - { url = "https://files.pythonhosted.org/packages/f8/4a/34599cac7dfcd888ff54e801afe06a19c17787dfd94495ab0c8d35fe99fb/cffi-1.17.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:fc48c783f9c87e60831201f2cce7f3b2e4846bf4d8728eabe54d60700b318a0b", size = 478604, upload-time = "2024-09-04T20:44:08.206Z" }, - { url = "https://files.pythonhosted.org/packages/34/33/e1b8a1ba29025adbdcda5fb3a36f94c03d771c1b7b12f726ff7fef2ebe36/cffi-1.17.1-cp311-cp311-win32.whl", hash = "sha256:85a950a4ac9c359340d5963966e3e0a94a676bd6245a4b55bc43949eee26a655", size = 171727, upload-time = "2024-09-04T20:44:09.481Z" }, - { url = "https://files.pythonhosted.org/packages/3d/97/50228be003bb2802627d28ec0627837ac0bf35c90cf769812056f235b2d1/cffi-1.17.1-cp311-cp311-win_amd64.whl", hash = "sha256:caaf0640ef5f5517f49bc275eca1406b0ffa6aa184892812030f04c2abf589a0", size = 181400, upload-time = "2024-09-04T20:44:10.873Z" }, - { url = "https://files.pythonhosted.org/packages/5a/84/e94227139ee5fb4d600a7a4927f322e1d4aea6fdc50bd3fca8493caba23f/cffi-1.17.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:805b4371bf7197c329fcb3ead37e710d1bca9da5d583f5073b799d5c5bd1eee4", size = 183178, upload-time = "2024-09-04T20:44:12.232Z" }, - { url = "https://files.pythonhosted.org/packages/da/ee/fb72c2b48656111c4ef27f0f91da355e130a923473bf5ee75c5643d00cca/cffi-1.17.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:733e99bc2df47476e3848417c5a4540522f234dfd4ef3ab7fafdf555b082ec0c", size = 178840, upload-time = "2024-09-04T20:44:13.739Z" }, - { url = "https://files.pythonhosted.org/packages/cc/b6/db007700f67d151abadf508cbfd6a1884f57eab90b1bb985c4c8c02b0f28/cffi-1.17.1-cp312-cp312-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1257bdabf294dceb59f5e70c64a3e2f462c30c7ad68092d01bbbfb1c16b1ba36", size = 454803, upload-time = "2024-09-04T20:44:15.231Z" }, - { url = "https://files.pythonhosted.org/packages/1a/df/f8d151540d8c200eb1c6fba8cd0dfd40904f1b0682ea705c36e6c2e97ab3/cffi-1.17.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:da95af8214998d77a98cc14e3a3bd00aa191526343078b530ceb0bd710fb48a5", size = 478850, upload-time = "2024-09-04T20:44:17.188Z" }, - { url = "https://files.pythonhosted.org/packages/28/c0/b31116332a547fd2677ae5b78a2ef662dfc8023d67f41b2a83f7c2aa78b1/cffi-1.17.1-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:d63afe322132c194cf832bfec0dc69a99fb9bb6bbd550f161a49e9e855cc78ff", size = 485729, upload-time = "2024-09-04T20:44:18.688Z" }, - { url = "https://files.pythonhosted.org/packages/91/2b/9a1ddfa5c7f13cab007a2c9cc295b70fbbda7cb10a286aa6810338e60ea1/cffi-1.17.1-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:f79fc4fc25f1c8698ff97788206bb3c2598949bfe0fef03d299eb1b5356ada99", size = 471256, upload-time = "2024-09-04T20:44:20.248Z" }, - { url = "https://files.pythonhosted.org/packages/b2/d5/da47df7004cb17e4955df6a43d14b3b4ae77737dff8bf7f8f333196717bf/cffi-1.17.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b62ce867176a75d03a665bad002af8e6d54644fad99a3c70905c543130e39d93", size = 479424, upload-time = "2024-09-04T20:44:21.673Z" }, - { url = "https://files.pythonhosted.org/packages/0b/ac/2a28bcf513e93a219c8a4e8e125534f4f6db03e3179ba1c45e949b76212c/cffi-1.17.1-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:386c8bf53c502fff58903061338ce4f4950cbdcb23e2902d86c0f722b786bbe3", size = 484568, upload-time = "2024-09-04T20:44:23.245Z" }, - { url = "https://files.pythonhosted.org/packages/d4/38/ca8a4f639065f14ae0f1d9751e70447a261f1a30fa7547a828ae08142465/cffi-1.17.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:4ceb10419a9adf4460ea14cfd6bc43d08701f0835e979bf821052f1805850fe8", size = 488736, upload-time = "2024-09-04T20:44:24.757Z" }, - { url = "https://files.pythonhosted.org/packages/86/c5/28b2d6f799ec0bdecf44dced2ec5ed43e0eb63097b0f58c293583b406582/cffi-1.17.1-cp312-cp312-win32.whl", hash = "sha256:a08d7e755f8ed21095a310a693525137cfe756ce62d066e53f502a83dc550f65", size = 172448, upload-time = "2024-09-04T20:44:26.208Z" }, - { url = "https://files.pythonhosted.org/packages/50/b9/db34c4755a7bd1cb2d1603ac3863f22bcecbd1ba29e5ee841a4bc510b294/cffi-1.17.1-cp312-cp312-win_amd64.whl", hash = "sha256:51392eae71afec0d0c8fb1a53b204dbb3bcabcb3c9b807eedf3e1e6ccf2de903", size = 181976, upload-time = "2024-09-04T20:44:27.578Z" }, - { url = "https://files.pythonhosted.org/packages/8d/f8/dd6c246b148639254dad4d6803eb6a54e8c85c6e11ec9df2cffa87571dbe/cffi-1.17.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:f3a2b4222ce6b60e2e8b337bb9596923045681d71e5a082783484d845390938e", size = 182989, upload-time = "2024-09-04T20:44:28.956Z" }, - { url = "https://files.pythonhosted.org/packages/8b/f1/672d303ddf17c24fc83afd712316fda78dc6fce1cd53011b839483e1ecc8/cffi-1.17.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:0984a4925a435b1da406122d4d7968dd861c1385afe3b45ba82b750f229811e2", size = 178802, upload-time = "2024-09-04T20:44:30.289Z" }, - { url = "https://files.pythonhosted.org/packages/0e/2d/eab2e858a91fdff70533cab61dcff4a1f55ec60425832ddfdc9cd36bc8af/cffi-1.17.1-cp313-cp313-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d01b12eeeb4427d3110de311e1774046ad344f5b1a7403101878976ecd7a10f3", size = 454792, upload-time = "2024-09-04T20:44:32.01Z" }, - { url = "https://files.pythonhosted.org/packages/75/b2/fbaec7c4455c604e29388d55599b99ebcc250a60050610fadde58932b7ee/cffi-1.17.1-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:706510fe141c86a69c8ddc029c7910003a17353970cff3b904ff0686a5927683", size = 478893, upload-time = "2024-09-04T20:44:33.606Z" }, - { url = "https://files.pythonhosted.org/packages/4f/b7/6e4a2162178bf1935c336d4da8a9352cccab4d3a5d7914065490f08c0690/cffi-1.17.1-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:de55b766c7aa2e2a3092c51e0483d700341182f08e67c63630d5b6f200bb28e5", size = 485810, upload-time = "2024-09-04T20:44:35.191Z" }, - { url = "https://files.pythonhosted.org/packages/c7/8a/1d0e4a9c26e54746dc08c2c6c037889124d4f59dffd853a659fa545f1b40/cffi-1.17.1-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:c59d6e989d07460165cc5ad3c61f9fd8f1b4796eacbd81cee78957842b834af4", size = 471200, upload-time = "2024-09-04T20:44:36.743Z" }, - { url = "https://files.pythonhosted.org/packages/26/9f/1aab65a6c0db35f43c4d1b4f580e8df53914310afc10ae0397d29d697af4/cffi-1.17.1-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:dd398dbc6773384a17fe0d3e7eeb8d1a21c2200473ee6806bb5e6a8e62bb73dd", size = 479447, upload-time = "2024-09-04T20:44:38.492Z" }, - { url = "https://files.pythonhosted.org/packages/5f/e4/fb8b3dd8dc0e98edf1135ff067ae070bb32ef9d509d6cb0f538cd6f7483f/cffi-1.17.1-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:3edc8d958eb099c634dace3c7e16560ae474aa3803a5df240542b305d14e14ed", size = 484358, upload-time = "2024-09-04T20:44:40.046Z" }, - { url = "https://files.pythonhosted.org/packages/f1/47/d7145bf2dc04684935d57d67dff9d6d795b2ba2796806bb109864be3a151/cffi-1.17.1-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:72e72408cad3d5419375fc87d289076ee319835bdfa2caad331e377589aebba9", size = 488469, upload-time = "2024-09-04T20:44:41.616Z" }, - { url = "https://files.pythonhosted.org/packages/bf/ee/f94057fa6426481d663b88637a9a10e859e492c73d0384514a17d78ee205/cffi-1.17.1-cp313-cp313-win32.whl", hash = "sha256:e03eab0a8677fa80d646b5ddece1cbeaf556c313dcfac435ba11f107ba117b5d", size = 172475, upload-time = "2024-09-04T20:44:43.733Z" }, - { url = "https://files.pythonhosted.org/packages/7c/fc/6a8cb64e5f0324877d503c854da15d76c1e50eb722e320b15345c4d0c6de/cffi-1.17.1-cp313-cp313-win_amd64.whl", hash = "sha256:f6a16c31041f09ead72d69f583767292f750d24913dadacf5756b966aacb3f1a", size = 182009, upload-time = "2024-09-04T20:44:45.309Z" }, -] - [[package]] name = "cffi" version = "2.0.0" source = { registry = "https://pypi.org/simple" } -resolution-markers = [ - "python_full_version >= '3.15' and platform_python_implementation != 'PyPy'", - "python_full_version == '3.14.*' and platform_python_implementation != 'PyPy'", - "python_full_version == '3.13.*' and platform_python_implementation != 'PyPy'", - "python_full_version == '3.12.*'", - "python_full_version < '3.12'", -] dependencies = [ { name = "pycparser", marker = "implementation_name != 'PyPy'" }, ] @@ -781,7 +713,7 @@ name = "cryptography" version = "50.0.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "cffi", version = "2.0.0", source = { registry = "https://pypi.org/simple" }, marker = "platform_python_implementation != 'PyPy'" }, + { name = "cffi", marker = "platform_python_implementation != 'PyPy'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/de/41/6cbdcf9142d00fe82836fbb51e503e58088575cf7a0fe1dbff6695bf0840/cryptography-50.0.0.tar.gz", hash = "sha256:eeac2acb5a20ed25e0ad6d1df9891a520b78b404266b6d11778f25d5d691a6c9", size = 880201, upload-time = "2026-07-31T14:25:10.11Z" } wheels = [ @@ -1878,8 +1810,7 @@ name = "lz4" version = "4.4.4" source = { registry = "https://pypi.org/simple" } resolution-markers = [ - "python_full_version == '3.13.*' and platform_python_implementation != 'PyPy'", - "python_full_version == '3.13.*' and platform_python_implementation == 'PyPy'", + "python_full_version == '3.13.*'", "python_full_version == '3.12.*'", "python_full_version < '3.12'", ] @@ -1916,10 +1847,8 @@ name = "lz4" version = "4.4.5" source = { registry = "https://pypi.org/simple" } resolution-markers = [ - "python_full_version >= '3.15' and platform_python_implementation == 'PyPy'", - "python_full_version >= '3.15' and platform_python_implementation != 'PyPy'", - "python_full_version == '3.14.*' and platform_python_implementation == 'PyPy'", - "python_full_version == '3.14.*' and platform_python_implementation != 'PyPy'", + "python_full_version >= '3.15'", + "python_full_version == '3.14.*'", ] sdist = { url = "https://files.pythonhosted.org/packages/57/51/f1b86d93029f418033dddf9b9f79c8d2641e7454080478ee2aab5123173e/lz4-4.4.5.tar.gz", hash = "sha256:5f0b9e53c1e82e88c10d7c180069363980136b9d7a8306c4dca4f760d60c39f0", size = 172886, upload-time = "2025-11-03T13:02:36.061Z" } wheels = [ @@ -3291,8 +3220,7 @@ name = "pyzmq" version = "27.1.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "cffi", version = "1.17.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.13' and implementation_name == 'pypy' and platform_python_implementation == 'PyPy'" }, - { name = "cffi", version = "2.0.0", source = { registry = "https://pypi.org/simple" }, marker = "(python_full_version < '3.13' and implementation_name == 'pypy') or (implementation_name == 'pypy' and platform_python_implementation != 'PyPy')" }, + { name = "cffi", marker = "implementation_name == 'pypy'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/04/0b/3c9baedbdf613ecaa7aa07027780b8867f57b6293b6ee50de316c9f3222b/pyzmq-27.1.0.tar.gz", hash = "sha256:ac0765e3d44455adb6ddbf4417dcce460fc40a05978c08efdf2948072f6db540", size = 281750, upload-time = "2025-09-08T23:10:18.157Z" } wheels = [ @@ -3626,6 +3554,7 @@ dependencies = [ { name = "jinja2" }, { name = "pydantic" }, { name = "pyyaml" }, + { name = "sidemantic-rs", marker = "sys_platform != 'emscripten'" }, { name = "sqlglot" }, { name = "typer" }, ] @@ -3848,6 +3777,7 @@ requires-dist = [ { name = "sidemantic", extras = ["postgres", "bigquery", "snowflake", "clickhouse", "databricks", "spark", "adbc"], marker = "extra == 'all-databases'" }, { name = "sidemantic", extras = ["workbench", "mcp", "apps", "charts", "lsp", "dax", "lookml", "malloy", "metricflow", "widget", "api", "ossie"], marker = "extra == 'full'" }, { name = "sidemantic-dax", marker = "extra == 'dax'", directory = "crates/dax-pyo3" }, + { name = "sidemantic-rs", marker = "sys_platform != 'emscripten'", directory = "sidemantic-rs" }, { name = "snowflake-connector-python", marker = "extra == 'snowflake'", specifier = ">=3.0.0" }, { name = "sqlglot", specifier = ">=30.1.0" }, { name = "sqlglot", extras = ["c"], marker = "extra == 'dev'", specifier = ">=30.1.0" }, @@ -3891,6 +3821,11 @@ name = "sidemantic-dax" version = "0.12.0" source = { directory = "crates/dax-pyo3" } +[[package]] +name = "sidemantic-rs" +version = "0.12.0" +source = { directory = "sidemantic-rs" } + [[package]] name = "six" version = "1.17.0"