diff --git a/.gitignore b/.gitignore index 1520277..69db8f7 100644 --- a/.gitignore +++ b/.gitignore @@ -39,3 +39,6 @@ content/stack/reference/drizzle/latest/ content/stack/reference/drizzle/v*/ .tmp-* .env.local + +# EQL release manifest extracted by generate-eql-docs.ts (see generate-eql-api-docs.ts) +.eql-manifest.release.json diff --git a/content/docs/reference/eql/booleans.mdx b/content/docs/reference/eql/booleans.mdx index 5403fc9..3cbbf73 100644 --- a/content/docs/reference/eql/booleans.mdx +++ b/content/docs/reference/eql/booleans.mdx @@ -1,13 +1,13 @@ --- title: Booleans -description: "Encrypted booleans are storage-only by design: eql_v3.bool stores and decrypts, carries no index terms, and blocks every comparison." +description: "Encrypted booleans are storage-only by design: public.boolean stores and decrypts, carries no index terms, and blocks every comparison." type: reference components: [eql] verifiedAgainst: eql: "3.0.0" --- -Every scalar type has a storage-only variant — for `bool` it's the only one. EQL ships `eql_v3.bool` and nothing else: there is no `bool_eq` and no `bool_ord`. An encrypted boolean column can be stored, decrypted, and null-checked; it cannot be filtered, sorted, grouped, or joined on. +Every scalar type has a storage-only variant — for `bool` it's the only one. EQL ships `public.boolean` and nothing else: there is no `bool_eq` and no `bool_ord`. An encrypted boolean column can be stored, decrypted, and null-checked; it cannot be filtered, sorted, grouped, or joined on. ## Why there are no query variants @@ -15,11 +15,11 @@ A two-value column has too little cardinality for any searchable index to be saf ## What works, what raises -`eql_v3.bool` follows the bare-variant contract described in [Core concepts](/reference/eql/core-concepts#variants-declare-capability): it carries no index terms, so `IS NULL` / `IS NOT NULL` are the only predicates that work. Every comparison operator routes to a blocker and raises — the [fail-loud behavior](/reference/eql/core-concepts#unsupported-operations-fail-loudly) shared by all encrypted variants: +`public.boolean` follows the bare-variant contract described in [Core concepts](/reference/eql/core-concepts#variants-declare-capability): it carries no index terms, so `IS NULL` / `IS NOT NULL` are the only predicates that work. Every comparison operator routes to a blocker and raises — the [fail-loud behavior](/reference/eql/core-concepts#unsupported-operations-fail-loudly) shared by all encrypted variants: ```sql --- ❌ Raises: operator = is not supported for eql_v3.bool -SELECT * FROM users WHERE is_active = $1::eql_v3.bool; +-- ❌ Raises: operator = is not supported for public.boolean +SELECT * FROM users WHERE is_active = $1::public.boolean; -- ✅ Works: NULL columns are not encrypted SELECT * FROM users WHERE is_active IS NOT NULL; @@ -32,16 +32,16 @@ Query on other columns, decrypt the boolean in your application, and filter ther ```sql CREATE TABLE users ( id bigint GENERATED ALWAYS AS IDENTITY PRIMARY KEY, - email eql_v3.text_eq, -- exact lookup - created_at eql_v3.timestamp_ord, -- range queries, ORDER BY - is_active eql_v3.bool -- storage only (by design) + email public.text_eq, -- exact lookup + created_at public.timestamp_ord, -- range queries, ORDER BY + is_active public.boolean -- storage only (by design) ); ``` ```sql -- Narrow the result set with the columns that do carry index terms… SELECT id, email, is_active FROM users -WHERE created_at >= $1::eql_v3.timestamp_ord; +WHERE created_at >= $1::public.timestamp_ord; -- …then decrypt is_active in the client and filter on the plaintext. ``` @@ -51,12 +51,12 @@ If a boolean genuinely needs to be a server-side predicate, that is a data-model ## Storing without querying -`bool` is the forced case of a pattern available to every scalar type: the bare variant `eql_v3.` (for example `eql_v3.int4`, `eql_v3.text`, `eql_v3.timestamp`) is storage-and-decryption only. It carries no index terms, and every comparison operator raises — use it for columns you only ever store and decrypt, so the database holds no searchable material for them at all. +`bool` is the forced case of a pattern available to every scalar type: the bare variant `public.` (for example `public.integer`, `public.text`, `public.timestamp`) is storage-and-decryption only. It carries no index terms, and every comparison operator raises — use it for columns you only ever store and decrypt, so the database holds no searchable material for them at all. For every type other than `bool`, storage-only is a choice you can walk back. If you later need to query, retype the column as a query variant — or, if the payloads already carry the needed term (the client decides which terms travel in the payload), cast at the call site: ```sql -SELECT * FROM readings WHERE value::eql_v3.int4_ord > $1::eql_v3.int4_ord; +SELECT * FROM readings WHERE value::public.integer_ord > $1::public.integer_ord; ``` The variant families and what each one enables are covered in [Core concepts](/reference/eql/core-concepts); the per-type specifics live in [Numbers](/reference/eql/numbers), [Dates & times](/reference/eql/dates-and-times), and [Text](/reference/eql/text). diff --git a/content/docs/reference/eql/core-concepts.mdx b/content/docs/reference/eql/core-concepts.mdx index 427bc7b..ffdd927 100644 --- a/content/docs/reference/eql/core-concepts.mdx +++ b/content/docs/reference/eql/core-concepts.mdx @@ -11,7 +11,7 @@ Everything in the EQL reference builds on four ideas: columns are typed as **dom ## Variants declare capability -EQL ships its searchable-encryption surface as PostgreSQL **domains in the `eql_v3` schema**, all backed by `jsonb`. Each scalar type generates a *family* of domain variants, and the variant you type a column as fixes its query capability. Each domain carries a `CHECK` constraint that validates the encrypted payload on insert, so a malformed or wrong-version value is rejected at write time rather than surfacing at query time. +EQL ships its searchable-encryption surface as PostgreSQL **domains in the `public` schema**, all backed by `jsonb` (the functions behind them live in `eql_v3` — [The three schemas](#the-three-schemas) explains the split). Each scalar type generates a *family* of domain variants, and the variant you type a column as fixes its query capability. Each domain carries a `CHECK` constraint that validates the encrypted payload on insert, so a malformed or wrong-version value is rejected at write time rather than surfacing at query time. There is no database-side configuration table. Earlier EQL versions tracked encryption config in the database (`config_add_table`, `config_add_column`, and friends) — those are gone in v3. The searchable surface of a column is fixed by the domain variant you type it as, and which index terms travel in a value's payload is decided by the encryption client (the [Stack SDK](/reference/stack) or [CipherStash Proxy](/reference/proxy)). The domain makes the matching operators resolve; the term in the payload is what makes them answer. @@ -19,24 +19,24 @@ For any scalar type ``, the family looks like this: | Domain variant | Capability | | --- | --- | -| `eql_v3.` | Storage and decryption only. | -| `eql_v3._eq` | Equality: `=`, `<>`, `IN`, `GROUP BY`, `DISTINCT`, equijoins. | -| `eql_v3._ord` | Comparisons (`<` … `>=`), `BETWEEN`, `ORDER BY`, `MIN` / `MAX` — plus equality. | -| `eql_v3._ord_ore` | As `_ord`, with the ORE mechanism pinned — see [SEM specifiers](#sem-specifiers). | -| `eql_v3.text_match` (text only) | Free-text token containment: `@>` / `<@`. | -| `eql_v3.text_search` (text only) | Equality + ordering + token containment. | +| `public.` | Storage and decryption only. | +| `public._eq` | Equality: `=`, `<>`, `IN`, `GROUP BY`, `DISTINCT`, equijoins. | +| `public._ord` | Comparisons (`<` … `>=`), `BETWEEN`, `ORDER BY`, `MIN` / `MAX` — plus equality. | +| `public._ord_ore` | As `_ord`, with the ORE mechanism pinned — see [SEM specifiers](#sem-specifiers). | +| `public.text_match` (text only) | Free-text token containment: `@>` / `<@`. | +| `public.text_search` (text only) | Equality + ordering + token containment. | Two things worth calling out: -- **The bare variant blocks everything.** `eql_v3.` carries no index term. Querying it with any comparison operator raises an "operator not supported" exception. Use it for columns you only ever store and decrypt — [Booleans](/reference/eql/booleans) covers this pattern in full. +- **The bare variant blocks everything.** `public.` carries no index term. Querying it with any comparison operator raises an "operator not supported" exception. Use it for columns you only ever store and decrypt — [Booleans](/reference/eql/booleans) covers this pattern in full. - **Which index term backs each capability** is an implementation detail of the payload — covered in [Anatomy of an encrypted value](#anatomy-of-an-encrypted-value) below. ### SEM specifiers A trailing mechanism suffix — the `_ore` in `_ord_ore` — is a **SEM specifier**: it pins *which* searchable-encryption mechanism implements the capability, rather than just declaring the capability itself. -- `eql_v3._ord` declares *orderable* and leaves the mechanism to EQL's default — currently ORE (order-revealing encryption). -- `eql_v3._ord_ore` declares *orderable via ORE, explicitly*. Today the two are byte-identical surfaces backed by the same term. +- `public._ord` declares *orderable* and leaves the mechanism to EQL's default — currently ORE (order-revealing encryption). +- `public._ord_ore` declares *orderable via ORE, explicitly*. Today the two are byte-identical surfaces backed by the same term. The distinction earns its keep as mechanisms multiply: the EQL v3 release adds an **OPE** (order-preserving encryption) specifier for every orderable type — including `text` — at which point pinning a specifier documents and freezes a column's mechanism choice, while unspecified variants track the default. Each type page lists its available specifiers under an "SEM specifiers" heading. @@ -45,13 +45,29 @@ Declaring a table is just typing each column as the variant it needs: ```sql CREATE TABLE users ( id BIGINT GENERATED ALWAYS AS IDENTITY PRIMARY KEY, - email eql_v3.text_eq, -- equality only - salary eql_v3.int4_ord, -- equality + range + ORDER BY - created_at eql_v3.timestamp_ord + email public.text_eq, -- equality only + salary public.integer_ord, -- equality + range + ORDER BY + created_at public.timestamp_ord ); ``` -Every scalar type — `int2`, `int4`, `int8`, `numeric`, `float4`, `float8`, `date`, `timestamp`, `text`, and `bool` in EQL 3.0.0 — ships some subset of this family. The per-category pages list exactly which variants each type has and how to choose between them: [Numbers](/reference/eql/numbers), [Dates & times](/reference/eql/dates-and-times), [Text](/reference/eql/text), and [Booleans](/reference/eql/booleans). Encrypted JSON documents use a separate domain, `eql_v3.json`, with its own operator surface — see [JSON](/reference/eql/json). +Every scalar type — `int2`, `int4`, `int8`, `numeric`, `float4`, `float8`, `date`, `timestamp`, `text`, and `bool` in EQL 3.0.0 — ships some subset of this family. The per-category pages list exactly which variants each type has and how to choose between them: [Numbers](/reference/eql/numbers), [Dates & times](/reference/eql/dates-and-times), [Text](/reference/eql/text), and [Booleans](/reference/eql/booleans). Encrypted JSON documents use a separate domain, `public.json`, with its own operator surface — see [JSON](/reference/eql/json). + +## The three schemas + +EQL spreads its surface across three PostgreSQL schemas, and the split is what makes EQL **upgradable in place**: + +| Schema | Holds | Do you call it? | +| --- | --- | --- | +| `public` | The encrypted **domain types** — every `public.` variant you type a column as. | Referenced in your table DDL. | +| `eql_v3` | All **user-callable functions and operators** — the searchable-encryption API (`eql_v3.eq_term`, `eql_v3.jsonb_path_query`, the encrypted `=` / `<` / `@>` operators, `eql_v3.version()`). | Yes — this is the API. | +| `eql_v3_internal` | The **implementation functions** the domains and operators are built from. | No — never call these directly. | + +**Why the types live in `public`.** Your columns are typed as `public.integer_ord`, `public.text_eq`, and so on — never `eql_v3.*`. Keeping the types in the unversioned `public` schema means an EQL upgrade never rewrites your table definitions: the logic ships in a *versioned* schema (`eql_v3` today, a future `eql_v4` alongside it tomorrow) while the type names your schema depends on stay put. `eql_v2` was removed wholesale in 3.0.0 without any `public.*` type changing. + +**Why `eql_v3` is versioned.** The schema name encodes the major API version and is itself part of the public contract — a breaking change introduces a new `eql_vN` schema *beside* the old one rather than mutating it, so you migrate on your own timeline. Everything in `eql_v3` is fair game to call. + +**Why `eql_v3_internal` is off-limits.** These are the building blocks — term comparators, unsupported-operator blockers, cast helpers — that the operators and domain `CHECK` constraints wire together. They carry no stability guarantee and can change or disappear between releases. If you're reaching for `eql_v3_internal.*`, there's a `public` type or an `eql_v3` function that does what you want. ## Anatomy of an encrypted value @@ -77,17 +93,17 @@ A `k` discriminator (`"ct"` for a scalar ciphertext, `"sv"` for a JSON document) ### Index-term keys -Alongside the envelope, a payload carries the index terms for its column's capability. Each key is backed by a SEM (searchable encrypted metadata) type in the `eql_v3` schema: +Alongside the envelope, a payload carries the index terms for its column's capability. Each key is backed by a SEM (searchable encrypted metadata) type in the `eql_v3_internal` schema: | Key | SEM type | Wire shape | Enables | Reveals | | --- | --- | --- | --- | --- | -| `hm` | `eql_v3.hmac_256` (domain over `text`) | Hex string (HMAC-SHA-256) | `=`, `<>` on `_eq` and `text_search` domains | Whether two values are equal — nothing else | -| `ob` | `eql_v3.ore_block_256` (composite: array of `bytea` block terms) | Array of hex-encoded ORE blocks (block count varies by scalar width) | `<`, `<=`, `>`, `>=`, `ORDER BY` on `_ord` / `_ord_ore` domains — and `=` / `<>`, since ORE comparison collapses to equality | The relative order of two values | -| `bf` | `eql_v3.bloom_filter` (domain over `smallint[]`) | Array of set bit positions (**signed** 16-bit — large filters emit negative positions) | `@>` / `<@` token containment on `_match` domains | Probabilistic token overlap between values | +| `hm` | `eql_v3_internal.hmac_256` (domain over `text`) | Hex string (HMAC-SHA-256) | `=`, `<>` on `_eq` and `text_search` domains | Whether two values are equal — nothing else | +| `ob` | `eql_v3_internal.ore_block_256` (composite: array of `bytea` block terms) | Array of hex-encoded ORE blocks (block count varies by scalar width) | `<`, `<=`, `>`, `>=`, `ORDER BY` on `_ord` / `_ord_ore` domains — and `=` / `<>`, since ORE comparison collapses to equality | The relative order of two values | +| `bf` | `eql_v3_internal.bloom_filter` (domain over `smallint[]`) | Array of set bit positions (**signed** 16-bit — large filters emit negative positions) | `@>` / `<@` token containment on `_match` domains | Probabilistic token overlap between values | -The capability is encoded as **required keys**: the payload for an `eql_v3.text_eq` column must carry `hm`; an `eql_v3.int4_ord` payload must carry `ob` (and only `ob`); a `text_match` payload must carry `bf`; a `text_search` payload carries all three. A payload missing its term key fails the domain `CHECK` — and fails to deserialize in the client bindings. +The capability is encoded as **required keys**: the payload for a `public.text_eq` column must carry `hm`; a `public.integer_ord` payload must carry `ob` (and only `ob`); a `text_match` payload must carry `bf`; a `text_search` payload carries all three. A payload missing its term key fails the domain `CHECK` — and fails to deserialize in the client bindings. -A scalar payload for an `eql_v3.text_search` column (lookup + ordering + free-text match, so all three terms are required): +A scalar payload for a `public.text_search` column (lookup + ordering + free-text match, so all three terms are required): ```json { @@ -124,10 +140,10 @@ The `eql_v3` domains are backed by `jsonb`. When an operand has no known type SELECT * FROM users WHERE email = $1; -- ✅ Right: typed operand — the encrypted `=` resolves. -SELECT * FROM users WHERE email = $1::eql_v3.text_eq; +SELECT * FROM users WHERE email = $1::public.text_eq; ``` -Always type the operand: a typed parameter (`$1::eql_v3.text_eq`) or an explicit cast (`'…'::eql_v3.int4_ord`). The [Stack SDK](/reference/stack) and [CipherStash Proxy](/reference/proxy) type bound parameters automatically — raw SQL must do it by hand. +Always type the operand: a typed parameter (`$1::public.text_eq`) or an explicit cast (`'…'::public.integer_ord`). The [Stack SDK](/reference/stack) and [CipherStash Proxy](/reference/proxy) type bound parameters automatically — raw SQL must do it by hand. This is the one place where a mistake is *silent*. Everything else fails loudly: @@ -136,9 +152,9 @@ This is the one place where a mistake is *silent*. Everything else fails loudly: Unsupported operators are not silent no-ops. Every operator that a variant doesn't support is still *defined* — it routes to a blocker function that raises an `operator … is not supported` exception. A mis-typed query fails loudly instead of silently returning wrong results: ```sql --- salary is eql_v3.int8_eq (equality only) -SELECT * FROM users WHERE salary > $1::eql_v3.int8_eq; --- ERROR: operator > is not supported for eql_v3.int8_eq +-- salary is public.bigint_eq (equality only) +SELECT * FROM users WHERE salary > $1::public.bigint_eq; +-- ERROR: operator > is not supported for public.bigint_eq ``` A `NULL` operand still raises — the blockers are deliberately not `STRICT`, so PostgreSQL can't skip the check. (A SQL `NULL` column value is not encrypted, so `IS NULL` / `IS NOT NULL` themselves always work, on every variant.) diff --git a/content/docs/reference/eql/dates-and-times.mdx b/content/docs/reference/eql/dates-and-times.mdx index 3af2b9c..1635a89 100644 --- a/content/docs/reference/eql/dates-and-times.mdx +++ b/content/docs/reference/eql/dates-and-times.mdx @@ -15,17 +15,17 @@ Both types generate the same `jsonb`-backed domain variants. The generic form: | Domain variant | Capability | | --- | --- | -| `eql_v3.` | Storage and decryption only. | -| `eql_v3._eq` | Equality: `=`, `<>`, `IN`, `GROUP BY`, `DISTINCT`, equijoins. | -| `eql_v3._ord` | Comparisons, `BETWEEN`, `ORDER BY`, `MIN` / `MAX` — plus equality. | -| `eql_v3._ord_ore` | As `_ord`, with the ORE mechanism pinned — see [SEM specifiers](#sem-specifiers). | +| `public.` | Storage and decryption only. | +| `public._eq` | Equality: `=`, `<>`, `IN`, `GROUP BY`, `DISTINCT`, equijoins. | +| `public._ord` | Comparisons, `BETWEEN`, `ORDER BY`, `MIN` / `MAX` — plus equality. | +| `public._ord_ore` | As `_ord`, with the ORE mechanism pinned — see [SEM specifiers](#sem-specifiers). | And every concrete domain this page covers: | Type | Variants | | --- | --- | -| `date` | `eql_v3.date` · `eql_v3.date_eq` · `eql_v3.date_ord` · `eql_v3.date_ord_ore` | -| `timestamp` | `eql_v3.timestamp` · `eql_v3.timestamp_eq` · `eql_v3.timestamp_ord` · `eql_v3.timestamp_ord_ore` | +| `date` | `public.date` · `public.date_eq` · `public.date_ord` · `public.date_ord_ore` | +| `timestamp` | `public.timestamp` · `public.timestamp_eq` · `public.timestamp_ord` · `public.timestamp_ord_ore` | Time columns are nearly always ranged and sorted, so `_ord` is the usual choice. Declare only the capability you query on — each capability stores extra searchable material with defined leakage (see [Searchable encryption](/concepts/searchable-encryption)), and the variant model itself is covered in [Core concepts](/reference/eql/core-concepts). @@ -36,9 +36,9 @@ An audit-events table where the timestamps drive time-window queries and sorting ```sql CREATE TABLE audit_events ( id bigint GENERATED ALWAYS AS IDENTITY PRIMARY KEY, - occurred_at eql_v3.timestamp_ord, -- time windows, newest-first, MIN/MAX - review_due eql_v3.date_ord, -- range filters - sealed_on eql_v3.date -- store and decrypt only + occurred_at public.timestamp_ord, -- time windows, newest-first, MIN/MAX + review_due public.date_ord, -- range filters + sealed_on public.date -- store and decrypt only ); ``` @@ -55,7 +55,7 @@ The EQL v3 release adds an OPE specifier for every orderable type; unspecified ` ## Payload -A value for an `_ord` column carries the shared envelope keys (`v`, `i`, `c` — see [Core concepts](/reference/eql/core-concepts)) plus the `ob` ordering term. Here is a payload for the `eql_v3.timestamp_ord` `occurred_at` column: +A value for an `_ord` column carries the shared envelope keys (`v`, `i`, `c` — see [Core concepts](/reference/eql/core-concepts)) plus the `ob` ordering term. Here is a payload for the `public.timestamp_ord` `occurred_at` column: ```json { @@ -75,7 +75,7 @@ A value for an `_ord` column carries the shared envelope keys (`v`, `i`, `c` — ## Operators -| SQL operator | `eql_v3.` | `_eq` | `_ord` / `_ord_ore` | +| SQL operator | `public.` | `_eq` | `_ord` / `_ord_ore` | | --- | :---: | :---: | :---: | | `=` / `<>` | ❌ | ✅ | ✅ | | `<` `<=` `>` `>=` | ❌ | ❌ | ✅ | @@ -85,7 +85,7 @@ A value for an `_ord` column carries the shared envelope keys (`v`, `i`, `c` — | `ORDER BY` | ❌ | ❌ | ✅ | | `IS NULL` / `IS NOT NULL` | ✅ | ✅ | ✅ | -Blocked *operator* cells raise an `operator … is not supported` exception — they never silently return wrong rows. `ORDER BY` is the one blocked cell that doesn't raise: it isn't an operator, so sorting a variant without an ordering term runs — but the order is meaningless (see [Sorting](/reference/eql/sorting)). Operands must be typed (`$1::eql_v3.timestamp_ord`), or PostgreSQL resolves the native `jsonb` operator instead of the encrypted one. Both rules are covered in [Core concepts](/reference/eql/core-concepts). +Blocked *operator* cells raise an `operator … is not supported` exception — they never silently return wrong rows. `ORDER BY` is the one blocked cell that doesn't raise: it isn't an operator, so sorting a variant without an ordering term runs — but the order is meaningless (see [Sorting](/reference/eql/sorting)). Operands must be typed (`$1::public.timestamp_ord`), or PostgreSQL resolves the native `jsonb` operator instead of the encrypted one. Both rules are covered in [Core concepts](/reference/eql/core-concepts). ## Functions @@ -103,17 +103,17 @@ Every operator has a function form, for managed platforms that disallow custom o ```sql SELECT * FROM audit_events -WHERE occurred_at BETWEEN $1::eql_v3.timestamp_ord AND $2::eql_v3.timestamp_ord; +WHERE occurred_at BETWEEN $1::public.timestamp_ord AND $2::public.timestamp_ord; SELECT * FROM audit_events -WHERE review_due BETWEEN $1::eql_v3.date_ord AND $2::eql_v3.date_ord; +WHERE review_due BETWEEN $1::public.date_ord AND $2::public.date_ord; ``` ### Retention cutoff ```sql SELECT id FROM audit_events -WHERE occurred_at < $1::eql_v3.timestamp_ord; +WHERE occurred_at < $1::public.timestamp_ord; ``` ### Newest-first listing @@ -122,7 +122,7 @@ Write the sort key in extractor form to stream rows out of the index already ord ```sql SELECT * FROM audit_events -WHERE occurred_at >= $1::eql_v3.timestamp_ord +WHERE occurred_at >= $1::public.timestamp_ord ORDER BY eql_v3.ord_term(occurred_at) DESC LIMIT 10; ``` diff --git a/content/docs/reference/eql/filtering.mdx b/content/docs/reference/eql/filtering.mdx index 9fe45a2..9dc4394 100644 --- a/content/docs/reference/eql/filtering.mdx +++ b/content/docs/reference/eql/filtering.mdx @@ -7,25 +7,25 @@ verifiedAgainst: eql: "3.0.0" --- -Every filter below is ordinary SQL — the encrypted operators resolve from the column's domain variant, and a functional index on the matching term extractor serves the predicate. One rule applies throughout: **operands must be typed** (`$1::eql_v3.text_eq`, not a bare literal), or PostgreSQL falls through to native `jsonb` semantics. See [Core concepts](/reference/eql/core-concepts) for the typed-operand rule and how unsupported operators fail loudly instead of returning wrong rows. +Every filter below is ordinary SQL — the encrypted operators resolve from the column's domain variant, and a functional index on the matching term extractor serves the predicate. One rule applies throughout: **operands must be typed** (`$1::public.text_eq`, not a bare literal), or PostgreSQL falls through to native `jsonb` semantics. See [Core concepts](/reference/eql/core-concepts) for the typed-operand rule and how unsupported operators fail loudly instead of returning wrong rows. ## Equality: `=` and `<>` Works on `_eq` and `_ord` / `_ord_ore` variants of every scalar, and on `text_search`: ```sql -SELECT * FROM users WHERE email = $1::eql_v3.text_eq; -SELECT * FROM users WHERE tax_id <> $1::eql_v3.text_eq; +SELECT * FROM users WHERE email = $1::public.text_eq; +SELECT * FROM users WHERE tax_id <> $1::public.text_eq; ``` On `_eq` and `text_search` columns equality compares the HMAC (`hm`) term. On `_ord` variants there is no `hm` — equality compares the ORE (`ob`) term, which collapses to equality, so `_ord` columns get `=` and `<>` for free. See [Core concepts](/reference/eql/core-concepts) for the mechanism. ```sql --- salary is eql_v3.int8_ord: equality works without an hm term -SELECT * FROM users WHERE salary = $1::eql_v3.int8_ord; +-- salary is public.bigint_ord: equality works without an hm term +SELECT * FROM users WHERE salary = $1::public.bigint_ord; ``` -Bare storage-only variants (`eql_v3.text`, `eql_v3.int4`, …) block every comparison — see the type pages for what each variant supports: [Numbers](/reference/eql/numbers), [Dates & times](/reference/eql/dates-and-times), [Text](/reference/eql/text), [Booleans](/reference/eql/booleans). +Bare storage-only variants (`public.text`, `public.integer`, …) block every comparison — see the type pages for what each variant supports: [Numbers](/reference/eql/numbers), [Dates & times](/reference/eql/dates-and-times), [Text](/reference/eql/text), [Booleans](/reference/eql/booleans). ## `IN` lists @@ -33,7 +33,7 @@ Bare storage-only variants (`eql_v3.text`, `eql_v3.int4`, …) block every compa ```sql SELECT * FROM users -WHERE email IN ($1::eql_v3.text_eq, $2::eql_v3.text_eq, $3::eql_v3.text_eq); +WHERE email IN ($1::public.text_eq, $2::public.text_eq, $3::public.text_eq); ``` There is no way to encrypt a list as one value — the client encrypts each element and binds it as its own parameter. `IN (subquery)` also works, subject to the same-keyset rule covered in [Joins](/reference/eql/joins). @@ -43,19 +43,19 @@ There is no way to encrypt a list as one value — the client encrypts each elem `<`, `<=`, `>`, `>=` work on `_ord` / `_ord_ore` variants and `text_search` — the variants carrying an ORE (`ob`) term: ```sql -SELECT * FROM users WHERE salary >= $1::eql_v3.int8_ord; +SELECT * FROM users WHERE salary >= $1::public.bigint_ord; -- BETWEEN desugars to >= and <= SELECT * FROM users -WHERE created_at BETWEEN $1::eql_v3.timestamp_ord AND $2::eql_v3.timestamp_ord; +WHERE created_at BETWEEN $1::public.timestamp_ord AND $2::public.timestamp_ord; ``` Half-open ranges compose the same way: ```sql SELECT * FROM events -WHERE occurred_at >= $1::eql_v3.timestamp_ord - AND occurred_at < $2::eql_v3.timestamp_ord; +WHERE occurred_at >= $1::public.timestamp_ord + AND occurred_at < $2::public.timestamp_ord; ``` ## Text token matching: `@>` @@ -63,25 +63,25 @@ WHERE occurred_at >= $1::eql_v3.timestamp_ord There is no `LIKE` on encrypted columns — encrypted free-text matching is bloom-filter token containment via `@>` on a `text_match` or `text_search` column: ```sql -SELECT * FROM users WHERE name @> $1::eql_v3.text_match; +SELECT * FROM users WHERE name @> $1::public.text_match; ``` The client encrypts the search term into a bloom-filter query value; matching is probabilistic (false positives possible, false negatives not). For the full no-`LIKE` story and match-term tuning, see [Text](/reference/eql/text). ## JSON containment and path filters -Encrypted JSON documents (`eql_v3.json`) filter by containment and path existence: +Encrypted JSON documents (`public.json`) filter by containment and path existence: ```sql -- Does the document contain this (encrypted) structure? -SELECT * FROM orders WHERE metadata @> $1::eql_v3.ste_vec_query; +SELECT * FROM orders WHERE metadata @> $1::public.jsonb_query; -- Does this path exist in the document? SELECT * FROM orders WHERE eql_v3.jsonb_path_exists(metadata, 'region_selector'); -- Equality on an extracted leaf SELECT * FROM orders -WHERE metadata -> 'email_selector'::text = $1::eql_v3.ste_vec_entry; +WHERE metadata -> 'email_selector'::text = $1::public.jsonb_entry; ``` Field access is by selector hash, not plaintext path. The full JSON surface — containment, field access, path queries, and range filters on extracted leaves — is in [JSON](/reference/eql/json). @@ -93,9 +93,9 @@ Encrypted predicates compose with `AND`, `OR`, `NOT`, and parentheses like any o ```sql SELECT * FROM users WHERE status = 'active' -- plaintext column, native operator - AND created_at >= $1::eql_v3.timestamp_ord -- encrypted range - AND (email = $2::eql_v3.text_eq -- encrypted equality - OR name @> $3::eql_v3.text_match); -- encrypted token match + AND created_at >= $1::public.timestamp_ord -- encrypted range + AND (email = $2::public.text_eq -- encrypted equality + OR name @> $3::public.text_match); -- encrypted token match ``` The planner treats each encrypted predicate independently, so it can combine an index on a plaintext column with a functional index on an encrypted one (bitmap-AND, or whichever plan is cheapest). @@ -118,7 +118,7 @@ Don't confuse this with a JSON `null` *inside* an encrypted document, which is a | Equality | `=` `<>` `IN` | `_eq`, `_ord` / `_ord_ore`, `text_search` | hash (or btree) on `eql_v3.eq_term` — btree on `eql_v3.ord_term` for `_ord` | | Range | `<` `<=` `>` `>=` `BETWEEN` | `_ord` / `_ord_ore`, `text_search` | btree on `eql_v3.ord_term` | | Text token match | `@>` `<@` | `text_match`, `text_search` | GIN on `eql_v3.match_term` | -| JSON containment | `@>` `<@` | `eql_v3.json` | GIN on `eql_v3.to_ste_vec_query(col)::jsonb` | +| JSON containment | `@>` `<@` | `public.json` | GIN on `eql_v3.to_ste_vec_query(col)::jsonb` | | Null check | `IS NULL` / `IS NOT NULL` | every variant | — | Every one of these has a full index recipe — which method, which extractor, and how to confirm the index engages with `EXPLAIN` — in [Indexes](/reference/eql/indexes). diff --git a/content/docs/reference/eql/functions.mdx b/content/docs/reference/eql/functions.mdx new file mode 100644 index 0000000..e9f86a5 --- /dev/null +++ b/content/docs/reference/eql/functions.mdx @@ -0,0 +1,70 @@ +--- +title: Functions +description: "Generated catalog of EQL SQL functions and operators (EQL 3.0.0-sample)." +type: reference +components: [eql] +verifiedAgainst: + eql: "3.0.0-sample" +--- + +{/* GENERATED — do not edit. Produced by scripts/generate-eql-api-docs.ts from the EQL manifest (v3.0.0-sample). */} + + +Generated from the **EQL 3.0.0-sample** manifest (the Doxygen'd SQL is the source of truth). For the model behind these — variants, terms, typed operands — see [Core concepts](/reference/eql/core-concepts). + + +The EQL SQL surface — encrypted domains (in the `public` schema) and the `eql_v3` functions behind them. The type and query pages explain *when* to use these; this page is the exhaustive reference they link to. + +## Encrypted domains + +A column's capability is declared by its **domain variant**. This matrix comes straight from the Rust catalog (`eql-codegen dump-catalog`) — the source of truth the SQL is generated from. See [Core concepts](/reference/eql/core-concepts) for the model. + +| Domain | Type | Variant | Capabilities | Operators | +| --- | --- | --- | --- | --- | +| `public.integer` | integer | _(storage only)_ | storage | — | +| `public.integer_eq` | integer | `_eq` | equality | `=` `<>` | +| `public.integer_ord` | integer | `_ord` | equality, order | `=` `<>` `<` `<=` `>` `>=` | +| `public.integer_ord_ope` | integer | `_ord_ope` | equality, order | `=` `<>` `<` `<=` `>` `>=` | +| `public.text_match` | text | `_match` | match | `@>` `<@` | +| `public.text_search` | text | `_search` | equality, order, match | `=` `<>` `<` `<=` `>` `>=` `@>` `<@` | +| `public.json` | jsonb | _(storage only)_ | json | — | +| `public.jsonb_entry` | jsonb | _(storage only)_ | json | — | +| `public.jsonb_query` | jsonb | _(storage only)_ | json | — | + +## Functions + +The public `eql_v3` API — 2 functions (2 overloads). Internal `eql_v3_internal` functions (3) are implementation detail and omitted. + +### `jsonb_path_query` + +Query encrypted JSONB for sv elements matching a selector. + +Returns one jsonb_entry row per matching encrypted element. + +**Signature:** + +```sql +jsonb_path_query(jsonb, text) +``` + +| Parameter | Type | Description | +| --- | --- | --- | +| `val` | `jsonb` | encrypted EQL payload with sv | +| `selector` | `text` | selector hash | + + +**Returns:** `public.jsonb_entry` — matching encrypted entries + +### `version` + +Get the installed EQL version string. + +Returns the version string for the installed EQL library. + +**Signature:** + +```sql +version() +``` + +**Returns:** `text` — the version string diff --git a/content/docs/reference/eql/grouping-and-aggregates.mdx b/content/docs/reference/eql/grouping-and-aggregates.mdx index 544a91c..f4db3f3 100644 --- a/content/docs/reference/eql/grouping-and-aggregates.mdx +++ b/content/docs/reference/eql/grouping-and-aggregates.mdx @@ -39,7 +39,7 @@ Note the trade-off: grouping on `eq_term` returns the *term*, not the encrypted Plain `COUNT(col)` counts non-`NULL` rows — it never compares values, so it works on **any** variant, including storage-only ones: ```sql -SELECT COUNT(tax_id) FROM users; -- works even on bare eql_v3.text +SELECT COUNT(tax_id) FROM users; -- works even on bare public.text ``` `COUNT(DISTINCT col)` deduplicates, so it needs an equality-capable variant — and the same extractor advice applies: @@ -53,10 +53,10 @@ SELECT COUNT(DISTINCT eql_v3.eq_term(email)) FROM logins; EQL ships `min` / `max` aggregates per ord-capable variant of every scalar type. The input type selects the aggregate, and the return type matches the input: ```sql -eql_v3.min(eql_v3._ord) RETURNS eql_v3._ord -eql_v3.max(eql_v3._ord) RETURNS eql_v3._ord -eql_v3.min(eql_v3._ord_ore) RETURNS eql_v3._ord_ore -eql_v3.max(eql_v3._ord_ore) RETURNS eql_v3._ord_ore +eql_v3.min(public._ord) RETURNS public._ord +eql_v3.max(public._ord) RETURNS public._ord +eql_v3.min(public._ord_ore) RETURNS public._ord_ore +eql_v3.max(public._ord_ore) RETURNS public._ord_ore ``` Comparison routes through the variant's `<` / `>` operator on the ORE term — no decryption happens in the database, and the result is an encrypted value the client decrypts. `NULL` inputs are skipped; an all-`NULL` input set returns `NULL`, matching native aggregate semantics. @@ -74,7 +74,7 @@ SELECT eql_v3.eq_term(department_code) AS dept, eql_v3.max(salary) If the column is generic `jsonb` rather than a domain, cast to the right variant at the call site so overload resolution can pick the aggregate: ```sql -SELECT eql_v3.min(salary_jsonb::eql_v3.int8_ord) FROM users; +SELECT eql_v3.min(salary_jsonb::public.bigint_ord) FROM users; ``` A btree on `eql_v3.ord_term(col)` serves `MIN` / `MAX` — the [Indexes](/reference/eql/indexes) page has the recipe. diff --git a/content/docs/reference/eql/index.mdx b/content/docs/reference/eql/index.mdx index 6b9ce9e..dc52454 100644 --- a/content/docs/reference/eql/index.mdx +++ b/content/docs/reference/eql/index.mdx @@ -11,7 +11,7 @@ Encrypt Query Language (EQL) is a set of types, operators, and functions for sto EQL itself never encrypts anything. Encryption and decryption happen in the client, using the [Stack SDK](/reference/stack) or [CipherStash Proxy](/reference/proxy). EQL provides the database-side surface those clients query against: encrypted column types, the operators that compare them, and the term-extractor functions that make indexes work. -Every encrypted column is a `jsonb`-backed domain type in the `eql_v3` schema, and the domain variant you choose declares what the column can do — the full model is in [Core concepts](/reference/eql/core-concepts). +Every encrypted column is a `jsonb`-backed domain type in the `public` schema (the functions and operators behind them live in `eql_v3`), and the domain variant you choose declares what the column can do — the full model is in [Core concepts](/reference/eql/core-concepts). ## Install @@ -51,7 +51,7 @@ SELECT eql_v3.version(); -`DROP SCHEMA eql_v3 CASCADE` drops every column typed as an `eql_v3` domain. The domain types live in the schema, and your columns depend on them. +To uninstall, drop both EQL schemas: `DROP SCHEMA eql_v3, eql_v3_internal CASCADE`. This removes the encrypted operators and functions, so queries against encrypted columns stop resolving — but your data survives: the `public.*` domain types (and the columns typed as them) live in the `public` schema and are left untouched. Remove those separately only if you intend to drop or re-type the columns. ### dbdev @@ -81,7 +81,7 @@ GRANT CREATE ON SCHEMA public TO your_migration_user; GRANT ALTER ON ALL TABLES IN SCHEMA public TO your_migration_user; ``` -`CREATE ON DATABASE` creates the `eql_v3` schema and its types; `CREATE ON SCHEMA` and `ALTER` are needed to add encrypted columns (typed as `eql_v3` domains, with their `CHECK` constraints) to your tables. +`CREATE ON DATABASE` lets EQL create its schemas (`eql_v3`, `eql_v3_internal`) and the `public.*` domain types; `CREATE ON SCHEMA` and `ALTER` are needed to add encrypted columns (typed as `public.*` domains, with their `CHECK` constraints) to your tables. **Runtime user** — the application's day-to-day access: diff --git a/content/docs/reference/eql/indexes.mdx b/content/docs/reference/eql/indexes.mdx index ff4b1fe..553a10b 100644 --- a/content/docs/reference/eql/indexes.mdx +++ b/content/docs/reference/eql/indexes.mdx @@ -18,7 +18,7 @@ EQL indexes are ordinary PostgreSQL functional indexes over **term-extractor fun The extractors are inlinable SQL functions, so the planner rewrites a bare-form predicate into the same expression the index was built on. You don't rewrite queries to use the index: ```sql -SELECT * FROM users WHERE email = $1::eql_v3.text_eq; +SELECT * FROM users WHERE email = $1::public.text_eq; -- planner inlines `=` to: eql_v3.eq_term(email) = eql_v3.eq_term($1) -- Index Cond on USING hash (eql_v3.eq_term(email)) ``` @@ -33,18 +33,18 @@ Type the column as the domain variant that carries the term (see [Core concepts] ```sql -- Equality: hash index on eq_term --- (columns typed eql_v3._eq or text_search; equality on _ord columns +-- (columns typed public._eq or text_search; equality on _ord columns -- compares ORE terms, so the btree on ord_term below serves it) CREATE INDEX users_email_eq ON users USING hash (eql_v3.eq_term(email)); -- Range / ordering: btree index on ord_term --- (columns typed eql_v3._ord or _ord_ore) +-- (columns typed public._ord or _ord_ore) CREATE INDEX users_created_at_ord ON users USING btree (eql_v3.ord_term(created_at)); -- Text match: GIN index on match_term --- (columns typed eql_v3.text_match or text_search) +-- (columns typed public.text_match or text_search) CREATE INDEX users_name_match ON users USING gin (eql_v3.match_term(name)); @@ -65,7 +65,7 @@ All three must hold: ```sql -- ✓ resolves the encrypted operator → uses the index -WHERE email = $1::eql_v3.text_eq; +WHERE email = $1::public.text_eq; WHERE email = $1; -- only when the client (Stack SDK / Proxy) binds $1 typed -- ✗ falls through to native jsonb semantics @@ -77,7 +77,7 @@ WHERE email = '{"hm":"abc"}'::jsonb; ### Equality ```sql -SELECT * FROM users WHERE email = $1::eql_v3.text_eq; +SELECT * FROM users WHERE email = $1::public.text_eq; -- Index Scan using users_email_eq -- Index Cond: (eql_v3.eq_term(email) = eql_v3.eq_term($1)) ``` @@ -87,14 +87,14 @@ SELECT * FROM users WHERE email = $1::eql_v3.text_eq; The `<`, `<=`, `>`, `>=` operators inline to comparisons on `eql_v3.ord_term`, so natural-form range predicates match the btree: ```sql -SELECT * FROM users WHERE created_at < $1::eql_v3.timestamp_ord; +SELECT * FROM users WHERE created_at < $1::public.timestamp_ord; ``` `ORDER BY` needs care. The planner inlines operators in *predicates* but does not rewrite *sort keys*: `ORDER BY created_at` uses the index for the `WHERE` clause but still adds a `Sort` node, which scales linearly with the rows passing the filter. To stream rows out of the btree already ordered, write the sort key in extractor form: ```sql SELECT * FROM users - WHERE created_at < $1::eql_v3.timestamp_ord + WHERE created_at < $1::public.timestamp_ord ORDER BY eql_v3.ord_term(created_at) DESC LIMIT 10; ``` @@ -119,7 +119,7 @@ SELECT eql_v3.eq_term(email), count(*) ## Encrypted JSON -Containment (`@>` / `<@`) on `eql_v3.json` document columns uses a GIN index over `eql_v3.to_ste_vec_query(col)::jsonb`, and field-level equality and ordering have their own extractor recipes. See [JSON](/reference/eql/json). +Containment (`@>` / `<@`) on `public.json` document columns uses a GIN index over `eql_v3.to_ste_vec_query(col)::jsonb`, and field-level equality and ordering have their own extractor recipes. See [JSON](/reference/eql/json). ## Verify with EXPLAIN @@ -175,7 +175,7 @@ Everything above is a functional index over an `IMMUTABLE` SQL function — no o FROM users LIMIT 1; ``` -2. Verify the operand is typed (`$1::eql_v3.text_eq`, not `$1::jsonb`). +2. Verify the operand is typed (`$1::public.text_eq`, not `$1::jsonb`). 3. Recreate the index if the column's terms changed after it was built. 4. Run `ANALYZE`. Very small tables may still choose a sequential scan — that's correct. diff --git a/content/docs/reference/eql/joins.mdx b/content/docs/reference/eql/joins.mdx index fdcc0e4..b4e2850 100644 --- a/content/docs/reference/eql/joins.mdx +++ b/content/docs/reference/eql/joins.mdx @@ -21,7 +21,7 @@ Equijoins work on equality-capable variants (`_eq`, `_ord` / `_ord_ore`, `text_s SELECT u.*, o.total FROM users u JOIN orders o ON u.email = o.customer_email; --- both columns eql_v3.text_eq, encrypted with the same keyset +-- both columns public.text_eq, encrypted with the same keyset ``` No typed-operand cast is needed here — both operands are encrypted columns, so their domain types resolve the encrypted operator directly. All join types (`INNER`, `LEFT`, `RIGHT`, `FULL`) work; `LEFT JOIN` null-extension behaves normally because SQL `NULL`s are not encrypted. @@ -47,17 +47,17 @@ If the two columns are under different keysets, `IN (subquery)` matches nothing, ## Worked example -Two tables sharing an encrypted customer identifier, both columns typed `eql_v3.text_eq` and encrypted by the same client configuration (same keyset): +Two tables sharing an encrypted customer identifier, both columns typed `public.text_eq` and encrypted by the same client configuration (same keyset): ```sql CREATE TABLE users ( id BIGINT GENERATED ALWAYS AS IDENTITY PRIMARY KEY, - email eql_v3.text_eq + email public.text_eq ); CREATE TABLE orders ( id BIGINT GENERATED ALWAYS AS IDENTITY PRIMARY KEY, - customer_email eql_v3.text_eq, + customer_email public.text_eq, total BIGINT NOT NULL ); @@ -72,7 +72,7 @@ Orders per user, filtered by an encrypted lookup on one side: SELECT u.id, COUNT(o.id) AS order_count FROM users u LEFT JOIN orders o ON u.email = o.customer_email -WHERE u.email = $1::eql_v3.text_eq +WHERE u.email = $1::public.text_eq GROUP BY u.id; ``` diff --git a/content/docs/reference/eql/json.mdx b/content/docs/reference/eql/json.mdx index 93eb920..44df57e 100644 --- a/content/docs/reference/eql/json.mdx +++ b/content/docs/reference/eql/json.mdx @@ -1,15 +1,15 @@ --- title: JSON -description: "The complete reference for encrypted JSON documents with eql_v3.json — the ste_vec payload shape, containment, field access, and path queries over ciphertext, with the native jsonb operators that don't apply blocked outright." +description: "The complete reference for encrypted JSON documents with public.json — the ste_vec payload shape, containment, field access, and path queries over ciphertext, with the native jsonb operators that don't apply blocked outright." type: reference components: [eql] verifiedAgainst: eql: "3.0.0" --- -`eql_v3.json` is EQL's encrypted JSON document type, built on structured encryption (**ste_vec**). The document is encrypted as a vector of encrypted entries — one entry per path inside the document — and every path is queryable without decryption: containment, field and array access, and equality or range comparisons on extracted leaves. +`public.json` is EQL's encrypted JSON document type, built on structured encryption (**ste_vec**). The document is encrypted as a vector of encrypted entries — one entry per path inside the document — and every path is queryable without decryption: containment, field and array access, and equality or range comparisons on extracted leaves. -Like every EQL type, `eql_v3.json` holds ciphertext the database can't read. Encryption, decryption, and selector generation happen in the client — the [Stack SDK](/reference/stack) or [CipherStash Proxy](/reference/proxy). See [Searchable encryption](/concepts/searchable-encryption) for how querying ciphertext works at all. +Like every EQL type, `public.json` holds ciphertext the database can't read. Encryption, decryption, and selector generation happen in the client — the [Stack SDK](/reference/stack) or [CipherStash Proxy](/reference/proxy). See [Searchable encryption](/concepts/searchable-encryption) for how querying ciphertext works at all. ## The types @@ -17,9 +17,9 @@ Three `jsonb`-backed domains make up the encrypted JSON surface: | Type | What it is | | --- | --- | -| `eql_v3.json` | The column type. An encrypted document envelope carrying an `sv` array — one encrypted entry per path in the document. | -| `eql_v3.ste_vec_entry` | A single entry from the vector: a selector, a ciphertext, and exactly one index term. This is what `->` returns. | -| `eql_v3.ste_vec_query` | A containment needle: entries with selectors and index terms but **no ciphertext**. This is what you cast a `@>` operand to. | +| `public.json` | The column type. An encrypted document envelope carrying an `sv` array — one encrypted entry per path in the document. | +| `public.jsonb_entry` | A single entry from the vector: a selector, a ciphertext, and exactly one index term. This is what `->` returns. | +| `public.jsonb_query` | A containment needle: entries with selectors and index terms but **no ciphertext**. This is what you cast a `@>` operand to. | ## Payload shape @@ -34,7 +34,7 @@ An encrypted JSON document uses a different payload shape from the scalar types: The decoded `oc` value starts with a domain-tag byte (`0x00` numeric, `0x01` string) followed by the CLLW ciphertext, so numeric and string values in one column keep a consistent total order. Earlier payload versions split this into two fields — `ocf` (fixed-width, numeric) and `ocv` (variable-width, string) — which consolidated into the single `oc` key; the tag byte now carries the distinction. -A document payload for an `eql_v3.json` column: +A document payload for a `public.json` column: ```json { @@ -53,7 +53,7 @@ A document payload for an `eql_v3.json` column: - Second entry: a string leaf — `oc` starting with tag `01` - Third entry: a numeric leaf — `oc` starting with tag `00` -A containment **query** payload (`eql_v3.ste_vec_query`) has the same `sv` shape but its entries carry no `c` — containment matches selectors and index terms, never ciphertexts. This is the needle the client builds for a `@>` query: +A containment **query** payload (`public.jsonb_query`) has the same `sv` shape but its entries carry no `c` — containment matches selectors and index terms, never ciphertexts. This is the needle the client builds for a `@>` query: ```json { @@ -65,16 +65,16 @@ A containment **query** payload (`eql_v3.ste_vec_query`) has the same `sv` shape ## Storing encrypted JSON -Type the column as `eql_v3.json`: +Type the column as `public.json`: ```sql CREATE TABLE orders ( id BIGINT GENERATED ALWAYS AS IDENTITY PRIMARY KEY, - metadata eql_v3.json + metadata public.json ); ``` -There is no database-side configuration step. Which index terms a document carries is decided by the encryption client; typing the column as `eql_v3.json` is what makes the encrypted operators and functions resolve. The domain's `CHECK` constraint validates the payload shape on insert, so malformed values are rejected at write time. +There is no database-side configuration step. Which index terms a document carries is decided by the encryption client; typing the column as `public.json` is what makes the encrypted operators and functions resolve. The domain's `CHECK` constraint validates the payload shape on insert, so malformed values are rejected at write time. Insert and read through the Stack SDK or Proxy, which encrypt the document into the ste_vec payload on write and decrypt it on read. @@ -96,7 +96,7 @@ JSON `null` here means a `null` literal *inside* the document. A SQL `NULL` colu ## Blocked native jsonb operators -These native PostgreSQL `jsonb` operators are **blocked** on `eql_v3.json`. They raise an error rather than silently running plaintext-jsonb semantics against the encrypted payload: +These native PostgreSQL `jsonb` operators are **blocked** on `public.json`. They raise an error rather than silently running plaintext-jsonb semantics against the encrypted payload: - Key/path existence: `?`, `?|`, `?&`, `@?`, `@@` - Path extraction: `#>`, `#>>` @@ -111,11 +111,11 @@ Use containment (`@>` / `<@`), field access (`->` / `->>`), or the `eql_v3.jsonb ## Containment: `@>` and `<@` -`@>` tests whether the encrypted document contains a structure; `<@` is the reverse. Build the needle with the client and cast it to `eql_v3.ste_vec_query` (a typed `eql_v3.json` or `eql_v3.ste_vec_entry` operand also works): +`@>` tests whether the encrypted document contains a structure; `<@` is the reverse. Build the needle with the client and cast it to `public.jsonb_query` (a typed `public.json` or `public.jsonb_entry` operand also works): ```sql SELECT * FROM orders -WHERE metadata @> $1::eql_v3.ste_vec_query; +WHERE metadata @> $1::public.jsonb_query; ``` This is the encrypted equivalent of the plaintext `metadata @> '{"customer": {"tier": "premium"}}'`: containment checks that every encrypted term in the needle exists in the document's `sv` vector. `eql_v3.to_ste_vec_query(doc)` converts a stored document into the needle shape, and `eql_v3.ste_vec_contains(a, b)` is the function form backing `@>`. @@ -135,7 +135,7 @@ See [Indexes](/reference/eql/indexes) for the full recipes. Fields are addressed by **selector hash** — the deterministic identifier the client emits for a JSON path during encryption — not a plaintext path string like `$.customer.tier`. ```sql --- Field access by selector (returns eql_v3.ste_vec_entry) +-- Field access by selector (returns public.jsonb_entry) SELECT metadata -> 'selector_hash'::text FROM orders; -- The entry serialized as text (ciphertext JSON, not decrypted plaintext) @@ -145,7 +145,7 @@ SELECT metadata ->> 'selector_hash'::text FROM orders; SELECT metadata -> 0 FROM orders; ``` -The extracted `eql_v3.ste_vec_entry` is itself comparable: +The extracted `public.jsonb_entry` is itself comparable: - `=` / `<>` resolve via `eql_v3.eq_term` — works on every node type - `<` / `<=` / `>` / `>=` resolve via `eql_v3.ore_cllw` — String and Number leaves only @@ -154,7 +154,7 @@ The extracted `eql_v3.ste_vec_entry` is itself comparable: ```sql -- Equality on an extracted leaf SELECT * FROM orders -WHERE metadata -> 'email_selector'::text = $1::eql_v3.ste_vec_entry; +WHERE metadata -> 'email_selector'::text = $1::public.jsonb_entry; -- Group by an extracted leaf's equality term SELECT eql_v3.eq_term(metadata -> 'region_selector'::text) AS region, COUNT(*) @@ -213,7 +213,7 @@ The client encrypts this into a ste_vec payload with selectors for `$`, `$.custo ```sql CREATE TABLE orders ( id BIGINT GENERATED ALWAYS AS IDENTITY PRIMARY KEY, - metadata eql_v3.json + metadata public.json ); INSERT INTO orders (metadata) VALUES ($1); @@ -229,7 +229,7 @@ Find premium orders. The client encrypts the needle `{"customer": {"tier": "prem ```sql SELECT id FROM orders -WHERE metadata @> $1::eql_v3.ste_vec_query; +WHERE metadata @> $1::public.jsonb_query; ``` Add the GIN index from above once the table grows. diff --git a/content/docs/reference/eql/meta.json b/content/docs/reference/eql/meta.json index f4268ce..0857084 100644 --- a/content/docs/reference/eql/meta.json +++ b/content/docs/reference/eql/meta.json @@ -14,6 +14,8 @@ "filtering", "sorting", "grouping-and-aggregates", - "joins" + "joins", + "---Reference---", + "functions" ] } diff --git a/content/docs/reference/eql/numbers.mdx b/content/docs/reference/eql/numbers.mdx index 2d9a2a0..aeb3a63 100644 --- a/content/docs/reference/eql/numbers.mdx +++ b/content/docs/reference/eql/numbers.mdx @@ -17,21 +17,21 @@ Each numeric type generates the same `jsonb`-backed domain variants. The generic | Domain variant | Capability | | --- | --- | -| `eql_v3.` | Storage and decryption only. | -| `eql_v3._eq` | Equality: `=`, `<>`, `IN`, `GROUP BY`, `DISTINCT`, equijoins. | -| `eql_v3._ord` | Comparisons, `BETWEEN`, `ORDER BY`, `MIN` / `MAX` — plus equality. | -| `eql_v3._ord_ore` | As `_ord`, with the ORE mechanism pinned — see [SEM specifiers](#sem-specifiers). | +| `public.` | Storage and decryption only. | +| `public._eq` | Equality: `=`, `<>`, `IN`, `GROUP BY`, `DISTINCT`, equijoins. | +| `public._ord` | Comparisons, `BETWEEN`, `ORDER BY`, `MIN` / `MAX` — plus equality. | +| `public._ord_ore` | As `_ord`, with the ORE mechanism pinned — see [SEM specifiers](#sem-specifiers). | And every concrete domain this page covers: | Type | Variants | | --- | --- | -| `int2` | `eql_v3.int2` · `eql_v3.int2_eq` · `eql_v3.int2_ord` · `eql_v3.int2_ord_ore` | -| `int4` | `eql_v3.int4` · `eql_v3.int4_eq` · `eql_v3.int4_ord` · `eql_v3.int4_ord_ore` | -| `int8` | `eql_v3.int8` · `eql_v3.int8_eq` · `eql_v3.int8_ord` · `eql_v3.int8_ord_ore` | -| `float4` | `eql_v3.float4` · `eql_v3.float4_eq` · `eql_v3.float4_ord` · `eql_v3.float4_ord_ore` | -| `float8` | `eql_v3.float8` · `eql_v3.float8_eq` · `eql_v3.float8_ord` · `eql_v3.float8_ord_ore` | -| `numeric` | `eql_v3.numeric` · `eql_v3.numeric_eq` · `eql_v3.numeric_ord` · `eql_v3.numeric_ord_ore` | +| `int2` | `public.smallint` · `public.smallint_eq` · `public.smallint_ord` · `public.smallint_ord_ore` | +| `int4` | `public.integer` · `public.integer_eq` · `public.integer_ord` · `public.integer_ord_ore` | +| `int8` | `public.bigint` · `public.bigint_eq` · `public.bigint_ord` · `public.bigint_ord_ore` | +| `float4` | `public.real` · `public.real_eq` · `public.real_ord` · `public.real_ord_ore` | +| `float8` | `public.double` · `public.double_eq` · `public.double_ord` · `public.double_ord_ore` | +| `numeric` | `public.numeric` · `public.numeric_eq` · `public.numeric_ord` · `public.numeric_ord_ore` | Declare only the capability you query on — each capability stores extra searchable material with defined leakage (see [Searchable encryption](/concepts/searchable-encryption)), and the variant model itself is covered in [Core concepts](/reference/eql/core-concepts). @@ -42,9 +42,9 @@ A payroll table mixing the variants by how each column is queried: ```sql CREATE TABLE employees ( id bigint GENERATED ALWAYS AS IDENTITY PRIMARY KEY, - salary eql_v3.int8_ord, -- range queries, ORDER BY, MIN/MAX - tax_rate eql_v3.numeric_eq, -- exact lookup only - net_worth eql_v3.numeric -- store and decrypt only, never queried + salary public.bigint_ord, -- range queries, ORDER BY, MIN/MAX + tax_rate public.numeric_eq, -- exact lookup only + net_worth public.numeric -- store and decrypt only, never queried ); ``` @@ -61,7 +61,7 @@ The EQL v3 release adds an OPE specifier for every orderable type; unspecified ` ## Payload -A value for an `_ord` column carries the shared envelope keys (`v`, `i`, `c` — see [Core concepts](/reference/eql/core-concepts)) plus the `ob` ordering term. Here is a payload for the `eql_v3.int8_ord` `salary` column: +A value for an `_ord` column carries the shared envelope keys (`v`, `i`, `c` — see [Core concepts](/reference/eql/core-concepts)) plus the `ob` ordering term. Here is a payload for the `public.bigint_ord` `salary` column: ```json { @@ -80,7 +80,7 @@ A value for an `_ord` column carries the shared envelope keys (`v`, `i`, `c` — ## Operators -| SQL operator | `eql_v3.` | `_eq` | `_ord` / `_ord_ore` | +| SQL operator | `public.` | `_eq` | `_ord` / `_ord_ore` | | --- | :---: | :---: | :---: | | `=` / `<>` | ❌ | ✅ | ✅ | | `<` `<=` `>` `>=` | ❌ | ❌ | ✅ | @@ -90,7 +90,7 @@ A value for an `_ord` column carries the shared envelope keys (`v`, `i`, `c` — | `ORDER BY` | ❌ | ❌ | ✅ | | `IS NULL` / `IS NOT NULL` | ✅ | ✅ | ✅ | -Blocked *operator* cells raise an `operator … is not supported` exception — they never silently return wrong rows. `ORDER BY` is the one blocked cell that doesn't raise: it isn't an operator, so sorting a variant without an ordering term runs — but the order is meaningless (see [Sorting](/reference/eql/sorting)). Operands must be typed (`$1::eql_v3.int8_ord`), or PostgreSQL resolves the native `jsonb` operator instead of the encrypted one. Both rules are covered in [Core concepts](/reference/eql/core-concepts). +Blocked *operator* cells raise an `operator … is not supported` exception — they never silently return wrong rows. `ORDER BY` is the one blocked cell that doesn't raise: it isn't an operator, so sorting a variant without an ordering term runs — but the order is meaningless (see [Sorting](/reference/eql/sorting)). Operands must be typed (`$1::public.bigint_ord`), or PostgreSQL resolves the native `jsonb` operator instead of the encrypted one. Both rules are covered in [Core concepts](/reference/eql/core-concepts). ## Functions @@ -110,10 +110,10 @@ Every operator has a function form, for managed platforms that disallow custom o ```sql SELECT * FROM employees -WHERE salary >= $1::eql_v3.int8_ord; +WHERE salary >= $1::public.bigint_ord; SELECT * FROM employees -WHERE salary BETWEEN $1::eql_v3.int8_ord AND $2::eql_v3.int8_ord; +WHERE salary BETWEEN $1::public.bigint_ord AND $2::public.bigint_ord; ``` ### MIN and MAX @@ -140,7 +140,7 @@ LIMIT 10; On a generic `jsonb` column whose payloads already carry the `ob` term, cast to the right domain in the query: ```sql -SELECT eql_v3.min(salary_jsonb::eql_v3.int8_ord) FROM employees; +SELECT eql_v3.min(salary_jsonb::public.bigint_ord) FROM employees; ``` ## Where to next diff --git a/content/docs/reference/eql/sorting.mdx b/content/docs/reference/eql/sorting.mdx index eb34f12..2e7ef2b 100644 --- a/content/docs/reference/eql/sorting.mdx +++ b/content/docs/reference/eql/sorting.mdx @@ -33,7 +33,7 @@ CREATE INDEX users_created_at_ord ANALYZE users; SELECT * FROM users - WHERE created_at < $1::eql_v3.timestamp_ord + WHERE created_at < $1::public.timestamp_ord ORDER BY eql_v3.ord_term(created_at) DESC LIMIT 10; -- Index Scan Backward using users_created_at_ord — no Sort node @@ -64,7 +64,7 @@ SELECT id, email, created_at FROM users -- Next page: pass the last row's created_at back, re-encrypted as the cursor SELECT id, email, created_at FROM users - WHERE created_at < $1::eql_v3.timestamp_ord + WHERE created_at < $1::public.timestamp_ord ORDER BY eql_v3.ord_term(created_at) DESC LIMIT 20; ``` diff --git a/content/docs/reference/eql/text.mdx b/content/docs/reference/eql/text.mdx index f08eafc..408ed97 100644 --- a/content/docs/reference/eql/text.mdx +++ b/content/docs/reference/eql/text.mdx @@ -15,12 +15,12 @@ All six are `jsonb`-backed domains. Which one you declare fixes the column's que | Domain variant | Capability | | --- | --- | -| `eql_v3.text` | Storage and decryption only. | -| `eql_v3.text_eq` | Equality: `=`, `<>`, `IN`, `GROUP BY`, `DISTINCT`, equijoins. | -| `eql_v3.text_ord` | Comparisons, `BETWEEN`, `ORDER BY`, `MIN` / `MAX` — plus equality. | -| `eql_v3.text_ord_ore` | As `text_ord`, with the ORE mechanism pinned — see [SEM specifiers](#sem-specifiers). | -| `eql_v3.text_match` | Free-text token containment: `@>` / `<@`. | -| `eql_v3.text_search` | Equality + ordering + token containment. | +| `public.text` | Storage and decryption only. | +| `public.text_eq` | Equality: `=`, `<>`, `IN`, `GROUP BY`, `DISTINCT`, equijoins. | +| `public.text_ord` | Comparisons, `BETWEEN`, `ORDER BY`, `MIN` / `MAX` — plus equality. | +| `public.text_ord_ore` | As `text_ord`, with the ORE mechanism pinned — see [SEM specifiers](#sem-specifiers). | +| `public.text_match` | Free-text token containment: `@>` / `<@`. | +| `public.text_search` | Equality + ordering + token containment. | Declare only the capabilities you query on — each capability stores extra searchable material with defined leakage (see [Searchable encryption](/concepts/searchable-encryption)). @@ -31,10 +31,10 @@ A users table mixing the variants by how each column is queried: ```sql CREATE TABLE users ( id bigint GENERATED ALWAYS AS IDENTITY PRIMARY KEY, - email eql_v3.text_search, -- lookup, sort, and free-text match - name eql_v3.text_match, -- free-text match only - tax_id eql_v3.text_eq, -- exact lookup only - notes eql_v3.text -- store and decrypt only + email public.text_search, -- lookup, sort, and free-text match + name public.text_match, -- free-text match only + tax_id public.text_eq, -- exact lookup only + notes public.text -- store and decrypt only ); ``` @@ -74,7 +74,7 @@ The narrower variants carry only their own term: a `text_eq` payload carries `hm ## Operators -| SQL operator | `eql_v3.text` | `text_eq` | `text_ord` / `text_ord_ore` | `text_match` | `text_search` | +| SQL operator | `public.text` | `text_eq` | `text_ord` / `text_ord_ore` | `text_match` | `text_search` | | --- | :---: | :---: | :---: | :---: | :---: | | `=` / `<>` | ❌ | ✅ | ✅ | ❌ | ✅ | | `<` `<=` `>` `>=` | ❌ | ❌ | ✅ | ❌ | ✅ | @@ -84,7 +84,7 @@ The narrower variants carry only their own term: a `text_eq` payload carries `hm | `ORDER BY` | ❌ | ❌ | ✅ | ❌ | ✅ | | `IS NULL` / `IS NOT NULL` | ✅ | ✅ | ✅ | ✅ | ✅ | -Blocked *operator* cells raise an `operator … is not supported` exception — they never silently return wrong rows. `ORDER BY` is the one blocked cell that doesn't raise: it isn't an operator, so sorting a variant without an ordering term runs — but the order is meaningless (see [Sorting](/reference/eql/sorting)). Operands must be typed (`$1::eql_v3.text_eq`), or PostgreSQL resolves the native `jsonb` operator instead of the encrypted one. Both rules are covered in [Core concepts](/reference/eql/core-concepts). +Blocked *operator* cells raise an `operator … is not supported` exception — they never silently return wrong rows. `ORDER BY` is the one blocked cell that doesn't raise: it isn't an operator, so sorting a variant without an ordering term runs — but the order is meaningless (see [Sorting](/reference/eql/sorting)). Operands must be typed (`$1::public.text_eq`), or PostgreSQL resolves the native `jsonb` operator instead of the encrypted one. Both rules are covered in [Core concepts](/reference/eql/core-concepts). ## Functions @@ -108,7 +108,7 @@ There are no `like` / `ilike` function forms — encrypted text matching is `eql SELECT * FROM users WHERE email LIKE '%alice%'; -- ✅ Encrypted free-text match -SELECT * FROM users WHERE email @> $1::eql_v3.text_match; +SELECT * FROM users WHERE email @> $1::public.text_match; ``` `@>` / `<@` here is **probabilistic ngram-bloom containment** — it tests whether the encrypted text contains the (encrypted) search terms. It is not JSONB containment and not `LIKE`. The client encrypts the search term into a bloom-filter query value; false positives are possible, false negatives are not. There are no `like` / `ilike` function forms either — text matching is `eql_v3.contains` on a `text_match` value. @@ -120,10 +120,10 @@ SELECT * FROM users WHERE email @> $1::eql_v3.text_match; Equality on a `text_eq` column compares HMAC terms. `IN` desugars to `=`: ```sql -SELECT * FROM users WHERE tax_id = $1::eql_v3.text_eq; +SELECT * FROM users WHERE tax_id = $1::public.text_eq; SELECT * FROM users -WHERE tax_id IN ($1::eql_v3.text_eq, $2::eql_v3.text_eq); +WHERE tax_id IN ($1::public.text_eq, $2::public.text_eq); ``` ### Free-text match @@ -131,10 +131,10 @@ WHERE tax_id IN ($1::eql_v3.text_eq, $2::eql_v3.text_eq); The client encrypts the search term into the bloom-filter needle: ```sql -SELECT * FROM users WHERE name @> $1::eql_v3.text_match; +SELECT * FROM users WHERE name @> $1::public.text_match; -- Function form, for platforms without custom operators -SELECT * FROM users WHERE eql_v3.contains(name, $1::eql_v3.text_match); +SELECT * FROM users WHERE eql_v3.contains(name, $1::public.text_match); ``` ### The works: `text_search` @@ -143,8 +143,8 @@ A `text_search` column answers exact lookup, free-text match, and ordering — h ```sql SELECT id, email FROM users -WHERE email @> $1::eql_v3.text_match -- token containment on bf - AND email <> $2::eql_v3.text_eq -- exclude an exact value via hm +WHERE email @> $1::public.text_match -- token containment on bf + AND email <> $2::public.text_eq -- exclude an exact value via hm ORDER BY eql_v3.ord_term(email) -- sort on ob LIMIT 20; ``` diff --git a/package.json b/package.json index 3b4681a..b281edb 100644 --- a/package.json +++ b/package.json @@ -3,7 +3,7 @@ "version": "0.0.0", "private": true, "scripts": { - "prebuild": "bun run generate-docs && bun run generate-docs:eql && bun run validate-links && bun run validate-redirects", + "prebuild": "bun run generate-docs && bun run generate-docs:eql && bun run generate-docs:eql-api && bun run validate-links && bun run validate-redirects", "build": "next build", "dev": "next dev -p 3001", "start": "next start", @@ -13,6 +13,7 @@ "format": "biome format --write", "generate-docs": "tsx scripts/generate-docs.ts", "generate-docs:eql": "tsx scripts/generate-eql-docs.ts", + "generate-docs:eql-api": "tsx scripts/generate-eql-api-docs.ts", "validate-links": "tsx scripts/validate-links.ts", "validate-redirects": "tsx scripts/validate-v2-redirects.ts" }, diff --git a/scripts/fixtures/eql-manifest.sample.json b/scripts/fixtures/eql-manifest.sample.json new file mode 100644 index 0000000..25a5246 --- /dev/null +++ b/scripts/fixtures/eql-manifest.sample.json @@ -0,0 +1,192 @@ +{ + "_note": "ILLUSTRATIVE SAMPLE — shape only. Replaced at build time by the real eql-manifest.json from the eql-docs release asset (cipherstash/encrypt-query-language). Do not treat these signatures/domains as authoritative. Mirrors the real schema: domains live in `public.`, public functions in `eql_v3.`, internal ctors in `eql_v3_internal.` (visibility: private).", + "$schema": "https://schemas.cipherstash.com/eql/manifest/v1.json", + "name": "eql", + "version": "3.0.0-sample", + "generatedFrom": "doxygen-xml + catalog", + "counts": { "functions": 5, "public": 2, "private": 3, "domains": 9 }, + "functions": [ + { + "name": "version", + "signature": "version()", + "visibility": "public", + "brief": "Get the installed EQL version string.", + "description": "Returns the version string for the installed EQL library.", + "params": [], + "returns": { "type": "text", "description": "the version string" }, + "throws": [], + "notes": "", + "warnings": "", + "seeAlso": "", + "source": { "file": "src/v3/version.sql", "line": 28 } + }, + { + "name": "jsonb_path_query", + "signature": "jsonb_path_query(jsonb, text)", + "visibility": "public", + "brief": "Query encrypted JSONB for sv elements matching a selector.", + "description": "Returns one jsonb_entry row per matching encrypted element.", + "params": [ + { + "name": "val", + "type": "jsonb", + "description": "encrypted EQL payload with sv" + }, + { "name": "selector", "type": "text", "description": "selector hash" } + ], + "returns": { + "type": "public.jsonb_entry", + "description": "matching encrypted entries" + }, + "throws": [], + "notes": "", + "warnings": "", + "seeAlso": "", + "source": { "file": "src/v3/jsonb/functions.sql", "line": 358 } + }, + { + "name": "hmac_256", + "signature": "hmac_256(jsonb)", + "visibility": "private", + "brief": "Extract the HMAC-SHA-256 equality term from an encrypted value.", + "description": "Backs equality (`=`, `IN`, joins) on `_eq` and `text_search` columns.", + "params": [ + { "name": "val", "type": "jsonb", "description": "the encrypted value" } + ], + "returns": { "type": "text", "description": "the HMAC term" }, + "throws": [], + "notes": "", + "warnings": "", + "seeAlso": "", + "source": { "file": "src/v3/sem/hmac_256/functions.sql", "line": 12 } + }, + { + "name": "bloom_filter", + "signature": "bloom_filter(jsonb)", + "visibility": "private", + "brief": "Extract the bloom-filter match term from an encrypted value.", + "description": "Backs token containment (`@>`) on `text_match` / `text_search` columns.", + "params": [ + { "name": "val", "type": "jsonb", "description": "the encrypted value" } + ], + "returns": { + "type": "smallint[]", + "description": "the set bit positions" + }, + "throws": [], + "notes": "", + "warnings": "", + "seeAlso": "", + "source": { "file": "src/v3/sem/bloom_filter/functions.sql", "line": 20 } + }, + { + "name": "ore_block_256", + "signature": "ore_block_256(jsonb)", + "visibility": "private", + "brief": "Extract the ORE ordering term from an encrypted value.", + "description": "Backs range and ordering (`<`, `>`, `ORDER BY`) on `_ord` columns.", + "params": [ + { "name": "val", "type": "jsonb", "description": "the encrypted value" } + ], + "returns": { + "type": "eql_v3_internal.ore_block_256", + "description": "the ORE term" + }, + "throws": [], + "notes": "", + "warnings": "", + "seeAlso": "", + "source": { "file": "src/v3/sem/ore_block_256/functions.sql", "line": 8 } + } + ], + "domains": [ + { + "name": "public.integer", + "type": "integer", + "variant": "", + "base": "jsonb", + "capabilities": ["storage"], + "supportedOperators": [], + "termFunctions": [] + }, + { + "name": "public.integer_eq", + "type": "integer", + "variant": "eq", + "base": "jsonb", + "capabilities": ["equality"], + "supportedOperators": ["=", "<>"], + "termFunctions": ["eql_v3.eq_term"] + }, + { + "name": "public.integer_ord", + "type": "integer", + "variant": "ord", + "base": "jsonb", + "capabilities": ["equality", "order"], + "supportedOperators": ["=", "<>", "<", "<=", ">", ">="], + "termFunctions": ["eql_v3.ord_term"] + }, + { + "name": "public.integer_ord_ope", + "type": "integer", + "variant": "ord_ope", + "base": "jsonb", + "capabilities": ["equality", "order"], + "supportedOperators": ["=", "<>", "<", "<=", ">", ">="], + "termFunctions": ["eql_v3.ord_ope_term"] + }, + { + "name": "public.text_match", + "type": "text", + "variant": "match", + "base": "jsonb", + "capabilities": ["match"], + "supportedOperators": ["@>", "<@"], + "termFunctions": ["eql_v3.match_term"] + }, + { + "name": "public.text_search", + "type": "text", + "variant": "search", + "base": "jsonb", + "capabilities": ["equality", "order", "match"], + "supportedOperators": ["=", "<>", "<", "<=", ">", ">=", "@>", "<@"], + "termFunctions": [ + "eql_v3.eq_term", + "eql_v3.ord_term", + "eql_v3.match_term" + ] + }, + { + "name": "public.json", + "type": "jsonb", + "variant": "", + "base": "jsonb", + "shape": "stevec", + "capabilities": ["json"], + "supportedOperators": [], + "termFunctions": [] + }, + { + "name": "public.jsonb_entry", + "type": "jsonb", + "variant": "", + "base": "jsonb", + "shape": "stevec", + "capabilities": ["json"], + "supportedOperators": [], + "termFunctions": ["eql_v3.eq_term", "eql_v3.ore_cllw"] + }, + { + "name": "public.jsonb_query", + "type": "jsonb", + "variant": "", + "base": "jsonb", + "shape": "stevec", + "capabilities": ["json"], + "supportedOperators": [], + "termFunctions": [] + } + ] +} diff --git a/scripts/generate-eql-api-docs.ts b/scripts/generate-eql-api-docs.ts new file mode 100644 index 0000000..c6cbfa6 --- /dev/null +++ b/scripts/generate-eql-api-docs.ts @@ -0,0 +1,291 @@ +#!/usr/bin/env tsx +/** + * EQL API reference generator + drift guard (docs V2). + * + * Consumes the structured `eql-manifest.json` that the EQL repo emits from its + * Doxygen'd SQL (cipherstash/encrypt-query-language#364) and: + * + * 1. Generates a version-stamped function catalog at + * content/docs/reference/eql/functions.mdx — the exhaustive, drift-proof + * low-level reference the hand-written pedagogical pages link to. + * 2. Drift-lints the hand-written pages: every schema-qualified reference + * (`public.`, `eql_v3.`, `eql_v3_internal.`) must match the + * manifest by schema AND name. This catches fabricated names, stale + * payloads, and right-name/wrong-schema refs (e.g. `eql_v3.text_eq` for a + * domain that lives in `public`) — the classes of drift that slipped into + * the v2 docs. + * + * ── Manifest source ──────────────────────────────────────────────────────── + * Reads the committed illustrative sample by default. Set EQL_MANIFEST_PATH to + * a real manifest (the `eql-docs-` release asset extracted by + * `generate-eql-docs.ts`, or a locally-generated one) — the renderer and lint + * are identical either way. + * + * The drift-lint is REPORT-ONLY against the sample (tiny, so most real symbols + * read as "unknown"); it becomes a failing gate against a real manifest + * (auto-strict when EQL_MANIFEST_PATH is set). See STRICT below. + */ +import fs from "node:fs"; +import path from "node:path"; + +// Manifest source, in priority order: +// 1. EQL_MANIFEST_PATH — explicit override (local testing / CI). +// 2. .eql-manifest.release.json — extracted from the eql-docs release asset +// by generate-eql-docs.ts (present once a release ships the manifest). +// 3. the committed illustrative sample — offline fallback. +const SAMPLE_MANIFEST = path.join( + process.cwd(), + "scripts/fixtures/eql-manifest.sample.json", +); +const RELEASE_MANIFEST = path.join(process.cwd(), ".eql-manifest.release.json"); +const MANIFEST_PATH = + process.env.EQL_MANIFEST_PATH ?? + (fs.existsSync(RELEASE_MANIFEST) ? RELEASE_MANIFEST : SAMPLE_MANIFEST); +const EQL_DIR = path.join(process.cwd(), "content/docs/reference/eql"); +const OUT_FILE = path.join(EQL_DIR, "functions.mdx"); +// Report-only against the illustrative sample; a failing gate against any real +// manifest (the release asset or an explicit override). +const STRICT = MANIFEST_PATH !== SAMPLE_MANIFEST; + +interface Param { + name: string; + type?: string; + description?: string; +} +interface Fn { + name: string; + signature: string; + visibility: "public" | "private"; + brief: string; + description?: string; + params: Param[]; + returns?: { type?: string; description?: string }; + source?: { file?: string; line?: number }; +} +interface Domain { + name: string; + type: string; + variant: string; + base?: string; + capabilities: string[]; + supportedOperators?: string[]; + termFunctions?: string[]; + shape?: string; +} +interface Manifest { + version: string; + functions: Fn[]; + domains?: Domain[]; +} + +function loadManifest(): Manifest { + return JSON.parse(fs.readFileSync(MANIFEST_PATH, "utf8")); +} + +// ── Render the generated catalog ───────────────────────────────────────────── +function paramsTable(params: Param[]): string { + if (!params.length) return ""; + const rows = params + .map( + (p) => + `| \`${p.name}\` | ${p.type ? `\`${p.type}\`` : ""} | ${(p.description ?? "").replace(/\|/g, "\\|")} |`, + ) + .join("\n"); + return `\n| Parameter | Type | Description |\n| --- | --- | --- |\n${rows}\n`; +} + +// Public functions are heavily overloaded — one name per operation with a +// per-encrypted-type variant, all sharing the same doc. Group by name so the +// page is one entry per function (its distinct signatures listed) instead of +// ~hundreds of near-identical overload blocks. +function renderPublicFunctions(fns: Fn[]): string { + const byName = new Map(); + for (const f of fns) { + const g = byName.get(f.name) ?? []; + g.push(f); + byName.set(f.name, g); + } + const sections: string[] = []; + const entries = [...byName.entries()].sort((a, b) => + a[0].localeCompare(b[0]), + ); + for (const [name, overloads] of entries) { + const rep = + overloads.find((o) => o.params.length || o.brief) ?? overloads[0]; + const sigs = [...new Set(overloads.map((o) => o.signature))].sort(); + const parts = [`### \`${name}\``, "", rep.brief]; + if (rep.description && rep.description !== rep.brief) + parts.push("", rep.description); + parts.push( + "", + sigs.length > 1 + ? `**${sigs.length} overloads** (one per encrypted type):` + : "**Signature:**", + "", + "```sql", + ...sigs, + "```", + ); + if (rep.params.length) parts.push(paramsTable(rep.params)); + if (rep.returns?.type || rep.returns?.description) { + const t = rep.returns.type ? `\`${rep.returns.type}\`` : ""; + parts.push( + "", + `**Returns:** ${t}${rep.returns.description ? ` — ${rep.returns.description}` : ""}`, + ); + } + sections.push(parts.join("\n")); + } + return sections.join("\n\n"); +} + +function renderDomains(domains: Domain[]): string { + if (!domains.length) return ""; + const rows = domains + .map((d) => { + const variant = d.variant ? `\`_${d.variant}\`` : "_(storage only)_"; + const ops = d.supportedOperators?.length + ? d.supportedOperators.map((o) => `\`${o}\``).join(" ") + : "—"; + return `| \`${d.name}\` | ${d.type} | ${variant} | ${d.capabilities.join(", ")} | ${ops} |`; + }) + .join("\n"); + return [ + "## Encrypted domains", + "", + "A column's capability is declared by its **domain variant**. This matrix comes straight from the Rust catalog (`eql-codegen dump-catalog`) — the source of truth the SQL is generated from. See [Core concepts](/reference/eql/core-concepts) for the model.", + "", + "| Domain | Type | Variant | Capabilities | Operators |", + "| --- | --- | --- | --- | --- |", + rows, + "", + ].join("\n"); +} + +function render(manifest: Manifest): string { + const version = manifest.version; + // Operators (`->`, `>`) reach the manifest as pseudo-functions via Doxygen's + // operator-name remapping; they render poorly and are already covered by the + // domain matrix's Operators column, so keep only real named functions here. + const isNamed = (f: Fn) => /^[a-z_][a-z0-9_]*$/.test(f.name); + const publicFns = manifest.functions.filter( + (f) => f.visibility === "public" && isNamed(f), + ); + const privateFns = manifest.functions.filter( + (f) => f.visibility === "private", + ); + + const frontmatter = [ + "---", + "title: Functions", + `description: "Generated catalog of EQL SQL functions and operators (EQL ${version})."`, + "type: reference", + "components: [eql]", + "verifiedAgainst:", + ` eql: "${version}"`, + "---", + ].join("\n"); + + const body = [ + frontmatter, + "", + `{/* GENERATED — do not edit. Produced by scripts/generate-eql-api-docs.ts from the EQL manifest (v${version}). */}`, + "", + ``, + `Generated from the **EQL ${version}** manifest (the Doxygen'd SQL is the source of truth). For the model behind these — variants, terms, typed operands — see [Core concepts](/reference/eql/core-concepts).`, + ``, + "", + "The EQL SQL surface — encrypted domains (in the `public` schema) and the `eql_v3` functions behind them. The type and query pages explain *when* to use these; this page is the exhaustive reference they link to.", + "", + renderDomains(manifest.domains ?? []), + "## Functions", + "", + `The public \`eql_v3\` API — ${new Set(publicFns.map((f) => f.name)).size} functions (${publicFns.length} overloads). Internal \`eql_v3_internal\` functions (${privateFns.length}) are implementation detail and omitted.`, + "", + renderPublicFunctions(publicFns), + ]; + + return `${body.join("\n").trimEnd()}\n`; +} + +// ── Drift guard ────────────────────────────────────────────────────────────── +// The known surface is fully schema-qualified: domains live in `public.`, +// functions in `eql_v3.` (public) or `eql_v3_internal.` (private), and the +// per-domain extractor term-functions come pre-qualified from the catalog. +// Matching schema-AND-name means a right-name/wrong-schema reference — e.g. +// `eql_v3.text_eq`, whose domain is actually `public.text_eq` — is flagged too, +// not just fabricated names. +function knownSymbols(manifest: Manifest): Set { + const known = new Set(); + for (const d of manifest.domains ?? []) { + known.add(d.name); // public. + for (const fn of d.termFunctions ?? []) known.add(fn); // eql_v3. + } + for (const f of manifest.functions) { + const schema = f.visibility === "private" ? "eql_v3_internal" : "eql_v3"; + known.add(`${schema}.${f.name}`); + } + return known; +} + +function driftCheck(manifest: Manifest): string[] { + const known = knownSymbols(manifest); + const referenced = new Map>(); // fqn -> pages + + for (const file of fs.readdirSync(EQL_DIR)) { + if (!file.endsWith(".mdx") || file === "functions.mdx") continue; + const text = fs.readFileSync(path.join(EQL_DIR, file), "utf8"); + // Any schema-qualified reference — function call, domain cast, or type. + // A trailing `*` marks a prose family (e.g. `eql_v3.jsonb_path_*`), which + // names a set rather than one symbol, so it's skipped. + for (const m of text.matchAll( + /\b(public|eql_v3_internal|eql_v3)\.([a-z0-9_]+)(\*?)/g, + )) { + if (m[3] === "*") continue; + const fqn = `${m[1]}.${m[2]}`; + const pages = referenced.get(fqn) ?? new Set(); + pages.add(file); + referenced.set(fqn, pages); + } + } + + const unknown: string[] = []; + for (const [fqn, pages] of referenced) { + if (!known.has(fqn)) unknown.push(`${fqn} (in ${[...pages].join(", ")})`); + } + return unknown.sort(); +} + +// ── Main ───────────────────────────────────────────────────────────────────── +function main() { + const manifest = loadManifest(); + + fs.mkdirSync(EQL_DIR, { recursive: true }); + fs.writeFileSync(OUT_FILE, render(manifest)); + console.log( + `✓ Generated ${path.relative(process.cwd(), OUT_FILE)} from EQL ${manifest.version} (${manifest.functions.length} functions)`, + ); + + const unknown = driftCheck(manifest); + if (unknown.length) { + const header = `⚠ ${unknown.length} schema-qualified symbol(s) referenced in hand-written pages are not in the manifest (domains or functions):`; + console.warn(`\n${header}`); + for (const u of unknown) console.warn(` - ${u}`); + if (STRICT) { + console.error( + "\nDrift check failed (STRICT). Fix the reference or update the pinned EQL version.", + ); + process.exit(1); + } else { + console.warn( + "\n(Report-only: using the illustrative sample manifest, which covers only a few symbols. A real manifest — the release asset or EQL_MANIFEST_PATH — makes this a failing gate.)", + ); + } + } else { + console.log( + "✓ Drift check: all schema-qualified symbols resolve against the manifest.", + ); + } +} + +main(); diff --git a/scripts/generate-eql-docs.ts b/scripts/generate-eql-docs.ts index 4c85527..6df726c 100644 --- a/scripts/generate-eql-docs.ts +++ b/scripts/generate-eql-docs.ts @@ -15,6 +15,9 @@ const GITHUB_API_URL = "https://api.github.com/repos/cipherstash/encrypt-query-language/releases"; const TEMP_DIR = ".tmp-eql"; const OUTPUT_DIR = path.join(process.cwd(), "content/stack/reference/eql"); +// Where the machine-readable manifest is surfaced for the v2 API-reference +// generator (scripts/generate-eql-api-docs.ts). Gitignored; best-effort. +const RELEASE_MANIFEST = path.join(process.cwd(), ".eql-manifest.release.json"); /** * Check if a tarball URL exists (returns HTTP 200) @@ -116,14 +119,16 @@ function escapeMdxSpecials(content: string): string { const escaped = parts .map((part, i) => { if (i % 2 === 1) return part; - return part - .replace(/\{/g, "\\{") - .replace(/\}/g, "\\}") - // Escape `<` unless it begins a real JSX/HTML tag, a closing - // tag, or an autolink (followed by a lowercase letter, `_`, `$`, - // or `/`). Uppercase-led tokens like `` are type placeholders - // in the API reference, not JSX, so they must be escaped too. - .replace(/<(?![a-z_$/])/g, "\\<"); + return ( + part + .replace(/\{/g, "\\{") + .replace(/\}/g, "\\}") + // Escape `<` unless it begins a real JSX/HTML tag, a closing + // tag, or an autolink (followed by a lowercase letter, `_`, `$`, + // or `/`). Uppercase-led tokens like `` are type placeholders + // in the API reference, not JSX, so they must be escaped too. + .replace(/<(?![a-z_$/])/g, "\\<") + ); }) .join(""); result.push(escaped); @@ -253,6 +258,22 @@ async function main() { "utf8", ); + // Surface the machine-readable manifest for the v2 API-reference generator. + // Only releases packaged with the manifest carry it; older ones don't, so + // this is best-effort and the generator falls back to the committed sample. + const manifestSrc = path.join(extractPath, "json", "eql-manifest.json"); + try { + await fs.copyFile(manifestSrc, RELEASE_MANIFEST); + console.log( + `✓ Extracted eql-manifest.json → ${path.basename(RELEASE_MANIFEST)}`, + ); + } catch { + await fs.rm(RELEASE_MANIFEST, { force: true }); // clear any stale cache + console.log( + "• No eql-manifest.json in this release; API reference uses the sample.", + ); + } + // Clean up console.log("Cleaning up..."); await fs.rm(extractPath, { recursive: true, force: true });