diff --git a/docs/warehouse-rollups.md b/docs/warehouse-rollups.md index 3f202663a5..db12e432e0 100644 --- a/docs/warehouse-rollups.md +++ b/docs/warehouse-rollups.md @@ -20,7 +20,7 @@ DDL snapshot, `local-schema.sql` for the embedded chDB engine, and the Rust inse **A query must read the coarsest tier that can answer it.** The routing guards (`canUseAnnualServiceOverview`, `canUseTracesAggregatesMv`, `canUseServiceOverviewMv`, -`canUseLogsAggregatesHourly`) exist to enforce that, and each one names the tier it unlocks. +`canUseLogsAggregatesHourly`, `canUseTraceFacetsRollup`) exist to enforce that, and each one names the tier it unlocks. Rollup routes union a **raw edge** with a **rollup interior**: the rollup answers whole buckets, and the raw table covers the partial buckets at each end of the window. Getting the diff --git a/packages/backend/src/services/warehouse/QueryEngineService.test.ts b/packages/backend/src/services/warehouse/QueryEngineService.test.ts index 9b884ddfbc..66e7b25ef6 100644 --- a/packages/backend/src/services/warehouse/QueryEngineService.test.ts +++ b/packages/backend/src/services/warehouse/QueryEngineService.test.ts @@ -4,7 +4,12 @@ import { TestClock } from "effect/testing" import { Deferred, Effect, Exit, Fiber, Option, Schema } from "effect" import { strict as nodeAssert } from "node:assert" import { MetricName, OrgId, ServiceName, UserId } from "@maple/domain" -import { RawSqlValidationError, WarehouseUpstreamError } from "@maple/domain/http" +import { + RawSqlValidationError, + WarehouseConfigError, + WarehouseQuotaExceededError, + WarehouseUpstreamError, +} from "@maple/domain/http" import { baselineWarehouseCapabilities, type QueryEngineEvaluateRequest, @@ -293,6 +298,80 @@ describe("makeQueryEngineExecute", () => { }), ) + // The sidebar's two reads: a failure on the first one is what either returns. + const traceSidebarStub = (failRollup: () => unknown) => { + const queries: Array = [] + const execute = makeQueryEngineExecute( + makeTinybirdStub({ + sqlQuery: (_tenant, sql) => { + queries.push(sql) + return sql.includes("trace_facets_hourly") + ? Effect.fail(failRollup() as WarehouseConfigError) + : Effect.succeed( + sql.includes("facetType") + ? [{ name: "api", count: 3, facetType: "service" }] + : [ + { + minDurationMs: 1, + maxDurationMs: 9, + p50DurationMs: 4, + p95DurationMs: 8, + }, + ], + ) + }, + }), + ) + return { queries, execute } + } + const sidebarRequest = (kind: "facets" | "stats") => ({ + startTime: "2026-01-01 00:00:00", + endTime: "2026-01-08 00:00:00", + query: { kind, source: "traces" as const }, + }) + + it.effect("reads trace_list_mv when the cluster lacks trace_facets_hourly", () => + Effect.gen(function* () { + for (const kind of ["facets", "stats"] as const) { + const { queries, execute } = traceSidebarStub( + () => + new WarehouseConfigError({ + message: "Unknown table expression identifier 'trace_facets_hourly'", + pipeName: "tracesFacets", + clickhouseType: "UNKNOWN_TABLE", + }), + ) + const response = yield* execute(tenant, sidebarRequest(kind)) + + assert.strictEqual(queries.length, 2, kind) + assert.ok(queries[0]!.includes("trace_facets_hourly"), kind) + assert.ok(!queries[1]!.includes("trace_facets_hourly"), kind) + assert.strictEqual(response.result.kind, kind) + } + }), + ) + + it.effect("surfaces a rollup read that failed for another reason instead of rereading raw", () => + Effect.gen(function* () { + const { queries, execute } = traceSidebarStub( + () => + new WarehouseQuotaExceededError({ + message: "Timeout exceeded while reading from table default.trace_facets_hourly", + pipeName: "tracesFacets", + setting: "max_execution_time", + }), + ) + const exit = yield* Effect.exit(execute(tenant, sidebarRequest("facets"))) + + assert.isTrue(Exit.isFailure(exit)) + assert.strictEqual(queries.length, 1) + assert.strictEqual( + (getFailure(exit) as { _tag?: string } | undefined)?._tag, + "@maple/http/errors/WarehouseQuotaExceededError", + ) + }), + ) + it.effect("fills missing buckets while preserving existing traces values", () => Effect.gen(function* () { const execute = makeQueryEngineExecute( diff --git a/packages/backend/src/services/warehouse/trace-facets-hourly-materialization.clickhouse.e2e.test.ts b/packages/backend/src/services/warehouse/trace-facets-hourly-materialization.clickhouse.e2e.test.ts index f9a16e3c43..ca0d4323f5 100644 --- a/packages/backend/src/services/warehouse/trace-facets-hourly-materialization.clickhouse.e2e.test.ts +++ b/packages/backend/src/services/warehouse/trace-facets-hourly-materialization.clickhouse.e2e.test.ts @@ -8,6 +8,8 @@ import { afterAll, assert, beforeAll, describe, it } from "@effect/vitest" import { migrations, renderStatementFull } from "@maple/domain/clickhouse" +import * as CH from "@maple/query-engine/ch" +import { normalizeSqlForClickHouseClient } from "@maple/query-engine/execution" import { applyRealMigrations, clickhouseE2eEnabled, @@ -53,7 +55,15 @@ const BATCH_1: ReadonlyArray = [ // Not a root span: never reaches trace_list_mv, so never the rollup. { traceId: "t1", ms: BASE_MS + 61_000, service: "api", durationNs: 1_000, parentSpanId: "root-t1" }, ] -const BATCH_2: ReadonlyArray = [root("t5", BASE_MS + 240_000, "api", 3_000_000)] +const BATCH_2: ReadonlyArray = [ + root("t5", BASE_MS + 240_000, "api", 3_000_000), + // The longest span sits in a whole hour, so only the hourly tier can report it. + root("t6", BASE_MS + HOUR_MS + 120_000, "api", 80_000_000), + root("t7", BASE_MS + 2 * HOUR_MS + 300_000, "api", 4_000_000), + root("t8", BASE_MS + 2 * HOUR_MS + 900_000, "api", 6_000_000), + // The only failing staging root inside a whole hour: proves filters reach the hourly tier. + root("t9", BASE_MS + HOUR_MS + 180_000, "worker", 30_000_000, { status: "Error", env: "staging" }), +] const insert = async (spans: ReadonlyArray): Promise => { const rows = spans @@ -114,7 +124,7 @@ describe.skipIf(!clickhouseE2eEnabled)("trace_facets_hourly materialization", () const expected = await fromTraceList() assert.strictEqual( expected.reduce((total, row) => total + Number(row.traces), 0), - 5, + 9, ) assert.deepStrictEqual(await fromRollup(), expected) @@ -125,4 +135,42 @@ describe.skipIf(!clickhouseE2eEnabled)("trace_facets_hourly materialization", () await clickhouseExec(renderStatementFull(backfill, database), database) assert.deepStrictEqual(await fromRollup(), expected) }) + + // Starts mid-hour after t1 and ends mid-hour between t7 and t8, so both raw + // edges hold rows in and out of the window; the aligned window has an empty + // raw tier, whose zero extremes must not reach the result. + it("answers the sidebar identically from the splice and from trace_list_mv alone", async () => { + const run = (sql: string) => runJson(normalizeSqlForClickHouseClient(sql)) + const byFacet = (rows: ReadonlyArray>) => + [...rows].sort((a, b) => `${a.facetType}:${a.name}`.localeCompare(`${b.facetType}:${b.name}`)) + + for (const [filters, longestMs] of [ + [{}, 80], + [{ hasError: true, deploymentEnvs: ["staging"] }, 30], + ] as const) { + for (const [startMs, endMs] of [ + [BASE_MS + 90_000, BASE_MS + 2 * HOUR_MS + 600_000], + [BASE_MS + HOUR_MS, BASE_MS + 2 * HOUR_MS], + ] as const) { + const window = { orgId: ORG_ID, startTime: chDateTime(startMs), endTime: chDateTime(endMs) } + const facets = (rawOnly: boolean) => + CH.compileUnionUnsafe(CH.tracesFacetsQuery({ ...filters, rawOnly }), window).sql + const stats = (rawOnly: boolean) => + CH.compileUnsafe(CH.tracesDurationStatsQuery({ ...filters, rawOnly }), window).sql + assert.include(facets(false), "trace_facets_hourly") + assert.notInclude(facets(true), "trace_facets_hourly") + assert.notInclude(stats(true), "trace_facets_hourly") + + assert.deepStrictEqual(byFacet(await run(facets(false))), byFacet(await run(facets(true)))) + const [spliced] = await run(stats(false)) + const [raw] = await run(stats(true)) + assert.strictEqual(spliced!.minDurationMs, raw!.minDurationMs) + assert.strictEqual(spliced!.maxDurationMs, longestMs) + assert.strictEqual(raw!.maxDurationMs, longestMs) + for (const key of ["p50DurationMs", "p95DurationMs"]) { + assert.closeTo(Number(spliced![key]), Number(raw![key]), Number(raw![key]) * 0.01) + } + } + } + }) }) diff --git a/packages/query-engine/src/__sql_baseline__/catalog.sql b/packages/query-engine/src/__sql_baseline__/catalog.sql index 906f5d4224..7cc21615e9 100644 --- a/packages/query-engine/src/__sql_baseline__/catalog.sql +++ b/packages/query-engine/src/__sql_baseline__/catalog.sql @@ -683,6 +683,255 @@ SELECT LIMIT 1 FORMAT JSON +-- builder:errors:tracesDurationStatsQuery:rollup [b8a8d457] +SELECT + minIf(durationMin, traceCount > 0) / 1000000 AS minDurationMs, + maxIf(durationMax, traceCount > 0) / 1000000 AS maxDurationMs, + ifNull(ifNotFinite(arrayElement(quantilesTDigestMerge(0.5, 0.95)(durationQuantiles), 1) / 1000000, 0), 0) AS p50DurationMs, + ifNull(ifNotFinite(arrayElement(quantilesTDigestMerge(0.5, 0.95)(durationQuantiles), 2) / 1000000, 0), 0) AS p95DurationMs + FROM ( +SELECT + count() AS traceCount, + min(Duration) AS durationMin, + max(Duration) AS durationMax, + quantilesTDigestState(0.5, 0.95)(Duration) AS durationQuantiles + FROM trace_list_mv + WHERE OrgId = 'org_sql_catalog' + AND Timestamp >= '2026-01-01 10:30:00' + AND Timestamp <= '2026-01-03 14:15:00' + AND ServiceName = 'api' + AND (Timestamp < if(toDateTime('2026-01-01 10:30:00') = toStartOfHour(toDateTime('2026-01-01 10:30:00')), toStartOfHour(toDateTime('2026-01-01 10:30:00')), toStartOfHour(toDateTime('2026-01-01 10:30:00')) + INTERVAL 1 HOUR) OR Timestamp >= toStartOfHour(toDateTime('2026-01-03 14:15:00'))) +UNION ALL +SELECT + sum(TraceCount) AS traceCount, + min(DurationMin) AS durationMin, + max(DurationMax) AS durationMax, + quantilesTDigestMergeState(0.5, 0.95)(DurationQuantiles) AS durationQuantiles + FROM trace_facets_hourly + WHERE OrgId = 'org_sql_catalog' + AND Hour >= if(toDateTime('2026-01-01 10:30:00') = toStartOfHour(toDateTime('2026-01-01 10:30:00')), toStartOfHour(toDateTime('2026-01-01 10:30:00')), toStartOfHour(toDateTime('2026-01-01 10:30:00')) + INTERVAL 1 HOUR) + AND Hour < toStartOfHour(toDateTime('2026-01-03 14:15:00')) + AND ServiceName = 'api' +) AS duration_tiers + FORMAT JSON + +-- builder:errors:tracesFacetsQuery:rollup [39248faa] +SELECT + name AS name, + sum(count) AS count, + 'service' AS facetType + FROM ( +SELECT + ServiceName AS name, + count() AS count + FROM trace_list_mv + WHERE OrgId = 'org_sql_catalog' + AND Timestamp >= '2026-01-01 10:30:00' + AND Timestamp <= '2026-01-03 14:15:00' + AND ServiceName = 'api' + AND (Timestamp < if(toDateTime('2026-01-01 10:30:00') = toStartOfHour(toDateTime('2026-01-01 10:30:00')), toStartOfHour(toDateTime('2026-01-01 10:30:00')), toStartOfHour(toDateTime('2026-01-01 10:30:00')) + INTERVAL 1 HOUR) OR Timestamp >= toStartOfHour(toDateTime('2026-01-03 14:15:00'))) + GROUP BY name +UNION ALL +SELECT + ServiceName AS name, + sum(TraceCount) AS count + FROM trace_facets_hourly + WHERE OrgId = 'org_sql_catalog' + AND Hour >= if(toDateTime('2026-01-01 10:30:00') = toStartOfHour(toDateTime('2026-01-01 10:30:00')), toStartOfHour(toDateTime('2026-01-01 10:30:00')), toStartOfHour(toDateTime('2026-01-01 10:30:00')) + INTERVAL 1 HOUR) + AND Hour < toStartOfHour(toDateTime('2026-01-03 14:15:00')) + AND ServiceName = 'api' + GROUP BY name +) AS service_tiers + GROUP BY name + ORDER BY count DESC + LIMIT 50 +UNION ALL +SELECT + name AS name, + sum(count) AS count, + 'spanName' AS facetType + FROM ( +SELECT + SpanName AS name, + count() AS count + FROM trace_list_mv + WHERE OrgId = 'org_sql_catalog' + AND Timestamp >= '2026-01-01 10:30:00' + AND Timestamp <= '2026-01-03 14:15:00' + AND ServiceName = 'api' + AND (Timestamp < if(toDateTime('2026-01-01 10:30:00') = toStartOfHour(toDateTime('2026-01-01 10:30:00')), toStartOfHour(toDateTime('2026-01-01 10:30:00')), toStartOfHour(toDateTime('2026-01-01 10:30:00')) + INTERVAL 1 HOUR) OR Timestamp >= toStartOfHour(toDateTime('2026-01-03 14:15:00'))) + AND SpanName != '' + GROUP BY name +UNION ALL +SELECT + SpanName AS name, + sum(TraceCount) AS count + FROM trace_facets_hourly + WHERE OrgId = 'org_sql_catalog' + AND Hour >= if(toDateTime('2026-01-01 10:30:00') = toStartOfHour(toDateTime('2026-01-01 10:30:00')), toStartOfHour(toDateTime('2026-01-01 10:30:00')), toStartOfHour(toDateTime('2026-01-01 10:30:00')) + INTERVAL 1 HOUR) + AND Hour < toStartOfHour(toDateTime('2026-01-03 14:15:00')) + AND ServiceName = 'api' + AND SpanName != '' + GROUP BY name +) AS spanName_tiers + GROUP BY name + ORDER BY count DESC + LIMIT 20 +UNION ALL +SELECT + name AS name, + sum(count) AS count, + 'httpMethod' AS facetType + FROM ( +SELECT + HttpMethod AS name, + count() AS count + FROM trace_list_mv + WHERE OrgId = 'org_sql_catalog' + AND Timestamp >= '2026-01-01 10:30:00' + AND Timestamp <= '2026-01-03 14:15:00' + AND ServiceName = 'api' + AND (Timestamp < if(toDateTime('2026-01-01 10:30:00') = toStartOfHour(toDateTime('2026-01-01 10:30:00')), toStartOfHour(toDateTime('2026-01-01 10:30:00')), toStartOfHour(toDateTime('2026-01-01 10:30:00')) + INTERVAL 1 HOUR) OR Timestamp >= toStartOfHour(toDateTime('2026-01-03 14:15:00'))) + AND HttpMethod != '' + GROUP BY name +UNION ALL +SELECT + HttpMethod AS name, + sum(TraceCount) AS count + FROM trace_facets_hourly + WHERE OrgId = 'org_sql_catalog' + AND Hour >= if(toDateTime('2026-01-01 10:30:00') = toStartOfHour(toDateTime('2026-01-01 10:30:00')), toStartOfHour(toDateTime('2026-01-01 10:30:00')), toStartOfHour(toDateTime('2026-01-01 10:30:00')) + INTERVAL 1 HOUR) + AND Hour < toStartOfHour(toDateTime('2026-01-03 14:15:00')) + AND ServiceName = 'api' + AND HttpMethod != '' + GROUP BY name +) AS httpMethod_tiers + GROUP BY name + ORDER BY count DESC + LIMIT 20 +UNION ALL +SELECT + name AS name, + sum(count) AS count, + 'httpStatus' AS facetType + FROM ( +SELECT + HttpStatusCode AS name, + count() AS count + FROM trace_list_mv + WHERE OrgId = 'org_sql_catalog' + AND Timestamp >= '2026-01-01 10:30:00' + AND Timestamp <= '2026-01-03 14:15:00' + AND ServiceName = 'api' + AND (Timestamp < if(toDateTime('2026-01-01 10:30:00') = toStartOfHour(toDateTime('2026-01-01 10:30:00')), toStartOfHour(toDateTime('2026-01-01 10:30:00')), toStartOfHour(toDateTime('2026-01-01 10:30:00')) + INTERVAL 1 HOUR) OR Timestamp >= toStartOfHour(toDateTime('2026-01-03 14:15:00'))) + AND HttpStatusCode != '' + GROUP BY name +UNION ALL +SELECT + HttpStatusCode AS name, + sum(TraceCount) AS count + FROM trace_facets_hourly + WHERE OrgId = 'org_sql_catalog' + AND Hour >= if(toDateTime('2026-01-01 10:30:00') = toStartOfHour(toDateTime('2026-01-01 10:30:00')), toStartOfHour(toDateTime('2026-01-01 10:30:00')), toStartOfHour(toDateTime('2026-01-01 10:30:00')) + INTERVAL 1 HOUR) + AND Hour < toStartOfHour(toDateTime('2026-01-03 14:15:00')) + AND ServiceName = 'api' + AND HttpStatusCode != '' + GROUP BY name +) AS httpStatus_tiers + GROUP BY name + ORDER BY count DESC + LIMIT 20 +UNION ALL +SELECT + name AS name, + sum(count) AS count, + 'deploymentEnv' AS facetType + FROM ( +SELECT + DeploymentEnv AS name, + count() AS count + FROM trace_list_mv + WHERE OrgId = 'org_sql_catalog' + AND Timestamp >= '2026-01-01 10:30:00' + AND Timestamp <= '2026-01-03 14:15:00' + AND ServiceName = 'api' + AND (Timestamp < if(toDateTime('2026-01-01 10:30:00') = toStartOfHour(toDateTime('2026-01-01 10:30:00')), toStartOfHour(toDateTime('2026-01-01 10:30:00')), toStartOfHour(toDateTime('2026-01-01 10:30:00')) + INTERVAL 1 HOUR) OR Timestamp >= toStartOfHour(toDateTime('2026-01-03 14:15:00'))) + AND DeploymentEnv != '' + GROUP BY name +UNION ALL +SELECT + DeploymentEnv AS name, + sum(TraceCount) AS count + FROM trace_facets_hourly + WHERE OrgId = 'org_sql_catalog' + AND Hour >= if(toDateTime('2026-01-01 10:30:00') = toStartOfHour(toDateTime('2026-01-01 10:30:00')), toStartOfHour(toDateTime('2026-01-01 10:30:00')), toStartOfHour(toDateTime('2026-01-01 10:30:00')) + INTERVAL 1 HOUR) + AND Hour < toStartOfHour(toDateTime('2026-01-03 14:15:00')) + AND ServiceName = 'api' + AND DeploymentEnv != '' + GROUP BY name +) AS deploymentEnv_tiers + GROUP BY name + ORDER BY count DESC + LIMIT 20 +UNION ALL +SELECT + name AS name, + sum(count) AS count, + 'serviceNamespace' AS facetType + FROM ( +SELECT + ServiceNamespace AS name, + count() AS count + FROM trace_list_mv + WHERE OrgId = 'org_sql_catalog' + AND Timestamp >= '2026-01-01 10:30:00' + AND Timestamp <= '2026-01-03 14:15:00' + AND ServiceName = 'api' + AND (Timestamp < if(toDateTime('2026-01-01 10:30:00') = toStartOfHour(toDateTime('2026-01-01 10:30:00')), toStartOfHour(toDateTime('2026-01-01 10:30:00')), toStartOfHour(toDateTime('2026-01-01 10:30:00')) + INTERVAL 1 HOUR) OR Timestamp >= toStartOfHour(toDateTime('2026-01-03 14:15:00'))) + AND ServiceNamespace != '' + GROUP BY name +UNION ALL +SELECT + ServiceNamespace AS name, + sum(TraceCount) AS count + FROM trace_facets_hourly + WHERE OrgId = 'org_sql_catalog' + AND Hour >= if(toDateTime('2026-01-01 10:30:00') = toStartOfHour(toDateTime('2026-01-01 10:30:00')), toStartOfHour(toDateTime('2026-01-01 10:30:00')), toStartOfHour(toDateTime('2026-01-01 10:30:00')) + INTERVAL 1 HOUR) + AND Hour < toStartOfHour(toDateTime('2026-01-03 14:15:00')) + AND ServiceName = 'api' + AND ServiceNamespace != '' + GROUP BY name +) AS serviceNamespace_tiers + GROUP BY name + ORDER BY count DESC + LIMIT 20 +UNION ALL +SELECT + 'error' AS name, + sum(count) AS count, + 'errorCount' AS facetType + FROM ( +SELECT + count() AS count + FROM trace_list_mv + WHERE OrgId = 'org_sql_catalog' + AND Timestamp >= '2026-01-01 10:30:00' + AND Timestamp <= '2026-01-03 14:15:00' + AND ServiceName = 'api' + AND (Timestamp < if(toDateTime('2026-01-01 10:30:00') = toStartOfHour(toDateTime('2026-01-01 10:30:00')), toStartOfHour(toDateTime('2026-01-01 10:30:00')), toStartOfHour(toDateTime('2026-01-01 10:30:00')) + INTERVAL 1 HOUR) OR Timestamp >= toStartOfHour(toDateTime('2026-01-03 14:15:00'))) + AND HasError = 1 +UNION ALL +SELECT + sum(TraceCount) AS count + FROM trace_facets_hourly + WHERE OrgId = 'org_sql_catalog' + AND Hour >= if(toDateTime('2026-01-01 10:30:00') = toStartOfHour(toDateTime('2026-01-01 10:30:00')), toStartOfHour(toDateTime('2026-01-01 10:30:00')), toStartOfHour(toDateTime('2026-01-01 10:30:00')) + INTERVAL 1 HOUR) + AND Hour < toStartOfHour(toDateTime('2026-01-03 14:15:00')) + AND ServiceName = 'api' + AND HasError = 1 +) AS errorCount_tiers +FORMAT JSON + -- builder:infra:hostGaugeTimeseriesQuery:default [7d9ca740] SELECT toStartOfInterval(TimeUnix, INTERVAL 300 SECOND) AS bucket, @@ -10349,23 +10598,34 @@ SELECT LIMIT 20 FORMAT JSON --- pipe:traces_duration_stats:default:baseline [dc834b4b] +-- pipe:traces_duration_stats:default:baseline [379189b9] +SELECT + minIf(durationMin, traceCount > 0) / 1000000 AS minDurationMs, + maxIf(durationMax, traceCount > 0) / 1000000 AS maxDurationMs, + ifNull(ifNotFinite(arrayElement(quantilesTDigestMerge(0.5, 0.95)(durationQuantiles), 1) / 1000000, 0), 0) AS p50DurationMs, + ifNull(ifNotFinite(arrayElement(quantilesTDigestMerge(0.5, 0.95)(durationQuantiles), 2) / 1000000, 0), 0) AS p95DurationMs + FROM ( SELECT - min(Duration) / 1000000 AS minDurationMs, - max(Duration) / 1000000 AS maxDurationMs, - ifNull(ifNotFinite(quantile(0.5)(Duration) / 1000000, 0), 0) AS p50DurationMs, - ifNull(ifNotFinite(quantile(0.95)(Duration) / 1000000, 0), 0) AS p95DurationMs + count() AS traceCount, + min(Duration) AS durationMin, + max(Duration) AS durationMax, + quantilesTDigestState(0.5, 0.95)(Duration) AS durationQuantiles FROM trace_list_mv WHERE OrgId = 'org_sql_catalog' AND Timestamp >= '2026-01-01 10:30:00' AND Timestamp <= '2026-01-03 14:15:00' +) AS duration_tiers FORMAT JSON --- pipe:traces_facets:attribute-filtered:baseline [952a9803] +-- pipe:traces_facets:attribute-filtered:baseline [2e8577ae] SELECT - ServiceName AS name, - count() AS count, + name AS name, + sum(count) AS count, 'service' AS facetType + FROM ( +SELECT + ServiceName AS name, + count() AS count FROM trace_list_mv WHERE OrgId = 'org_sql_catalog' AND Timestamp >= '2026-01-01 10:30:00' @@ -10387,13 +10647,19 @@ SELECT AND t_res.Timestamp <= '2026-01-03 14:15:00' AND t_res.ResourceAttributes['host.name'] = 'web') GROUP BY name +) AS service_tiers + GROUP BY name ORDER BY count DESC LIMIT 50 UNION ALL SELECT - SpanName AS name, - count() AS count, + name AS name, + sum(count) AS count, 'spanName' AS facetType + FROM ( +SELECT + SpanName AS name, + count() AS count FROM trace_list_mv WHERE OrgId = 'org_sql_catalog' AND Timestamp >= '2026-01-01 10:30:00' @@ -10416,13 +10682,19 @@ SELECT AND t_res.ResourceAttributes['host.name'] = 'web') AND SpanName != '' GROUP BY name +) AS spanName_tiers + GROUP BY name ORDER BY count DESC LIMIT 20 UNION ALL SELECT - HttpMethod AS name, - count() AS count, + name AS name, + sum(count) AS count, 'httpMethod' AS facetType + FROM ( +SELECT + HttpMethod AS name, + count() AS count FROM trace_list_mv WHERE OrgId = 'org_sql_catalog' AND Timestamp >= '2026-01-01 10:30:00' @@ -10445,13 +10717,19 @@ SELECT AND t_res.ResourceAttributes['host.name'] = 'web') AND HttpMethod != '' GROUP BY name +) AS httpMethod_tiers + GROUP BY name ORDER BY count DESC LIMIT 20 UNION ALL SELECT - HttpStatusCode AS name, - count() AS count, + name AS name, + sum(count) AS count, 'httpStatus' AS facetType + FROM ( +SELECT + HttpStatusCode AS name, + count() AS count FROM trace_list_mv WHERE OrgId = 'org_sql_catalog' AND Timestamp >= '2026-01-01 10:30:00' @@ -10474,13 +10752,19 @@ SELECT AND t_res.ResourceAttributes['host.name'] = 'web') AND HttpStatusCode != '' GROUP BY name +) AS httpStatus_tiers + GROUP BY name ORDER BY count DESC LIMIT 20 UNION ALL SELECT - DeploymentEnv AS name, - count() AS count, + name AS name, + sum(count) AS count, 'deploymentEnv' AS facetType + FROM ( +SELECT + DeploymentEnv AS name, + count() AS count FROM trace_list_mv WHERE OrgId = 'org_sql_catalog' AND Timestamp >= '2026-01-01 10:30:00' @@ -10503,13 +10787,19 @@ SELECT AND t_res.ResourceAttributes['host.name'] = 'web') AND DeploymentEnv != '' GROUP BY name +) AS deploymentEnv_tiers + GROUP BY name ORDER BY count DESC LIMIT 20 UNION ALL SELECT - ServiceNamespace AS name, - count() AS count, + name AS name, + sum(count) AS count, 'serviceNamespace' AS facetType + FROM ( +SELECT + ServiceNamespace AS name, + count() AS count FROM trace_list_mv WHERE OrgId = 'org_sql_catalog' AND Timestamp >= '2026-01-01 10:30:00' @@ -10532,13 +10822,18 @@ SELECT AND t_res.ResourceAttributes['host.name'] = 'web') AND ServiceNamespace != '' GROUP BY name +) AS serviceNamespace_tiers + GROUP BY name ORDER BY count DESC LIMIT 20 UNION ALL SELECT 'error' AS name, - count() AS count, + sum(count) AS count, 'errorCount' AS facetType + FROM ( +SELECT + count() AS count FROM trace_list_mv WHERE OrgId = 'org_sql_catalog' AND Timestamp >= '2026-01-01 10:30:00' @@ -10560,13 +10855,18 @@ SELECT AND t_res.Timestamp <= '2026-01-03 14:15:00' AND t_res.ResourceAttributes['host.name'] = 'web') AND HasError = 1 +) AS errorCount_tiers FORMAT JSON --- pipe:traces_facets:attribute-filtered:bloom [952a9803] +-- pipe:traces_facets:attribute-filtered:bloom [2e8577ae] SELECT - ServiceName AS name, - count() AS count, + name AS name, + sum(count) AS count, 'service' AS facetType + FROM ( +SELECT + ServiceName AS name, + count() AS count FROM trace_list_mv WHERE OrgId = 'org_sql_catalog' AND Timestamp >= '2026-01-01 10:30:00' @@ -10588,13 +10888,19 @@ SELECT AND t_res.Timestamp <= '2026-01-03 14:15:00' AND t_res.ResourceAttributes['host.name'] = 'web') GROUP BY name +) AS service_tiers + GROUP BY name ORDER BY count DESC LIMIT 50 UNION ALL SELECT - SpanName AS name, - count() AS count, + name AS name, + sum(count) AS count, 'spanName' AS facetType + FROM ( +SELECT + SpanName AS name, + count() AS count FROM trace_list_mv WHERE OrgId = 'org_sql_catalog' AND Timestamp >= '2026-01-01 10:30:00' @@ -10617,13 +10923,19 @@ SELECT AND t_res.ResourceAttributes['host.name'] = 'web') AND SpanName != '' GROUP BY name +) AS spanName_tiers + GROUP BY name ORDER BY count DESC LIMIT 20 UNION ALL SELECT - HttpMethod AS name, - count() AS count, + name AS name, + sum(count) AS count, 'httpMethod' AS facetType + FROM ( +SELECT + HttpMethod AS name, + count() AS count FROM trace_list_mv WHERE OrgId = 'org_sql_catalog' AND Timestamp >= '2026-01-01 10:30:00' @@ -10646,13 +10958,19 @@ SELECT AND t_res.ResourceAttributes['host.name'] = 'web') AND HttpMethod != '' GROUP BY name +) AS httpMethod_tiers + GROUP BY name ORDER BY count DESC LIMIT 20 UNION ALL SELECT - HttpStatusCode AS name, - count() AS count, + name AS name, + sum(count) AS count, 'httpStatus' AS facetType + FROM ( +SELECT + HttpStatusCode AS name, + count() AS count FROM trace_list_mv WHERE OrgId = 'org_sql_catalog' AND Timestamp >= '2026-01-01 10:30:00' @@ -10675,13 +10993,19 @@ SELECT AND t_res.ResourceAttributes['host.name'] = 'web') AND HttpStatusCode != '' GROUP BY name +) AS httpStatus_tiers + GROUP BY name ORDER BY count DESC LIMIT 20 UNION ALL SELECT - DeploymentEnv AS name, - count() AS count, + name AS name, + sum(count) AS count, 'deploymentEnv' AS facetType + FROM ( +SELECT + DeploymentEnv AS name, + count() AS count FROM trace_list_mv WHERE OrgId = 'org_sql_catalog' AND Timestamp >= '2026-01-01 10:30:00' @@ -10704,13 +11028,19 @@ SELECT AND t_res.ResourceAttributes['host.name'] = 'web') AND DeploymentEnv != '' GROUP BY name +) AS deploymentEnv_tiers + GROUP BY name ORDER BY count DESC LIMIT 20 UNION ALL SELECT - ServiceNamespace AS name, - count() AS count, + name AS name, + sum(count) AS count, 'serviceNamespace' AS facetType + FROM ( +SELECT + ServiceNamespace AS name, + count() AS count FROM trace_list_mv WHERE OrgId = 'org_sql_catalog' AND Timestamp >= '2026-01-01 10:30:00' @@ -10733,13 +11063,18 @@ SELECT AND t_res.ResourceAttributes['host.name'] = 'web') AND ServiceNamespace != '' GROUP BY name +) AS serviceNamespace_tiers + GROUP BY name ORDER BY count DESC LIMIT 20 UNION ALL SELECT 'error' AS name, - count() AS count, + sum(count) AS count, 'errorCount' AS facetType + FROM ( +SELECT + count() AS count FROM trace_list_mv WHERE OrgId = 'org_sql_catalog' AND Timestamp >= '2026-01-01 10:30:00' @@ -10761,13 +11096,18 @@ SELECT AND t_res.Timestamp <= '2026-01-03 14:15:00' AND t_res.ResourceAttributes['host.name'] = 'web') AND HasError = 1 +) AS errorCount_tiers FORMAT JSON --- pipe:traces_facets:attribute-filtered:text [952a9803] +-- pipe:traces_facets:attribute-filtered:text [2e8577ae] SELECT - ServiceName AS name, - count() AS count, + name AS name, + sum(count) AS count, 'service' AS facetType + FROM ( +SELECT + ServiceName AS name, + count() AS count FROM trace_list_mv WHERE OrgId = 'org_sql_catalog' AND Timestamp >= '2026-01-01 10:30:00' @@ -10789,13 +11129,19 @@ SELECT AND t_res.Timestamp <= '2026-01-03 14:15:00' AND t_res.ResourceAttributes['host.name'] = 'web') GROUP BY name +) AS service_tiers + GROUP BY name ORDER BY count DESC LIMIT 50 UNION ALL SELECT - SpanName AS name, - count() AS count, + name AS name, + sum(count) AS count, 'spanName' AS facetType + FROM ( +SELECT + SpanName AS name, + count() AS count FROM trace_list_mv WHERE OrgId = 'org_sql_catalog' AND Timestamp >= '2026-01-01 10:30:00' @@ -10818,13 +11164,19 @@ SELECT AND t_res.ResourceAttributes['host.name'] = 'web') AND SpanName != '' GROUP BY name +) AS spanName_tiers + GROUP BY name ORDER BY count DESC LIMIT 20 UNION ALL SELECT - HttpMethod AS name, - count() AS count, + name AS name, + sum(count) AS count, 'httpMethod' AS facetType + FROM ( +SELECT + HttpMethod AS name, + count() AS count FROM trace_list_mv WHERE OrgId = 'org_sql_catalog' AND Timestamp >= '2026-01-01 10:30:00' @@ -10847,13 +11199,19 @@ SELECT AND t_res.ResourceAttributes['host.name'] = 'web') AND HttpMethod != '' GROUP BY name +) AS httpMethod_tiers + GROUP BY name ORDER BY count DESC LIMIT 20 UNION ALL SELECT - HttpStatusCode AS name, - count() AS count, + name AS name, + sum(count) AS count, 'httpStatus' AS facetType + FROM ( +SELECT + HttpStatusCode AS name, + count() AS count FROM trace_list_mv WHERE OrgId = 'org_sql_catalog' AND Timestamp >= '2026-01-01 10:30:00' @@ -10876,13 +11234,19 @@ SELECT AND t_res.ResourceAttributes['host.name'] = 'web') AND HttpStatusCode != '' GROUP BY name +) AS httpStatus_tiers + GROUP BY name ORDER BY count DESC LIMIT 20 UNION ALL SELECT - DeploymentEnv AS name, - count() AS count, + name AS name, + sum(count) AS count, 'deploymentEnv' AS facetType + FROM ( +SELECT + DeploymentEnv AS name, + count() AS count FROM trace_list_mv WHERE OrgId = 'org_sql_catalog' AND Timestamp >= '2026-01-01 10:30:00' @@ -10905,13 +11269,19 @@ SELECT AND t_res.ResourceAttributes['host.name'] = 'web') AND DeploymentEnv != '' GROUP BY name +) AS deploymentEnv_tiers + GROUP BY name ORDER BY count DESC LIMIT 20 UNION ALL SELECT - ServiceNamespace AS name, - count() AS count, + name AS name, + sum(count) AS count, 'serviceNamespace' AS facetType + FROM ( +SELECT + ServiceNamespace AS name, + count() AS count FROM trace_list_mv WHERE OrgId = 'org_sql_catalog' AND Timestamp >= '2026-01-01 10:30:00' @@ -10934,13 +11304,18 @@ SELECT AND t_res.ResourceAttributes['host.name'] = 'web') AND ServiceNamespace != '' GROUP BY name +) AS serviceNamespace_tiers + GROUP BY name ORDER BY count DESC LIMIT 20 UNION ALL SELECT 'error' AS name, - count() AS count, + sum(count) AS count, 'errorCount' AS facetType + FROM ( +SELECT + count() AS count FROM trace_list_mv WHERE OrgId = 'org_sql_catalog' AND Timestamp >= '2026-01-01 10:30:00' @@ -10962,273 +11337,394 @@ SELECT AND t_res.Timestamp <= '2026-01-03 14:15:00' AND t_res.ResourceAttributes['host.name'] = 'web') AND HasError = 1 +) AS errorCount_tiers FORMAT JSON --- pipe:traces_facets:default:baseline [3a9bffe8] +-- pipe:traces_facets:default:baseline [a8566199] SELECT - ServiceName AS name, - count() AS count, + name AS name, + sum(count) AS count, 'service' AS facetType + FROM ( +SELECT + ServiceName AS name, + count() AS count FROM trace_list_mv WHERE OrgId = 'org_sql_catalog' AND Timestamp >= '2026-01-01 10:30:00' AND Timestamp <= '2026-01-03 14:15:00' GROUP BY name +) AS service_tiers + GROUP BY name ORDER BY count DESC LIMIT 50 UNION ALL SELECT - SpanName AS name, - count() AS count, + name AS name, + sum(count) AS count, 'spanName' AS facetType + FROM ( +SELECT + SpanName AS name, + count() AS count FROM trace_list_mv WHERE OrgId = 'org_sql_catalog' AND Timestamp >= '2026-01-01 10:30:00' AND Timestamp <= '2026-01-03 14:15:00' AND SpanName != '' GROUP BY name +) AS spanName_tiers + GROUP BY name ORDER BY count DESC LIMIT 20 UNION ALL SELECT - HttpMethod AS name, - count() AS count, + name AS name, + sum(count) AS count, 'httpMethod' AS facetType + FROM ( +SELECT + HttpMethod AS name, + count() AS count FROM trace_list_mv WHERE OrgId = 'org_sql_catalog' AND Timestamp >= '2026-01-01 10:30:00' AND Timestamp <= '2026-01-03 14:15:00' AND HttpMethod != '' GROUP BY name +) AS httpMethod_tiers + GROUP BY name ORDER BY count DESC LIMIT 20 UNION ALL SELECT - HttpStatusCode AS name, - count() AS count, + name AS name, + sum(count) AS count, 'httpStatus' AS facetType + FROM ( +SELECT + HttpStatusCode AS name, + count() AS count FROM trace_list_mv WHERE OrgId = 'org_sql_catalog' AND Timestamp >= '2026-01-01 10:30:00' AND Timestamp <= '2026-01-03 14:15:00' AND HttpStatusCode != '' GROUP BY name +) AS httpStatus_tiers + GROUP BY name ORDER BY count DESC LIMIT 20 UNION ALL SELECT - DeploymentEnv AS name, - count() AS count, + name AS name, + sum(count) AS count, 'deploymentEnv' AS facetType + FROM ( +SELECT + DeploymentEnv AS name, + count() AS count FROM trace_list_mv WHERE OrgId = 'org_sql_catalog' AND Timestamp >= '2026-01-01 10:30:00' AND Timestamp <= '2026-01-03 14:15:00' AND DeploymentEnv != '' GROUP BY name +) AS deploymentEnv_tiers + GROUP BY name ORDER BY count DESC LIMIT 20 UNION ALL SELECT - ServiceNamespace AS name, - count() AS count, + name AS name, + sum(count) AS count, 'serviceNamespace' AS facetType + FROM ( +SELECT + ServiceNamespace AS name, + count() AS count FROM trace_list_mv WHERE OrgId = 'org_sql_catalog' AND Timestamp >= '2026-01-01 10:30:00' AND Timestamp <= '2026-01-03 14:15:00' AND ServiceNamespace != '' GROUP BY name +) AS serviceNamespace_tiers + GROUP BY name ORDER BY count DESC LIMIT 20 UNION ALL SELECT 'error' AS name, - count() AS count, + sum(count) AS count, 'errorCount' AS facetType + FROM ( +SELECT + count() AS count FROM trace_list_mv WHERE OrgId = 'org_sql_catalog' AND Timestamp >= '2026-01-01 10:30:00' AND Timestamp <= '2026-01-03 14:15:00' AND HasError = 1 +) AS errorCount_tiers FORMAT JSON --- pipe:traces_facets:default:bloom [3a9bffe8] +-- pipe:traces_facets:default:bloom [a8566199] SELECT - ServiceName AS name, - count() AS count, + name AS name, + sum(count) AS count, 'service' AS facetType + FROM ( +SELECT + ServiceName AS name, + count() AS count FROM trace_list_mv WHERE OrgId = 'org_sql_catalog' AND Timestamp >= '2026-01-01 10:30:00' AND Timestamp <= '2026-01-03 14:15:00' GROUP BY name +) AS service_tiers + GROUP BY name ORDER BY count DESC LIMIT 50 UNION ALL SELECT - SpanName AS name, - count() AS count, + name AS name, + sum(count) AS count, 'spanName' AS facetType + FROM ( +SELECT + SpanName AS name, + count() AS count FROM trace_list_mv WHERE OrgId = 'org_sql_catalog' AND Timestamp >= '2026-01-01 10:30:00' AND Timestamp <= '2026-01-03 14:15:00' AND SpanName != '' GROUP BY name +) AS spanName_tiers + GROUP BY name ORDER BY count DESC LIMIT 20 UNION ALL SELECT - HttpMethod AS name, - count() AS count, + name AS name, + sum(count) AS count, 'httpMethod' AS facetType + FROM ( +SELECT + HttpMethod AS name, + count() AS count FROM trace_list_mv WHERE OrgId = 'org_sql_catalog' AND Timestamp >= '2026-01-01 10:30:00' AND Timestamp <= '2026-01-03 14:15:00' AND HttpMethod != '' GROUP BY name +) AS httpMethod_tiers + GROUP BY name ORDER BY count DESC LIMIT 20 UNION ALL SELECT - HttpStatusCode AS name, - count() AS count, + name AS name, + sum(count) AS count, 'httpStatus' AS facetType + FROM ( +SELECT + HttpStatusCode AS name, + count() AS count FROM trace_list_mv WHERE OrgId = 'org_sql_catalog' AND Timestamp >= '2026-01-01 10:30:00' AND Timestamp <= '2026-01-03 14:15:00' AND HttpStatusCode != '' GROUP BY name +) AS httpStatus_tiers + GROUP BY name ORDER BY count DESC LIMIT 20 UNION ALL SELECT - DeploymentEnv AS name, - count() AS count, + name AS name, + sum(count) AS count, 'deploymentEnv' AS facetType + FROM ( +SELECT + DeploymentEnv AS name, + count() AS count FROM trace_list_mv WHERE OrgId = 'org_sql_catalog' AND Timestamp >= '2026-01-01 10:30:00' AND Timestamp <= '2026-01-03 14:15:00' AND DeploymentEnv != '' GROUP BY name +) AS deploymentEnv_tiers + GROUP BY name ORDER BY count DESC LIMIT 20 UNION ALL SELECT - ServiceNamespace AS name, - count() AS count, + name AS name, + sum(count) AS count, 'serviceNamespace' AS facetType + FROM ( +SELECT + ServiceNamespace AS name, + count() AS count FROM trace_list_mv WHERE OrgId = 'org_sql_catalog' AND Timestamp >= '2026-01-01 10:30:00' AND Timestamp <= '2026-01-03 14:15:00' AND ServiceNamespace != '' GROUP BY name +) AS serviceNamespace_tiers + GROUP BY name ORDER BY count DESC LIMIT 20 UNION ALL SELECT 'error' AS name, - count() AS count, + sum(count) AS count, 'errorCount' AS facetType + FROM ( +SELECT + count() AS count FROM trace_list_mv WHERE OrgId = 'org_sql_catalog' AND Timestamp >= '2026-01-01 10:30:00' AND Timestamp <= '2026-01-03 14:15:00' AND HasError = 1 +) AS errorCount_tiers FORMAT JSON --- pipe:traces_facets:default:text [3a9bffe8] +-- pipe:traces_facets:default:text [a8566199] SELECT - ServiceName AS name, - count() AS count, + name AS name, + sum(count) AS count, 'service' AS facetType + FROM ( +SELECT + ServiceName AS name, + count() AS count FROM trace_list_mv WHERE OrgId = 'org_sql_catalog' AND Timestamp >= '2026-01-01 10:30:00' AND Timestamp <= '2026-01-03 14:15:00' GROUP BY name +) AS service_tiers + GROUP BY name ORDER BY count DESC LIMIT 50 UNION ALL SELECT - SpanName AS name, - count() AS count, + name AS name, + sum(count) AS count, 'spanName' AS facetType + FROM ( +SELECT + SpanName AS name, + count() AS count FROM trace_list_mv WHERE OrgId = 'org_sql_catalog' AND Timestamp >= '2026-01-01 10:30:00' AND Timestamp <= '2026-01-03 14:15:00' AND SpanName != '' GROUP BY name +) AS spanName_tiers + GROUP BY name ORDER BY count DESC LIMIT 20 UNION ALL SELECT - HttpMethod AS name, - count() AS count, + name AS name, + sum(count) AS count, 'httpMethod' AS facetType + FROM ( +SELECT + HttpMethod AS name, + count() AS count FROM trace_list_mv WHERE OrgId = 'org_sql_catalog' AND Timestamp >= '2026-01-01 10:30:00' AND Timestamp <= '2026-01-03 14:15:00' AND HttpMethod != '' GROUP BY name +) AS httpMethod_tiers + GROUP BY name ORDER BY count DESC LIMIT 20 UNION ALL SELECT - HttpStatusCode AS name, - count() AS count, + name AS name, + sum(count) AS count, 'httpStatus' AS facetType + FROM ( +SELECT + HttpStatusCode AS name, + count() AS count FROM trace_list_mv WHERE OrgId = 'org_sql_catalog' AND Timestamp >= '2026-01-01 10:30:00' AND Timestamp <= '2026-01-03 14:15:00' AND HttpStatusCode != '' GROUP BY name +) AS httpStatus_tiers + GROUP BY name ORDER BY count DESC LIMIT 20 UNION ALL SELECT - DeploymentEnv AS name, - count() AS count, + name AS name, + sum(count) AS count, 'deploymentEnv' AS facetType + FROM ( +SELECT + DeploymentEnv AS name, + count() AS count FROM trace_list_mv WHERE OrgId = 'org_sql_catalog' AND Timestamp >= '2026-01-01 10:30:00' AND Timestamp <= '2026-01-03 14:15:00' AND DeploymentEnv != '' GROUP BY name +) AS deploymentEnv_tiers + GROUP BY name ORDER BY count DESC LIMIT 20 UNION ALL SELECT - ServiceNamespace AS name, - count() AS count, + name AS name, + sum(count) AS count, 'serviceNamespace' AS facetType - FROM trace_list_mv - WHERE OrgId = 'org_sql_catalog' + FROM ( +SELECT + ServiceNamespace AS name, + count() AS count + FROM trace_list_mv + WHERE OrgId = 'org_sql_catalog' AND Timestamp >= '2026-01-01 10:30:00' AND Timestamp <= '2026-01-03 14:15:00' AND ServiceNamespace != '' GROUP BY name +) AS serviceNamespace_tiers + GROUP BY name ORDER BY count DESC LIMIT 20 UNION ALL SELECT 'error' AS name, - count() AS count, + sum(count) AS count, 'errorCount' AS facetType + FROM ( +SELECT + count() AS count FROM trace_list_mv WHERE OrgId = 'org_sql_catalog' AND Timestamp >= '2026-01-01 10:30:00' AND Timestamp <= '2026-01-03 14:15:00' AND HasError = 1 +) AS errorCount_tiers FORMAT JSON -- spec:attribute-keys-logs:baseline [f2e10453] @@ -12293,330 +12789,733 @@ SELECT LIMIT 10 FORMAT JSON --- spec:traces-facets-single-dimension:baseline [a277cb3c] +-- spec:traces-facets-single-dimension:baseline [e072f533] SELECT - SpanName AS name, - count() AS count, + name AS name, + sum(count) AS count, 'spanName' AS facetType + FROM ( +SELECT + SpanName AS name, + count() AS count FROM trace_list_mv WHERE OrgId = 'org_sql_catalog' AND Timestamp >= '2026-01-01 10:30:00' AND Timestamp <= '2026-01-03 14:15:00' AND ServiceName = 'api' AND DeploymentEnv = 'production' + AND (Timestamp < if(toDateTime('2026-01-01 10:30:00') = toStartOfHour(toDateTime('2026-01-01 10:30:00')), toStartOfHour(toDateTime('2026-01-01 10:30:00')), toStartOfHour(toDateTime('2026-01-01 10:30:00')) + INTERVAL 1 HOUR) OR Timestamp >= toStartOfHour(toDateTime('2026-01-03 14:15:00'))) AND SpanName != '' GROUP BY name +UNION ALL +SELECT + SpanName AS name, + sum(TraceCount) AS count + FROM trace_facets_hourly + WHERE OrgId = 'org_sql_catalog' + AND Hour >= if(toDateTime('2026-01-01 10:30:00') = toStartOfHour(toDateTime('2026-01-01 10:30:00')), toStartOfHour(toDateTime('2026-01-01 10:30:00')), toStartOfHour(toDateTime('2026-01-01 10:30:00')) + INTERVAL 1 HOUR) + AND Hour < toStartOfHour(toDateTime('2026-01-03 14:15:00')) + AND ServiceName = 'api' + AND DeploymentEnv = 'production' + AND SpanName != '' + GROUP BY name +) AS spanName_tiers + GROUP BY name ORDER BY count DESC LIMIT 20 FORMAT JSON --- spec:traces-facets:baseline [eabfe99e] +-- spec:traces-facets:baseline [eb366b9e] SELECT - ServiceName AS name, - count() AS count, + name AS name, + sum(count) AS count, 'service' AS facetType + FROM ( +SELECT + ServiceName AS name, + count() AS count FROM trace_list_mv WHERE OrgId = 'org_sql_catalog' AND Timestamp >= '2026-01-01 10:30:00' AND Timestamp <= '2026-01-03 14:15:00' AND ServiceName = 'api' AND DeploymentEnv = 'production' + AND (Timestamp < if(toDateTime('2026-01-01 10:30:00') = toStartOfHour(toDateTime('2026-01-01 10:30:00')), toStartOfHour(toDateTime('2026-01-01 10:30:00')), toStartOfHour(toDateTime('2026-01-01 10:30:00')) + INTERVAL 1 HOUR) OR Timestamp >= toStartOfHour(toDateTime('2026-01-03 14:15:00'))) + GROUP BY name +UNION ALL +SELECT + ServiceName AS name, + sum(TraceCount) AS count + FROM trace_facets_hourly + WHERE OrgId = 'org_sql_catalog' + AND Hour >= if(toDateTime('2026-01-01 10:30:00') = toStartOfHour(toDateTime('2026-01-01 10:30:00')), toStartOfHour(toDateTime('2026-01-01 10:30:00')), toStartOfHour(toDateTime('2026-01-01 10:30:00')) + INTERVAL 1 HOUR) + AND Hour < toStartOfHour(toDateTime('2026-01-03 14:15:00')) + AND ServiceName = 'api' + AND DeploymentEnv = 'production' + GROUP BY name +) AS service_tiers GROUP BY name ORDER BY count DESC LIMIT 50 UNION ALL SELECT - SpanName AS name, - count() AS count, + name AS name, + sum(count) AS count, 'spanName' AS facetType + FROM ( +SELECT + SpanName AS name, + count() AS count FROM trace_list_mv WHERE OrgId = 'org_sql_catalog' AND Timestamp >= '2026-01-01 10:30:00' AND Timestamp <= '2026-01-03 14:15:00' AND ServiceName = 'api' AND DeploymentEnv = 'production' + AND (Timestamp < if(toDateTime('2026-01-01 10:30:00') = toStartOfHour(toDateTime('2026-01-01 10:30:00')), toStartOfHour(toDateTime('2026-01-01 10:30:00')), toStartOfHour(toDateTime('2026-01-01 10:30:00')) + INTERVAL 1 HOUR) OR Timestamp >= toStartOfHour(toDateTime('2026-01-03 14:15:00'))) + AND SpanName != '' + GROUP BY name +UNION ALL +SELECT + SpanName AS name, + sum(TraceCount) AS count + FROM trace_facets_hourly + WHERE OrgId = 'org_sql_catalog' + AND Hour >= if(toDateTime('2026-01-01 10:30:00') = toStartOfHour(toDateTime('2026-01-01 10:30:00')), toStartOfHour(toDateTime('2026-01-01 10:30:00')), toStartOfHour(toDateTime('2026-01-01 10:30:00')) + INTERVAL 1 HOUR) + AND Hour < toStartOfHour(toDateTime('2026-01-03 14:15:00')) + AND ServiceName = 'api' + AND DeploymentEnv = 'production' AND SpanName != '' GROUP BY name +) AS spanName_tiers + GROUP BY name ORDER BY count DESC LIMIT 20 UNION ALL SELECT - HttpMethod AS name, - count() AS count, + name AS name, + sum(count) AS count, 'httpMethod' AS facetType + FROM ( +SELECT + HttpMethod AS name, + count() AS count FROM trace_list_mv WHERE OrgId = 'org_sql_catalog' AND Timestamp >= '2026-01-01 10:30:00' AND Timestamp <= '2026-01-03 14:15:00' AND ServiceName = 'api' AND DeploymentEnv = 'production' + AND (Timestamp < if(toDateTime('2026-01-01 10:30:00') = toStartOfHour(toDateTime('2026-01-01 10:30:00')), toStartOfHour(toDateTime('2026-01-01 10:30:00')), toStartOfHour(toDateTime('2026-01-01 10:30:00')) + INTERVAL 1 HOUR) OR Timestamp >= toStartOfHour(toDateTime('2026-01-03 14:15:00'))) + AND HttpMethod != '' + GROUP BY name +UNION ALL +SELECT + HttpMethod AS name, + sum(TraceCount) AS count + FROM trace_facets_hourly + WHERE OrgId = 'org_sql_catalog' + AND Hour >= if(toDateTime('2026-01-01 10:30:00') = toStartOfHour(toDateTime('2026-01-01 10:30:00')), toStartOfHour(toDateTime('2026-01-01 10:30:00')), toStartOfHour(toDateTime('2026-01-01 10:30:00')) + INTERVAL 1 HOUR) + AND Hour < toStartOfHour(toDateTime('2026-01-03 14:15:00')) + AND ServiceName = 'api' + AND DeploymentEnv = 'production' AND HttpMethod != '' GROUP BY name +) AS httpMethod_tiers + GROUP BY name ORDER BY count DESC LIMIT 20 UNION ALL SELECT - HttpStatusCode AS name, - count() AS count, + name AS name, + sum(count) AS count, 'httpStatus' AS facetType + FROM ( +SELECT + HttpStatusCode AS name, + count() AS count FROM trace_list_mv WHERE OrgId = 'org_sql_catalog' AND Timestamp >= '2026-01-01 10:30:00' AND Timestamp <= '2026-01-03 14:15:00' AND ServiceName = 'api' AND DeploymentEnv = 'production' + AND (Timestamp < if(toDateTime('2026-01-01 10:30:00') = toStartOfHour(toDateTime('2026-01-01 10:30:00')), toStartOfHour(toDateTime('2026-01-01 10:30:00')), toStartOfHour(toDateTime('2026-01-01 10:30:00')) + INTERVAL 1 HOUR) OR Timestamp >= toStartOfHour(toDateTime('2026-01-03 14:15:00'))) AND HttpStatusCode != '' GROUP BY name +UNION ALL +SELECT + HttpStatusCode AS name, + sum(TraceCount) AS count + FROM trace_facets_hourly + WHERE OrgId = 'org_sql_catalog' + AND Hour >= if(toDateTime('2026-01-01 10:30:00') = toStartOfHour(toDateTime('2026-01-01 10:30:00')), toStartOfHour(toDateTime('2026-01-01 10:30:00')), toStartOfHour(toDateTime('2026-01-01 10:30:00')) + INTERVAL 1 HOUR) + AND Hour < toStartOfHour(toDateTime('2026-01-03 14:15:00')) + AND ServiceName = 'api' + AND DeploymentEnv = 'production' + AND HttpStatusCode != '' + GROUP BY name +) AS httpStatus_tiers + GROUP BY name ORDER BY count DESC LIMIT 20 UNION ALL SELECT - DeploymentEnv AS name, - count() AS count, + name AS name, + sum(count) AS count, 'deploymentEnv' AS facetType + FROM ( +SELECT + DeploymentEnv AS name, + count() AS count FROM trace_list_mv WHERE OrgId = 'org_sql_catalog' AND Timestamp >= '2026-01-01 10:30:00' AND Timestamp <= '2026-01-03 14:15:00' AND ServiceName = 'api' AND DeploymentEnv = 'production' + AND (Timestamp < if(toDateTime('2026-01-01 10:30:00') = toStartOfHour(toDateTime('2026-01-01 10:30:00')), toStartOfHour(toDateTime('2026-01-01 10:30:00')), toStartOfHour(toDateTime('2026-01-01 10:30:00')) + INTERVAL 1 HOUR) OR Timestamp >= toStartOfHour(toDateTime('2026-01-03 14:15:00'))) + AND DeploymentEnv != '' + GROUP BY name +UNION ALL +SELECT + DeploymentEnv AS name, + sum(TraceCount) AS count + FROM trace_facets_hourly + WHERE OrgId = 'org_sql_catalog' + AND Hour >= if(toDateTime('2026-01-01 10:30:00') = toStartOfHour(toDateTime('2026-01-01 10:30:00')), toStartOfHour(toDateTime('2026-01-01 10:30:00')), toStartOfHour(toDateTime('2026-01-01 10:30:00')) + INTERVAL 1 HOUR) + AND Hour < toStartOfHour(toDateTime('2026-01-03 14:15:00')) + AND ServiceName = 'api' + AND DeploymentEnv = 'production' AND DeploymentEnv != '' GROUP BY name +) AS deploymentEnv_tiers + GROUP BY name ORDER BY count DESC LIMIT 20 UNION ALL SELECT - ServiceNamespace AS name, - count() AS count, + name AS name, + sum(count) AS count, 'serviceNamespace' AS facetType + FROM ( +SELECT + ServiceNamespace AS name, + count() AS count FROM trace_list_mv WHERE OrgId = 'org_sql_catalog' AND Timestamp >= '2026-01-01 10:30:00' AND Timestamp <= '2026-01-03 14:15:00' AND ServiceName = 'api' AND DeploymentEnv = 'production' + AND (Timestamp < if(toDateTime('2026-01-01 10:30:00') = toStartOfHour(toDateTime('2026-01-01 10:30:00')), toStartOfHour(toDateTime('2026-01-01 10:30:00')), toStartOfHour(toDateTime('2026-01-01 10:30:00')) + INTERVAL 1 HOUR) OR Timestamp >= toStartOfHour(toDateTime('2026-01-03 14:15:00'))) AND ServiceNamespace != '' GROUP BY name +UNION ALL +SELECT + ServiceNamespace AS name, + sum(TraceCount) AS count + FROM trace_facets_hourly + WHERE OrgId = 'org_sql_catalog' + AND Hour >= if(toDateTime('2026-01-01 10:30:00') = toStartOfHour(toDateTime('2026-01-01 10:30:00')), toStartOfHour(toDateTime('2026-01-01 10:30:00')), toStartOfHour(toDateTime('2026-01-01 10:30:00')) + INTERVAL 1 HOUR) + AND Hour < toStartOfHour(toDateTime('2026-01-03 14:15:00')) + AND ServiceName = 'api' + AND DeploymentEnv = 'production' + AND ServiceNamespace != '' + GROUP BY name +) AS serviceNamespace_tiers + GROUP BY name ORDER BY count DESC LIMIT 20 UNION ALL SELECT 'error' AS name, - count() AS count, + sum(count) AS count, 'errorCount' AS facetType + FROM ( +SELECT + count() AS count FROM trace_list_mv WHERE OrgId = 'org_sql_catalog' AND Timestamp >= '2026-01-01 10:30:00' AND Timestamp <= '2026-01-03 14:15:00' AND ServiceName = 'api' AND DeploymentEnv = 'production' + AND (Timestamp < if(toDateTime('2026-01-01 10:30:00') = toStartOfHour(toDateTime('2026-01-01 10:30:00')), toStartOfHour(toDateTime('2026-01-01 10:30:00')), toStartOfHour(toDateTime('2026-01-01 10:30:00')) + INTERVAL 1 HOUR) OR Timestamp >= toStartOfHour(toDateTime('2026-01-03 14:15:00'))) AND HasError = 1 +UNION ALL +SELECT + sum(TraceCount) AS count + FROM trace_facets_hourly + WHERE OrgId = 'org_sql_catalog' + AND Hour >= if(toDateTime('2026-01-01 10:30:00') = toStartOfHour(toDateTime('2026-01-01 10:30:00')), toStartOfHour(toDateTime('2026-01-01 10:30:00')), toStartOfHour(toDateTime('2026-01-01 10:30:00')) + INTERVAL 1 HOUR) + AND Hour < toStartOfHour(toDateTime('2026-01-03 14:15:00')) + AND ServiceName = 'api' + AND DeploymentEnv = 'production' + AND HasError = 1 +) AS errorCount_tiers FORMAT JSON --- spec:traces-facets:bloom [eabfe99e] +-- spec:traces-facets:bloom [eb366b9e] SELECT - ServiceName AS name, - count() AS count, + name AS name, + sum(count) AS count, 'service' AS facetType + FROM ( +SELECT + ServiceName AS name, + count() AS count FROM trace_list_mv WHERE OrgId = 'org_sql_catalog' AND Timestamp >= '2026-01-01 10:30:00' AND Timestamp <= '2026-01-03 14:15:00' AND ServiceName = 'api' AND DeploymentEnv = 'production' + AND (Timestamp < if(toDateTime('2026-01-01 10:30:00') = toStartOfHour(toDateTime('2026-01-01 10:30:00')), toStartOfHour(toDateTime('2026-01-01 10:30:00')), toStartOfHour(toDateTime('2026-01-01 10:30:00')) + INTERVAL 1 HOUR) OR Timestamp >= toStartOfHour(toDateTime('2026-01-03 14:15:00'))) + GROUP BY name +UNION ALL +SELECT + ServiceName AS name, + sum(TraceCount) AS count + FROM trace_facets_hourly + WHERE OrgId = 'org_sql_catalog' + AND Hour >= if(toDateTime('2026-01-01 10:30:00') = toStartOfHour(toDateTime('2026-01-01 10:30:00')), toStartOfHour(toDateTime('2026-01-01 10:30:00')), toStartOfHour(toDateTime('2026-01-01 10:30:00')) + INTERVAL 1 HOUR) + AND Hour < toStartOfHour(toDateTime('2026-01-03 14:15:00')) + AND ServiceName = 'api' + AND DeploymentEnv = 'production' + GROUP BY name +) AS service_tiers GROUP BY name ORDER BY count DESC LIMIT 50 UNION ALL SELECT - SpanName AS name, - count() AS count, + name AS name, + sum(count) AS count, 'spanName' AS facetType + FROM ( +SELECT + SpanName AS name, + count() AS count FROM trace_list_mv WHERE OrgId = 'org_sql_catalog' AND Timestamp >= '2026-01-01 10:30:00' AND Timestamp <= '2026-01-03 14:15:00' AND ServiceName = 'api' AND DeploymentEnv = 'production' + AND (Timestamp < if(toDateTime('2026-01-01 10:30:00') = toStartOfHour(toDateTime('2026-01-01 10:30:00')), toStartOfHour(toDateTime('2026-01-01 10:30:00')), toStartOfHour(toDateTime('2026-01-01 10:30:00')) + INTERVAL 1 HOUR) OR Timestamp >= toStartOfHour(toDateTime('2026-01-03 14:15:00'))) + AND SpanName != '' + GROUP BY name +UNION ALL +SELECT + SpanName AS name, + sum(TraceCount) AS count + FROM trace_facets_hourly + WHERE OrgId = 'org_sql_catalog' + AND Hour >= if(toDateTime('2026-01-01 10:30:00') = toStartOfHour(toDateTime('2026-01-01 10:30:00')), toStartOfHour(toDateTime('2026-01-01 10:30:00')), toStartOfHour(toDateTime('2026-01-01 10:30:00')) + INTERVAL 1 HOUR) + AND Hour < toStartOfHour(toDateTime('2026-01-03 14:15:00')) + AND ServiceName = 'api' + AND DeploymentEnv = 'production' AND SpanName != '' GROUP BY name +) AS spanName_tiers + GROUP BY name ORDER BY count DESC LIMIT 20 UNION ALL SELECT - HttpMethod AS name, - count() AS count, + name AS name, + sum(count) AS count, 'httpMethod' AS facetType + FROM ( +SELECT + HttpMethod AS name, + count() AS count FROM trace_list_mv WHERE OrgId = 'org_sql_catalog' AND Timestamp >= '2026-01-01 10:30:00' AND Timestamp <= '2026-01-03 14:15:00' AND ServiceName = 'api' AND DeploymentEnv = 'production' + AND (Timestamp < if(toDateTime('2026-01-01 10:30:00') = toStartOfHour(toDateTime('2026-01-01 10:30:00')), toStartOfHour(toDateTime('2026-01-01 10:30:00')), toStartOfHour(toDateTime('2026-01-01 10:30:00')) + INTERVAL 1 HOUR) OR Timestamp >= toStartOfHour(toDateTime('2026-01-03 14:15:00'))) + AND HttpMethod != '' + GROUP BY name +UNION ALL +SELECT + HttpMethod AS name, + sum(TraceCount) AS count + FROM trace_facets_hourly + WHERE OrgId = 'org_sql_catalog' + AND Hour >= if(toDateTime('2026-01-01 10:30:00') = toStartOfHour(toDateTime('2026-01-01 10:30:00')), toStartOfHour(toDateTime('2026-01-01 10:30:00')), toStartOfHour(toDateTime('2026-01-01 10:30:00')) + INTERVAL 1 HOUR) + AND Hour < toStartOfHour(toDateTime('2026-01-03 14:15:00')) + AND ServiceName = 'api' + AND DeploymentEnv = 'production' AND HttpMethod != '' GROUP BY name +) AS httpMethod_tiers + GROUP BY name ORDER BY count DESC LIMIT 20 UNION ALL SELECT - HttpStatusCode AS name, - count() AS count, + name AS name, + sum(count) AS count, 'httpStatus' AS facetType + FROM ( +SELECT + HttpStatusCode AS name, + count() AS count FROM trace_list_mv WHERE OrgId = 'org_sql_catalog' AND Timestamp >= '2026-01-01 10:30:00' AND Timestamp <= '2026-01-03 14:15:00' AND ServiceName = 'api' AND DeploymentEnv = 'production' + AND (Timestamp < if(toDateTime('2026-01-01 10:30:00') = toStartOfHour(toDateTime('2026-01-01 10:30:00')), toStartOfHour(toDateTime('2026-01-01 10:30:00')), toStartOfHour(toDateTime('2026-01-01 10:30:00')) + INTERVAL 1 HOUR) OR Timestamp >= toStartOfHour(toDateTime('2026-01-03 14:15:00'))) AND HttpStatusCode != '' GROUP BY name +UNION ALL +SELECT + HttpStatusCode AS name, + sum(TraceCount) AS count + FROM trace_facets_hourly + WHERE OrgId = 'org_sql_catalog' + AND Hour >= if(toDateTime('2026-01-01 10:30:00') = toStartOfHour(toDateTime('2026-01-01 10:30:00')), toStartOfHour(toDateTime('2026-01-01 10:30:00')), toStartOfHour(toDateTime('2026-01-01 10:30:00')) + INTERVAL 1 HOUR) + AND Hour < toStartOfHour(toDateTime('2026-01-03 14:15:00')) + AND ServiceName = 'api' + AND DeploymentEnv = 'production' + AND HttpStatusCode != '' + GROUP BY name +) AS httpStatus_tiers + GROUP BY name ORDER BY count DESC LIMIT 20 UNION ALL SELECT - DeploymentEnv AS name, - count() AS count, + name AS name, + sum(count) AS count, 'deploymentEnv' AS facetType + FROM ( +SELECT + DeploymentEnv AS name, + count() AS count FROM trace_list_mv WHERE OrgId = 'org_sql_catalog' AND Timestamp >= '2026-01-01 10:30:00' AND Timestamp <= '2026-01-03 14:15:00' AND ServiceName = 'api' AND DeploymentEnv = 'production' + AND (Timestamp < if(toDateTime('2026-01-01 10:30:00') = toStartOfHour(toDateTime('2026-01-01 10:30:00')), toStartOfHour(toDateTime('2026-01-01 10:30:00')), toStartOfHour(toDateTime('2026-01-01 10:30:00')) + INTERVAL 1 HOUR) OR Timestamp >= toStartOfHour(toDateTime('2026-01-03 14:15:00'))) + AND DeploymentEnv != '' + GROUP BY name +UNION ALL +SELECT + DeploymentEnv AS name, + sum(TraceCount) AS count + FROM trace_facets_hourly + WHERE OrgId = 'org_sql_catalog' + AND Hour >= if(toDateTime('2026-01-01 10:30:00') = toStartOfHour(toDateTime('2026-01-01 10:30:00')), toStartOfHour(toDateTime('2026-01-01 10:30:00')), toStartOfHour(toDateTime('2026-01-01 10:30:00')) + INTERVAL 1 HOUR) + AND Hour < toStartOfHour(toDateTime('2026-01-03 14:15:00')) + AND ServiceName = 'api' + AND DeploymentEnv = 'production' AND DeploymentEnv != '' GROUP BY name +) AS deploymentEnv_tiers + GROUP BY name ORDER BY count DESC LIMIT 20 UNION ALL SELECT - ServiceNamespace AS name, - count() AS count, + name AS name, + sum(count) AS count, 'serviceNamespace' AS facetType + FROM ( +SELECT + ServiceNamespace AS name, + count() AS count FROM trace_list_mv WHERE OrgId = 'org_sql_catalog' AND Timestamp >= '2026-01-01 10:30:00' AND Timestamp <= '2026-01-03 14:15:00' AND ServiceName = 'api' AND DeploymentEnv = 'production' + AND (Timestamp < if(toDateTime('2026-01-01 10:30:00') = toStartOfHour(toDateTime('2026-01-01 10:30:00')), toStartOfHour(toDateTime('2026-01-01 10:30:00')), toStartOfHour(toDateTime('2026-01-01 10:30:00')) + INTERVAL 1 HOUR) OR Timestamp >= toStartOfHour(toDateTime('2026-01-03 14:15:00'))) AND ServiceNamespace != '' GROUP BY name +UNION ALL +SELECT + ServiceNamespace AS name, + sum(TraceCount) AS count + FROM trace_facets_hourly + WHERE OrgId = 'org_sql_catalog' + AND Hour >= if(toDateTime('2026-01-01 10:30:00') = toStartOfHour(toDateTime('2026-01-01 10:30:00')), toStartOfHour(toDateTime('2026-01-01 10:30:00')), toStartOfHour(toDateTime('2026-01-01 10:30:00')) + INTERVAL 1 HOUR) + AND Hour < toStartOfHour(toDateTime('2026-01-03 14:15:00')) + AND ServiceName = 'api' + AND DeploymentEnv = 'production' + AND ServiceNamespace != '' + GROUP BY name +) AS serviceNamespace_tiers + GROUP BY name ORDER BY count DESC LIMIT 20 UNION ALL SELECT 'error' AS name, - count() AS count, + sum(count) AS count, 'errorCount' AS facetType + FROM ( +SELECT + count() AS count FROM trace_list_mv WHERE OrgId = 'org_sql_catalog' AND Timestamp >= '2026-01-01 10:30:00' AND Timestamp <= '2026-01-03 14:15:00' AND ServiceName = 'api' AND DeploymentEnv = 'production' + AND (Timestamp < if(toDateTime('2026-01-01 10:30:00') = toStartOfHour(toDateTime('2026-01-01 10:30:00')), toStartOfHour(toDateTime('2026-01-01 10:30:00')), toStartOfHour(toDateTime('2026-01-01 10:30:00')) + INTERVAL 1 HOUR) OR Timestamp >= toStartOfHour(toDateTime('2026-01-03 14:15:00'))) + AND HasError = 1 +UNION ALL +SELECT + sum(TraceCount) AS count + FROM trace_facets_hourly + WHERE OrgId = 'org_sql_catalog' + AND Hour >= if(toDateTime('2026-01-01 10:30:00') = toStartOfHour(toDateTime('2026-01-01 10:30:00')), toStartOfHour(toDateTime('2026-01-01 10:30:00')), toStartOfHour(toDateTime('2026-01-01 10:30:00')) + INTERVAL 1 HOUR) + AND Hour < toStartOfHour(toDateTime('2026-01-03 14:15:00')) + AND ServiceName = 'api' + AND DeploymentEnv = 'production' AND HasError = 1 +) AS errorCount_tiers FORMAT JSON --- spec:traces-facets:text [eabfe99e] +-- spec:traces-facets:text [eb366b9e] SELECT - ServiceName AS name, - count() AS count, + name AS name, + sum(count) AS count, 'service' AS facetType + FROM ( +SELECT + ServiceName AS name, + count() AS count FROM trace_list_mv WHERE OrgId = 'org_sql_catalog' AND Timestamp >= '2026-01-01 10:30:00' AND Timestamp <= '2026-01-03 14:15:00' AND ServiceName = 'api' AND DeploymentEnv = 'production' + AND (Timestamp < if(toDateTime('2026-01-01 10:30:00') = toStartOfHour(toDateTime('2026-01-01 10:30:00')), toStartOfHour(toDateTime('2026-01-01 10:30:00')), toStartOfHour(toDateTime('2026-01-01 10:30:00')) + INTERVAL 1 HOUR) OR Timestamp >= toStartOfHour(toDateTime('2026-01-03 14:15:00'))) + GROUP BY name +UNION ALL +SELECT + ServiceName AS name, + sum(TraceCount) AS count + FROM trace_facets_hourly + WHERE OrgId = 'org_sql_catalog' + AND Hour >= if(toDateTime('2026-01-01 10:30:00') = toStartOfHour(toDateTime('2026-01-01 10:30:00')), toStartOfHour(toDateTime('2026-01-01 10:30:00')), toStartOfHour(toDateTime('2026-01-01 10:30:00')) + INTERVAL 1 HOUR) + AND Hour < toStartOfHour(toDateTime('2026-01-03 14:15:00')) + AND ServiceName = 'api' + AND DeploymentEnv = 'production' + GROUP BY name +) AS service_tiers GROUP BY name ORDER BY count DESC LIMIT 50 UNION ALL SELECT - SpanName AS name, - count() AS count, + name AS name, + sum(count) AS count, 'spanName' AS facetType + FROM ( +SELECT + SpanName AS name, + count() AS count FROM trace_list_mv WHERE OrgId = 'org_sql_catalog' AND Timestamp >= '2026-01-01 10:30:00' AND Timestamp <= '2026-01-03 14:15:00' AND ServiceName = 'api' AND DeploymentEnv = 'production' + AND (Timestamp < if(toDateTime('2026-01-01 10:30:00') = toStartOfHour(toDateTime('2026-01-01 10:30:00')), toStartOfHour(toDateTime('2026-01-01 10:30:00')), toStartOfHour(toDateTime('2026-01-01 10:30:00')) + INTERVAL 1 HOUR) OR Timestamp >= toStartOfHour(toDateTime('2026-01-03 14:15:00'))) AND SpanName != '' GROUP BY name +UNION ALL +SELECT + SpanName AS name, + sum(TraceCount) AS count + FROM trace_facets_hourly + WHERE OrgId = 'org_sql_catalog' + AND Hour >= if(toDateTime('2026-01-01 10:30:00') = toStartOfHour(toDateTime('2026-01-01 10:30:00')), toStartOfHour(toDateTime('2026-01-01 10:30:00')), toStartOfHour(toDateTime('2026-01-01 10:30:00')) + INTERVAL 1 HOUR) + AND Hour < toStartOfHour(toDateTime('2026-01-03 14:15:00')) + AND ServiceName = 'api' + AND DeploymentEnv = 'production' + AND SpanName != '' + GROUP BY name +) AS spanName_tiers + GROUP BY name ORDER BY count DESC LIMIT 20 UNION ALL SELECT - HttpMethod AS name, - count() AS count, + name AS name, + sum(count) AS count, 'httpMethod' AS facetType + FROM ( +SELECT + HttpMethod AS name, + count() AS count FROM trace_list_mv WHERE OrgId = 'org_sql_catalog' AND Timestamp >= '2026-01-01 10:30:00' AND Timestamp <= '2026-01-03 14:15:00' AND ServiceName = 'api' AND DeploymentEnv = 'production' + AND (Timestamp < if(toDateTime('2026-01-01 10:30:00') = toStartOfHour(toDateTime('2026-01-01 10:30:00')), toStartOfHour(toDateTime('2026-01-01 10:30:00')), toStartOfHour(toDateTime('2026-01-01 10:30:00')) + INTERVAL 1 HOUR) OR Timestamp >= toStartOfHour(toDateTime('2026-01-03 14:15:00'))) + AND HttpMethod != '' + GROUP BY name +UNION ALL +SELECT + HttpMethod AS name, + sum(TraceCount) AS count + FROM trace_facets_hourly + WHERE OrgId = 'org_sql_catalog' + AND Hour >= if(toDateTime('2026-01-01 10:30:00') = toStartOfHour(toDateTime('2026-01-01 10:30:00')), toStartOfHour(toDateTime('2026-01-01 10:30:00')), toStartOfHour(toDateTime('2026-01-01 10:30:00')) + INTERVAL 1 HOUR) + AND Hour < toStartOfHour(toDateTime('2026-01-03 14:15:00')) + AND ServiceName = 'api' + AND DeploymentEnv = 'production' AND HttpMethod != '' GROUP BY name +) AS httpMethod_tiers + GROUP BY name ORDER BY count DESC LIMIT 20 UNION ALL SELECT - HttpStatusCode AS name, - count() AS count, + name AS name, + sum(count) AS count, 'httpStatus' AS facetType + FROM ( +SELECT + HttpStatusCode AS name, + count() AS count FROM trace_list_mv WHERE OrgId = 'org_sql_catalog' AND Timestamp >= '2026-01-01 10:30:00' AND Timestamp <= '2026-01-03 14:15:00' AND ServiceName = 'api' AND DeploymentEnv = 'production' + AND (Timestamp < if(toDateTime('2026-01-01 10:30:00') = toStartOfHour(toDateTime('2026-01-01 10:30:00')), toStartOfHour(toDateTime('2026-01-01 10:30:00')), toStartOfHour(toDateTime('2026-01-01 10:30:00')) + INTERVAL 1 HOUR) OR Timestamp >= toStartOfHour(toDateTime('2026-01-03 14:15:00'))) + AND HttpStatusCode != '' + GROUP BY name +UNION ALL +SELECT + HttpStatusCode AS name, + sum(TraceCount) AS count + FROM trace_facets_hourly + WHERE OrgId = 'org_sql_catalog' + AND Hour >= if(toDateTime('2026-01-01 10:30:00') = toStartOfHour(toDateTime('2026-01-01 10:30:00')), toStartOfHour(toDateTime('2026-01-01 10:30:00')), toStartOfHour(toDateTime('2026-01-01 10:30:00')) + INTERVAL 1 HOUR) + AND Hour < toStartOfHour(toDateTime('2026-01-03 14:15:00')) + AND ServiceName = 'api' + AND DeploymentEnv = 'production' AND HttpStatusCode != '' GROUP BY name +) AS httpStatus_tiers + GROUP BY name ORDER BY count DESC LIMIT 20 UNION ALL SELECT - DeploymentEnv AS name, - count() AS count, + name AS name, + sum(count) AS count, 'deploymentEnv' AS facetType + FROM ( +SELECT + DeploymentEnv AS name, + count() AS count FROM trace_list_mv WHERE OrgId = 'org_sql_catalog' AND Timestamp >= '2026-01-01 10:30:00' AND Timestamp <= '2026-01-03 14:15:00' AND ServiceName = 'api' AND DeploymentEnv = 'production' + AND (Timestamp < if(toDateTime('2026-01-01 10:30:00') = toStartOfHour(toDateTime('2026-01-01 10:30:00')), toStartOfHour(toDateTime('2026-01-01 10:30:00')), toStartOfHour(toDateTime('2026-01-01 10:30:00')) + INTERVAL 1 HOUR) OR Timestamp >= toStartOfHour(toDateTime('2026-01-03 14:15:00'))) + AND DeploymentEnv != '' + GROUP BY name +UNION ALL +SELECT + DeploymentEnv AS name, + sum(TraceCount) AS count + FROM trace_facets_hourly + WHERE OrgId = 'org_sql_catalog' + AND Hour >= if(toDateTime('2026-01-01 10:30:00') = toStartOfHour(toDateTime('2026-01-01 10:30:00')), toStartOfHour(toDateTime('2026-01-01 10:30:00')), toStartOfHour(toDateTime('2026-01-01 10:30:00')) + INTERVAL 1 HOUR) + AND Hour < toStartOfHour(toDateTime('2026-01-03 14:15:00')) + AND ServiceName = 'api' + AND DeploymentEnv = 'production' AND DeploymentEnv != '' GROUP BY name +) AS deploymentEnv_tiers + GROUP BY name ORDER BY count DESC LIMIT 20 UNION ALL SELECT - ServiceNamespace AS name, - count() AS count, + name AS name, + sum(count) AS count, 'serviceNamespace' AS facetType + FROM ( +SELECT + ServiceNamespace AS name, + count() AS count FROM trace_list_mv WHERE OrgId = 'org_sql_catalog' AND Timestamp >= '2026-01-01 10:30:00' AND Timestamp <= '2026-01-03 14:15:00' AND ServiceName = 'api' AND DeploymentEnv = 'production' + AND (Timestamp < if(toDateTime('2026-01-01 10:30:00') = toStartOfHour(toDateTime('2026-01-01 10:30:00')), toStartOfHour(toDateTime('2026-01-01 10:30:00')), toStartOfHour(toDateTime('2026-01-01 10:30:00')) + INTERVAL 1 HOUR) OR Timestamp >= toStartOfHour(toDateTime('2026-01-03 14:15:00'))) + AND ServiceNamespace != '' + GROUP BY name +UNION ALL +SELECT + ServiceNamespace AS name, + sum(TraceCount) AS count + FROM trace_facets_hourly + WHERE OrgId = 'org_sql_catalog' + AND Hour >= if(toDateTime('2026-01-01 10:30:00') = toStartOfHour(toDateTime('2026-01-01 10:30:00')), toStartOfHour(toDateTime('2026-01-01 10:30:00')), toStartOfHour(toDateTime('2026-01-01 10:30:00')) + INTERVAL 1 HOUR) + AND Hour < toStartOfHour(toDateTime('2026-01-03 14:15:00')) + AND ServiceName = 'api' + AND DeploymentEnv = 'production' AND ServiceNamespace != '' GROUP BY name +) AS serviceNamespace_tiers + GROUP BY name ORDER BY count DESC LIMIT 20 UNION ALL SELECT 'error' AS name, - count() AS count, + sum(count) AS count, 'errorCount' AS facetType + FROM ( +SELECT + count() AS count FROM trace_list_mv WHERE OrgId = 'org_sql_catalog' AND Timestamp >= '2026-01-01 10:30:00' AND Timestamp <= '2026-01-03 14:15:00' AND ServiceName = 'api' AND DeploymentEnv = 'production' + AND (Timestamp < if(toDateTime('2026-01-01 10:30:00') = toStartOfHour(toDateTime('2026-01-01 10:30:00')), toStartOfHour(toDateTime('2026-01-01 10:30:00')), toStartOfHour(toDateTime('2026-01-01 10:30:00')) + INTERVAL 1 HOUR) OR Timestamp >= toStartOfHour(toDateTime('2026-01-03 14:15:00'))) + AND HasError = 1 +UNION ALL +SELECT + sum(TraceCount) AS count + FROM trace_facets_hourly + WHERE OrgId = 'org_sql_catalog' + AND Hour >= if(toDateTime('2026-01-01 10:30:00') = toStartOfHour(toDateTime('2026-01-01 10:30:00')), toStartOfHour(toDateTime('2026-01-01 10:30:00')), toStartOfHour(toDateTime('2026-01-01 10:30:00')) + INTERVAL 1 HOUR) + AND Hour < toStartOfHour(toDateTime('2026-01-03 14:15:00')) + AND ServiceName = 'api' + AND DeploymentEnv = 'production' AND HasError = 1 +) AS errorCount_tiers FORMAT JSON -- spec:traces-list-grouped-attr-fallback:baseline [6b9a4329] @@ -13001,18 +13900,38 @@ SELECT LIMIT 50 FORMAT JSON --- spec:traces-stats:baseline [333e79e5] +-- spec:traces-stats:baseline [f262a535] SELECT - min(Duration) / 1000000 AS minDurationMs, - max(Duration) / 1000000 AS maxDurationMs, - ifNull(ifNotFinite(quantile(0.5)(Duration) / 1000000, 0), 0) AS p50DurationMs, - ifNull(ifNotFinite(quantile(0.95)(Duration) / 1000000, 0), 0) AS p95DurationMs + minIf(durationMin, traceCount > 0) / 1000000 AS minDurationMs, + maxIf(durationMax, traceCount > 0) / 1000000 AS maxDurationMs, + ifNull(ifNotFinite(arrayElement(quantilesTDigestMerge(0.5, 0.95)(durationQuantiles), 1) / 1000000, 0), 0) AS p50DurationMs, + ifNull(ifNotFinite(arrayElement(quantilesTDigestMerge(0.5, 0.95)(durationQuantiles), 2) / 1000000, 0), 0) AS p95DurationMs + FROM ( +SELECT + count() AS traceCount, + min(Duration) AS durationMin, + max(Duration) AS durationMax, + quantilesTDigestState(0.5, 0.95)(Duration) AS durationQuantiles FROM trace_list_mv WHERE OrgId = 'org_sql_catalog' AND Timestamp >= '2026-01-01 10:30:00' AND Timestamp <= '2026-01-03 14:15:00' AND ServiceName = 'api' AND DeploymentEnv = 'production' + AND (Timestamp < if(toDateTime('2026-01-01 10:30:00') = toStartOfHour(toDateTime('2026-01-01 10:30:00')), toStartOfHour(toDateTime('2026-01-01 10:30:00')), toStartOfHour(toDateTime('2026-01-01 10:30:00')) + INTERVAL 1 HOUR) OR Timestamp >= toStartOfHour(toDateTime('2026-01-03 14:15:00'))) +UNION ALL +SELECT + sum(TraceCount) AS traceCount, + min(DurationMin) AS durationMin, + max(DurationMax) AS durationMax, + quantilesTDigestMergeState(0.5, 0.95)(DurationQuantiles) AS durationQuantiles + FROM trace_facets_hourly + WHERE OrgId = 'org_sql_catalog' + AND Hour >= if(toDateTime('2026-01-01 10:30:00') = toStartOfHour(toDateTime('2026-01-01 10:30:00')), toStartOfHour(toDateTime('2026-01-01 10:30:00')), toStartOfHour(toDateTime('2026-01-01 10:30:00')) + INTERVAL 1 HOUR) + AND Hour < toStartOfHour(toDateTime('2026-01-03 14:15:00')) + AND ServiceName = 'api' + AND DeploymentEnv = 'production' +) AS duration_tiers FORMAT JSON -- spec:traces-timeseries-aggregates-mv:baseline [cad07ec0] diff --git a/packages/query-engine/src/benchmark/builders.ts b/packages/query-engine/src/benchmark/builders.ts index 378a46c422..b6c8259e2e 100644 --- a/packages/query-engine/src/benchmark/builders.ts +++ b/packages/query-engine/src/benchmark/builders.ts @@ -754,6 +754,20 @@ export const builderFixtures: ReadonlyArray = [ ...webAnalyticsFixtures, ...webAnalyticsAiFixtures, + // Traces sidebar (query-engine runtime); the pipe fixtures cover the raw-only route. + { + module: "errors", + name: "tracesFacetsQuery", + label: "rollup", + compile: () => CH.compileUnionUnsafe(CH.tracesFacetsQuery({ serviceNames: ["api"] }), window), + }, + { + module: "errors", + name: "tracesDurationStatsQuery", + label: "rollup", + compile: () => CH.compileUnsafe(CH.tracesDurationStatsQuery({ serviceNames: ["api"] }), window), + }, + // Error fixtures reached through ErrorsService, v2 telemetry, and observability. { // telemetry.http.ts v2GetSpan / observability/span-detail.ts diff --git a/packages/query-engine/src/benchmark/catalog.test.ts b/packages/query-engine/src/benchmark/catalog.test.ts index d843292942..c7d3c827c6 100644 --- a/packages/query-engine/src/benchmark/catalog.test.ts +++ b/packages/query-engine/src/benchmark/catalog.test.ts @@ -306,8 +306,6 @@ const EXEMPT_BUILDERS: ReadonlySet = new Set([ "errors/errorsByTypeQuery", "errors/errorsTimeseriesQuery", "errors/spanHierarchyQuery", - "errors/tracesDurationStatsQuery", - "errors/tracesFacetsQuery", "errors/errorsFacetsQuery", "errors/errorsSummaryQuery", "errors/errorDetailTracesQuery", diff --git a/packages/query-engine/src/benchmark/catalog.ts b/packages/query-engine/src/benchmark/catalog.ts index 8c6473b147..f36be0924e 100644 --- a/packages/query-engine/src/benchmark/catalog.ts +++ b/packages/query-engine/src/benchmark/catalog.ts @@ -1187,7 +1187,7 @@ export function undecodedColumns( */ const ROLLUP_TABLE_RE = /\w_(?:hourly|minutely|daily)\b|\w_aggregates_\w/ const RAW_TABLE_RE = - /\bFROM\s+(?:traces|logs|service_map_spans|service_map_children|service_overview_spans)\b/ + /\bFROM\s+(?:traces|logs|service_map_spans|service_map_children|service_overview_spans|trace_list_mv)\b/ /** * The `firstFullBucket` fragment `makeGrain` emits. Structural rather than a diff --git a/packages/query-engine/src/ch/ch.test.ts b/packages/query-engine/src/ch/ch.test.ts index b7bcde54f2..faa76524b2 100644 --- a/packages/query-engine/src/ch/ch.test.ts +++ b/packages/query-engine/src/ch/ch.test.ts @@ -1331,9 +1331,8 @@ describe("converted queries", () => { matchModes: { serviceName: "contains" }, }) const { sql } = compileUnsafe(q, baseParams) - expect(sql).toContain("positionCaseInsensitive(ServiceName, 'api') > 0") - expect(sql).toContain("quantile(0.5)(Duration)") - expect(sql).toContain("FROM trace_list_mv") // tracesDurationStats always uses MV directly + // On both tiers: the trace_list_mv edges and the hourly interior. + expect(sql.match(/positionCaseInsensitive\(ServiceName, 'api'\) > 0/g)).toHaveLength(2) }) it("spanHierarchyQuery projects only the trimmed tree attribute keys", () => { diff --git a/packages/query-engine/src/ch/index.ts b/packages/query-engine/src/ch/index.ts index 0c7e069908..84c093d07e 100644 --- a/packages/query-engine/src/ch/index.ts +++ b/packages/query-engine/src/ch/index.ts @@ -334,6 +334,7 @@ export { spanDetailQuery, traceTimeProbeQuery, tracesDurationStatsQuery, + canUseTraceFacetsRollup, tracesFacetsQuery, errorsFacetsQuery, errorsSummaryQuery, diff --git a/packages/query-engine/src/ch/pipe-dispatch.ts b/packages/query-engine/src/ch/pipe-dispatch.ts index 699e3e8cff..b8acef8306 100644 --- a/packages/query-engine/src/ch/pipe-dispatch.ts +++ b/packages/query-engine/src/ch/pipe-dispatch.ts @@ -257,6 +257,8 @@ export function compilePipeQuery( eraseType( compile( tracesDurationStatsQuery({ + // The pipe surface has no missing-table retry, so it keeps the raw read. + rawOnly: true, serviceName: str("service"), spanName: str("span_name"), hasError, @@ -281,6 +283,8 @@ export function compilePipeQuery( eraseType( compileUnion( tracesFacetsQuery({ + // The pipe surface has no missing-table retry, so it keeps the raw read. + rawOnly: true, serviceName: str("service"), spanName: str("span_name"), hasError, diff --git a/packages/query-engine/src/ch/queries/errors.test.ts b/packages/query-engine/src/ch/queries/errors.test.ts index 5bacdcd42c..d05f00af5b 100644 --- a/packages/query-engine/src/ch/queries/errors.test.ts +++ b/packages/query-engine/src/ch/queries/errors.test.ts @@ -11,6 +11,7 @@ import { errorTickBootstrapIssuesQuery, errorTickIssuesQuery, errorFingerprintsQuery, + tracesDurationStatsQuery, tracesFacetsQuery, } from "./errors" @@ -483,8 +484,8 @@ describe("tracesFacetsQuery", () => { it("compiles UNION ALL with 7 facet dimensions", () => { const q = tracesFacetsQuery({}) const { sql } = compileUnionUnsafe(q, baseParams) - const unionCount = (sql.match(/UNION ALL/g) || []).length - expect(unionCount).toBe(6) // 7 queries = 6 UNION ALL + // 7 facet branches, each splicing a raw edge with the hourly interior. + expect(sql.match(/UNION ALL/g)).toHaveLength(6 + 7) expect(sql).toContain("'service' AS facetType") expect(sql).toContain("'spanName' AS facetType") expect(sql).toContain("'httpMethod' AS facetType") @@ -546,7 +547,8 @@ describe("tracesFacetsQuery", () => { it("compiles only the requested branch when facet is set", () => { const q = tracesFacetsQuery({ facet: "service" }) const { sql } = compileUnionUnsafe(q, baseParams) - expect(sql).not.toContain("UNION ALL") + // The only UNION left is the branch's own two tiers. + expect(sql.match(/UNION ALL/g)).toHaveLength(1) expect(sql).toContain("'service' AS facetType") expect(sql).not.toContain("'spanName' AS facetType") expect(sql).not.toContain("'errorCount' AS facetType") @@ -558,9 +560,45 @@ describe("tracesFacetsQuery", () => { it("keeps the non-service branch empty-value guard when facet-scoped", () => { const q = tracesFacetsQuery({ facet: "deploymentEnv" }) const { sql } = compileUnionUnsafe(q, baseParams) - expect(sql).not.toContain("UNION ALL") + expect(sql.match(/UNION ALL/g)).toHaveLength(1) expect(sql).toContain("'deploymentEnv' AS facetType") - expect(sql).toContain("DeploymentEnv != ''") + // On both tiers. + expect(sql.match(/DeploymentEnv != ''/g)).toHaveLength(2) expect(sql).toContain("LIMIT 20") }) }) + +describe("trace facets rollup routing", () => { + it("reads whole hours from trace_facets_hourly and the partial ends from trace_list_mv", () => { + const filters = { serviceNames: ["api", "web"], hasError: true, deploymentEnvs: ["production"] } + for (const sql of [ + compileUnionUnsafe(tracesFacetsQuery({ ...filters, facet: "spanName" }), baseParams).sql, + compileUnsafe(tracesDurationStatsQuery(filters), baseParams).sql, + ]) { + expect(sql).toContain("FROM trace_facets_hourly") + expect(sql).toContain("FROM trace_list_mv") + // Every filter reaches both tiers. + expect(sql.match(/ServiceName IN \('api', 'web'\)/g)).toHaveLength(2) + expect(sql.match(/HasError = 1/g)).toHaveLength(2) + expect(sql.match(/DeploymentEnv = 'production'/g)).toHaveLength(2) + } + }) + + it("reads only trace_list_mv, for the whole window, when the rollup cannot answer", () => { + for (const opts of [ + { rawOnly: true }, + { minDurationMs: 100 }, + { maxDurationMs: 100 }, + { attributeFilterKey: "http.route", attributeFilterValue: "/users" }, + { resourceFilterKey: "host.name", resourceFilterValue: "web-1" }, + ]) { + const facets = compileUnionUnsafe(tracesFacetsQuery(opts), baseParams).sql + expect(facets, JSON.stringify(opts)).not.toContain("trace_facets_hourly") + expect(facets, JSON.stringify(opts)).not.toContain("INTERVAL 1 HOUR") + expect(facets.match(/UNION ALL/g), JSON.stringify(opts)).toHaveLength(6) + } + const stats = compileUnsafe(tracesDurationStatsQuery({ rawOnly: true }), baseParams).sql + expect(stats).not.toContain("trace_facets_hourly") + expect(stats).not.toContain("INTERVAL 1 HOUR") + }) +}) diff --git a/packages/query-engine/src/ch/queries/errors.ts b/packages/query-engine/src/ch/queries/errors.ts index 4bd6268080..3711f266e9 100644 --- a/packages/query-engine/src/ch/queries/errors.ts +++ b/packages/query-engine/src/ch/queries/errors.ts @@ -21,6 +21,7 @@ import { ErrorFingerprintsMinutely, ServiceUsage, TraceDetailSpans, + TraceFacetsHourly, TraceListMv, Traces, } from "../tables" @@ -535,7 +536,12 @@ export function traceTimeProbeQuery(opts: { traceId: string; narrowByTime?: bool .format("JSON") } -// Traces duration stats +// Traces duration stats and facets +// +// Both count root spans over the window. When `trace_facets_hourly` carries the +// filters, its whole hours answer the interior and `trace_list_mv` only the +// partial hour at each end (`rollup-splice`); otherwise `trace_list_mv` answers +// the whole window through the same union. export interface TracesDurationStatsOpts { serviceName?: string @@ -563,16 +569,26 @@ export interface TracesDurationStatsOpts { deploymentEnv?: "contains" serviceNamespace?: "contains" } + /** Read only `trace_list_mv`: for clusters that have not applied migration 0034. */ + rawOnly?: boolean } -export interface TracesDurationStatsOutput { - readonly minDurationMs: number - readonly maxDurationMs: number - readonly p50DurationMs: number - readonly p95DurationMs: number -} - -export function tracesDurationStatsQuery(opts: TracesDurationStatsOpts) { +/** The facet dimensions, spelled the same on `trace_list_mv` and `trace_facets_hourly`. */ +type TraceFacetColumns = Pick< + typeof TraceListMv.columns, + | "ServiceName" + | "SpanName" + | "HttpMethod" + | "HttpStatusCode" + | "DeploymentEnv" + | "ServiceNamespace" + | "HasError" +> + +function traceFacetDimensionConditions( + $: ColumnAccessor, + opts: TracesDurationStatsOpts, +): Array { const mm = opts.matchModes const services = inclusionValues(opts.serviceName, opts.serviceNames) const spanNames = inclusionValues(opts.spanName, opts.spanNames) @@ -581,39 +597,123 @@ export function tracesDurationStatsQuery(opts: TracesDurationStatsOpts) { const envs = inclusionValues(opts.deploymentEnv, opts.deploymentEnvs) const namespaces = inclusionValues(opts.namespace, opts.namespaces) - return from(TraceListMv) + return [ + CH.when(services, (v: readonly string[]) => + matchOrIn($.ServiceName, v, mm?.serviceName === "contains"), + ), + CH.when(spanNames, (v: readonly string[]) => matchOrIn($.SpanName, v, mm?.spanName === "contains")), + CH.whenTrue(!!opts.hasError, () => $.HasError.eq(1)), + CH.when(httpMethods, (v: readonly string[]) => inclusionCondition($.HttpMethod, v)), + CH.when(httpStatusCodes, (v: readonly string[]) => inclusionCondition($.HttpStatusCode, v)), + CH.when(envs, (v: readonly string[]) => + matchOrIn($.DeploymentEnv, v, mm?.deploymentEnv === "contains"), + ), + CH.when(namespaces, (v: readonly string[]) => + matchOrIn($.ServiceNamespace, v, mm?.serviceNamespace === "contains"), + ), + ] +} + +/** + * Whether `trace_facets_hourly` can answer the whole-hour interior. It keeps the + * facet dimensions and nothing finer, so a duration bound or an attribute + * filter, which each need the individual root span, reads `trace_list_mv` for + * the whole window. + */ +export function canUseTraceFacetsRollup( + opts: TracesDurationStatsOpts & { + readonly attributeFilterKey?: string + readonly resourceFilterKey?: string + }, +): boolean { + return ( + !opts.rawOnly && + opts.minDurationMs == null && + opts.maxDurationMs == null && + !opts.attributeFilterKey && + !opts.resourceFilterKey + ) +} + +/** `trace_list_mv` rows in the window, narrowed to the partial end hours when the rollup has the rest. */ +function traceListWindowConditions($: ColumnAccessor, opts: TracesFacetsOpts) { + return [ + $.OrgId.eq(param.string("orgId")), + $.Timestamp.gte(param.dateTimeSeconds("startTime")), + $.Timestamp.lte(param.dateTimeSeconds("endTime")), + ...traceFacetDimensionConditions($, opts), + CH.when(opts.minDurationMs, (v: number) => $.Duration.gte(v * 1000000)), + CH.when(opts.maxDurationMs, (v: number) => $.Duration.lte(v * 1000000)), + CH.whenTrue(canUseTraceFacetsRollup(opts), () => edgeCondition("Timestamp")), + ] +} + +function traceFacetsHourlyInteriorConditions( + $: ColumnAccessor, + opts: TracesDurationStatsOpts, +) { + return [ + $.OrgId.eq(param.string("orgId")), + ...interiorConditions($.Hour), + ...traceFacetDimensionConditions($, opts), + ] +} + +/** The raw tier, plus the hourly interior when the rollup can answer it. */ +function traceFacetTiers>( + opts: TracesFacetsOpts, + raw: CHQuery, + hourly: () => CHQuery, +): CHUnionQuery { + return canUseTraceFacetsRollup(opts) ? unionAll(raw, hourly()) : unionAll(raw) +} + +export interface TracesDurationStatsOutput { + readonly minDurationMs: number + readonly maxDurationMs: number + readonly p50DurationMs: number + readonly p95DurationMs: number +} + +export function tracesDurationStatsQuery( + opts: TracesDurationStatsOpts, +): CHQuery { + // Each tier reports its row count beside its extremes: an aggregate over an + // empty tier returns 0 rather than nothing, and that 0 must not win the + // outer `min`. The t-digest states merge across tiers; an empty one is inert. + const raw = from(TraceListMv) .select(($) => ({ - minDurationMs: CH.min_($.Duration).div(1000000), - maxDurationMs: CH.max_($.Duration).div(1000000), - p50DurationMs: finiteOrZero(CH.quantile(0.5)($.Duration).div(1000000)), - p95DurationMs: finiteOrZero(CH.quantile(0.95)($.Duration).div(1000000)), + traceCount: CH.count(), + durationMin: CH.min_($.Duration), + durationMax: CH.max_($.Duration), + durationQuantiles: CH.rawExpr("quantilesTDigestState(0.5, 0.95)(Duration)", T.string), + })) + .where(($) => traceListWindowConditions($, opts)) + const hourly = () => + from(TraceFacetsHourly) + .select(($) => ({ + traceCount: CH.sum($.TraceCount), + durationMin: CH.min_($.DurationMin), + durationMax: CH.max_($.DurationMax), + durationQuantiles: CH.rawExpr( + "quantilesTDigestMergeState(0.5, 0.95)(DurationQuantiles)", + T.string, + ), + })) + .where(($) => traceFacetsHourlyInteriorConditions($, opts)) + + const quantiles = "quantilesTDigestMerge(0.5, 0.95)(durationQuantiles)" + return fromUnion(traceFacetTiers(opts, raw, hourly), "duration_tiers") + .select(() => ({ + minDurationMs: CH.rawExpr("minIf(durationMin, traceCount > 0) / 1000000", T.float64), + maxDurationMs: CH.rawExpr("maxIf(durationMax, traceCount > 0) / 1000000", T.float64), + p50DurationMs: finiteOrZero(CH.rawExpr(`arrayElement(${quantiles}, 1) / 1000000`, T.float64)), + p95DurationMs: finiteOrZero(CH.rawExpr(`arrayElement(${quantiles}, 2) / 1000000`, T.float64)), })) - .where(($) => [ - $.OrgId.eq(param.string("orgId")), - $.Timestamp.gte(param.dateTimeSeconds("startTime")), - $.Timestamp.lte(param.dateTimeSeconds("endTime")), - CH.when(services, (v: readonly string[]) => - matchOrIn($.ServiceName, v, mm?.serviceName === "contains"), - ), - CH.when(spanNames, (v: readonly string[]) => - matchOrIn($.SpanName, v, mm?.spanName === "contains"), - ), - CH.whenTrue(!!opts.hasError, () => $.HasError.eq(1)), - CH.when(opts.minDurationMs, (v: number) => $.Duration.gte(v * 1000000)), - CH.when(opts.maxDurationMs, (v: number) => $.Duration.lte(v * 1000000)), - CH.when(httpMethods, (v: readonly string[]) => inclusionCondition($.HttpMethod, v)), - CH.when(httpStatusCodes, (v: readonly string[]) => inclusionCondition($.HttpStatusCode, v)), - CH.when(envs, (v: readonly string[]) => - matchOrIn($.DeploymentEnv, v, mm?.deploymentEnv === "contains"), - ), - CH.when(namespaces, (v: readonly string[]) => - matchOrIn($.ServiceNamespace, v, mm?.serviceNamespace === "contains"), - ), - ]) .format("JSON") } -// Traces facets (UNION ALL — 6 facet dimensions on trace_list_mv) +// Traces facets (UNION ALL — 6 facet dimensions and the error count) export type TracesFacetDimension = | "service" @@ -623,32 +723,7 @@ export type TracesFacetDimension = | "deploymentEnv" | "serviceNamespace" -export interface TracesFacetsOpts { - serviceName?: string - spanName?: string - hasError?: boolean - minDurationMs?: number - maxDurationMs?: number - httpMethod?: string - httpStatusCode?: string - deploymentEnv?: string - namespace?: string - /** - * Multi-value spellings, compiled to `IN (...)` against trace_list_mv's - * pre-extracted columns. Each wins over its scalar counterpart when non-empty. - */ - serviceNames?: readonly string[] - spanNames?: readonly string[] - httpMethods?: readonly string[] - httpStatusCodes?: readonly string[] - deploymentEnvs?: readonly string[] - namespaces?: readonly string[] - matchModes?: { - serviceName?: "contains" - spanName?: "contains" - deploymentEnv?: "contains" - serviceNamespace?: "contains" - } +export interface TracesFacetsOpts extends TracesDurationStatsOpts { attributeFilterKey?: string attributeFilterValue?: string attributeFilterValueMatchMode?: "contains" @@ -668,39 +743,8 @@ type StringColumn = { }[keyof Cols & string] export function tracesFacetsQuery(opts: TracesFacetsOpts): CHUnionQuery { - const baseWhere = ($: ColumnAccessor): Array => { - const conditions: Array = [ - $.OrgId.eq(param.string("orgId")), - $.Timestamp.gte(param.dateTimeSeconds("startTime")), - $.Timestamp.lte(param.dateTimeSeconds("endTime")), - ] - - const services = inclusionValues(opts.serviceName, opts.serviceNames) - const spanNames = inclusionValues(opts.spanName, opts.spanNames) - const httpMethods = inclusionValues(opts.httpMethod, opts.httpMethods) - const httpStatusCodes = inclusionValues(opts.httpStatusCode, opts.httpStatusCodes) - const envs = inclusionValues(opts.deploymentEnv, opts.deploymentEnvs) - const namespaces = inclusionValues(opts.namespace, opts.namespaces) - - if (services) { - conditions.push(matchOrIn($.ServiceName, services, opts.matchModes?.serviceName === "contains")) - } - if (spanNames) { - conditions.push(matchOrIn($.SpanName, spanNames, opts.matchModes?.spanName === "contains")) - } - if (opts.hasError) conditions.push($.HasError.eq(1)) - if (opts.minDurationMs != null) conditions.push($.Duration.gte(opts.minDurationMs * 1000000)) - if (opts.maxDurationMs != null) conditions.push($.Duration.lte(opts.maxDurationMs * 1000000)) - if (httpMethods) conditions.push(inclusionCondition($.HttpMethod, httpMethods)) - if (httpStatusCodes) conditions.push(inclusionCondition($.HttpStatusCode, httpStatusCodes)) - if (envs) { - conditions.push(matchOrIn($.DeploymentEnv, envs, opts.matchModes?.deploymentEnv === "contains")) - } - if (namespaces) { - conditions.push( - matchOrIn($.ServiceNamespace, namespaces, opts.matchModes?.serviceNamespace === "contains"), - ) - } + const rawWhere = ($: ColumnAccessor): Array => { + const conditions: Array = traceListWindowConditions($, opts) // Attribute filter EXISTS subqueries (correlated — references outer TraceId) if (opts.attributeFilterKey) { @@ -757,35 +801,60 @@ export function tracesFacetsQuery(opts: TracesFacetsOpts): CHUnionQuery, + colName: StringColumn & StringColumn, facetType: string, - extraWhere?: ($: ColumnAccessor) => CH.Condition, + dropEmpty: boolean, limit = 50, - ) => - from(TraceListMv) + ) => { + const raw = from(TraceListMv) + .select(($) => ({ name: $[colName], count: CH.count() })) + .where(($) => [...rawWhere($), CH.whenTrue(dropEmpty, () => $[colName].neq(""))]) + .groupBy("name") + const hourly = () => + from(TraceFacetsHourly) + .select(($) => ({ name: $[colName], count: CH.sum($.TraceCount) })) + .where(($) => [ + ...traceFacetsHourlyInteriorConditions($, opts), + CH.whenTrue(dropEmpty, () => $[colName].neq("")), + ]) + .groupBy("name") + return fromUnion(traceFacetTiers(opts, raw, hourly), `${facetType}_tiers`) .select(($) => ({ - name: $[colName], - count: CH.count(), + name: $.name, + count: CH.sum($.count), facetType: CH.lit(facetType), })) - .where(($) => [...baseWhere($), extraWhere?.($)]) .groupBy("name") .orderBy(["count", "desc"]) .limit(limit) + } - const facetBranches: Record ReturnType> = { - service: () => makeFacetQuery("ServiceName", "service"), - spanName: () => makeFacetQuery("SpanName", "spanName", ($) => $.SpanName.neq(""), 20), - httpMethod: () => makeFacetQuery("HttpMethod", "httpMethod", ($) => $.HttpMethod.neq(""), 20), - httpStatus: () => makeFacetQuery("HttpStatusCode", "httpStatus", ($) => $.HttpStatusCode.neq(""), 20), - deploymentEnv: () => - makeFacetQuery("DeploymentEnv", "deploymentEnv", ($) => $.DeploymentEnv.neq(""), 20), - serviceNamespace: () => - makeFacetQuery("ServiceNamespace", "serviceNamespace", ($) => $.ServiceNamespace.neq(""), 20), + const errorCountQuery = () => { + const raw = from(TraceListMv) + .select(() => ({ count: CH.count() })) + .where(($) => [...rawWhere($), $.HasError.eq(1)]) + const hourly = () => + from(TraceFacetsHourly) + .select(($) => ({ count: CH.sum($.TraceCount) })) + .where(($) => [...traceFacetsHourlyInteriorConditions($, opts), $.HasError.eq(1)]) + return fromUnion(traceFacetTiers(opts, raw, hourly), "errorCount_tiers").select(($) => ({ + name: CH.lit("error"), + count: CH.sum($.count), + facetType: CH.lit("errorCount"), + })) + } + + const facetBranches = { + service: () => makeFacetQuery("ServiceName", "service", false), + spanName: () => makeFacetQuery("SpanName", "spanName", true, 20), + httpMethod: () => makeFacetQuery("HttpMethod", "httpMethod", true, 20), + httpStatus: () => makeFacetQuery("HttpStatusCode", "httpStatus", true, 20), + deploymentEnv: () => makeFacetQuery("DeploymentEnv", "deploymentEnv", true, 20), + serviceNamespace: () => makeFacetQuery("ServiceNamespace", "serviceNamespace", true, 20), } satisfies Record ReturnType> if (opts.facet) { @@ -799,13 +868,7 @@ export function tracesFacetsQuery(opts: TracesFacetsOpts): CHUnionQuery ({ - name: CH.lit("error"), - count: CH.count(), - facetType: CH.lit("errorCount"), - })) - .where(($) => [...baseWhere($), $.HasError.eq(1)]), + errorCountQuery(), ).format("JSON") } diff --git a/packages/query-engine/src/ch/tables.ts b/packages/query-engine/src/ch/tables.ts index 0a76923c1e..43d7231f09 100644 --- a/packages/query-engine/src/ch/tables.ts +++ b/packages/query-engine/src/ch/tables.ts @@ -190,6 +190,21 @@ export const TraceListMv = table("trace_list_mv", { TraceState: T.string, }) +export const TraceFacetsHourly = table("trace_facets_hourly", { + OrgId: orgId, + Hour: dateTime, + ServiceName: T.string, + SpanName: T.string, + HttpMethod: T.string, + HttpStatusCode: T.string, + DeploymentEnv: T.string, + ServiceNamespace: T.string, + HasError: T.uint8, + TraceCount: T.uint64, + DurationMin: T.uint64, + DurationMax: T.uint64, +}) + export const Logs = table("logs", { OrgId: orgId, Timestamp: dateTime64, diff --git a/packages/query-engine/src/runtime/query-engine.ts b/packages/query-engine/src/runtime/query-engine.ts index 5107a59712..818341fd03 100644 --- a/packages/query-engine/src/runtime/query-engine.ts +++ b/packages/query-engine/src/runtime/query-engine.ts @@ -1277,6 +1277,45 @@ const isMissingServiceOverviewMinutely = (error: unknown): boolean => { return candidate.clickhouseType === "UNKNOWN_TABLE" } +/** + * The missing-table config error naming the rollup. A timeout or memory error + * on the rollup read also names it, and must surface rather than retry against + * the heavier raw table. + */ +const isMissingTraceFacetsRollup = (error: unknown): boolean => { + if (typeof error !== "object" || error === null) return false + const candidate = error as { readonly _tag?: unknown; readonly message?: unknown } + return ( + candidate._tag === "@maple/http/errors/WarehouseConfigError" && + typeof candidate.message === "string" && + /trace_facets_hourly/i.test(candidate.message) + ) +} + +/** + * `trace_facets_hourly` ships in a `requiredForIngest: false` migration (0034), + * so a BYO cluster may not have it yet; the sidebar then reads `trace_list_mv`. + * Only a read that actually included the rollup can be missing it. + */ +const withTraceFacetsFallback = ( + orgId: string, + usesRollup: boolean, + run: (rawOnly: boolean) => Effect.Effect, +): Effect.Effect => + usesRollup + ? run(false).pipe( + Effect.catchIf(isMissingTraceFacetsRollup, () => + Effect.gen(function* () { + yield* Effect.logWarning( + "trace_facets_hourly is absent on this cluster; reading trace_list_mv. Apply ClickHouse schema to restore the fast path.", + ).pipe(Effect.annotateLogs({ orgId })) + yield* Effect.annotateCurrentSpan("query.rollup.fallback", true) + return yield* run(true) + }), + ), + ) + : run(true) + export const makeQueryEngineExecute = (warehouse: QueryEngineWarehouse) => Effect.fn("QueryEngineService.execute")(function* ( tenant: T, @@ -2004,13 +2043,18 @@ export const makeQueryEngineExecute = (warehouse: QueryEn request.query.filters as Record | undefined, ) const facet = request.query.facet - const rows = yield* executeCHUnionQuery( - warehouse, - tenant, - CH.tracesFacetsQuery({ ...opts, facet }), - baseParams, - facet ? `tracesFacets:${facet}` : "tracesFacets", - "discovery", + const rows = yield* withTraceFacetsFallback( + tenant.orgId, + CH.canUseTraceFacetsRollup(opts), + (rawOnly) => + executeCHUnionQuery( + warehouse, + tenant, + CH.tracesFacetsQuery({ ...opts, facet, rawOnly }), + baseParams, + facet ? `tracesFacets:${facet}` : "tracesFacets", + "discovery", + ), ) return new QueryEngineExecuteResponse({ result: { @@ -2130,12 +2174,17 @@ export const makeQueryEngineExecute = (warehouse: QueryEn const opts = extractTracesDurationStatsOpts( request.query.filters as Record | undefined, ) - const rows = yield* executeCHQuery( - warehouse, - tenant, - CH.tracesDurationStatsQuery(opts), - { orgId: tenant.orgId, startTime: request.startTime, endTime: request.endTime }, - "tracesDurationStats", + const rows = yield* withTraceFacetsFallback( + tenant.orgId, + CH.canUseTraceFacetsRollup(opts), + (rawOnly) => + executeCHQuery( + warehouse, + tenant, + CH.tracesDurationStatsQuery({ ...opts, rawOnly }), + { orgId: tenant.orgId, startTime: request.startTime, endTime: request.endTime }, + "tracesDurationStats", + ), ) const row = rows[0] return new QueryEngineExecuteResponse({