diff --git a/CMakeLists.txt b/CMakeLists.txt index cf15eb9..9cc6b47 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -149,6 +149,9 @@ set(EXTENSION_SOURCES src/yardstick_extension.cpp src/yardstick_parser_ffi.cpp src/frontend_peg.cpp + src/aggregate_decorations.cpp + src/aggregate_state.cpp + src/measure_windows.cpp ) if(NOT COMMAND build_static_extension) diff --git a/README.md b/README.md index eaef6db..2089aee 100644 --- a/README.md +++ b/README.md @@ -158,7 +158,33 @@ On DuckDB builds with grammar-extension support, Yardstick recognizes `AS MEASUR Native query traversal lowers CTE bodies, subqueries, and set-operation operands independently. Aggregate calls are discovered from expression nodes, including parenthesized `AT` operands and queries inside INSERT, UPDATE, DELETE, CREATE VIEW, CREATE TABLE AS, EXPLAIN, and COPY statements. Visible filters retain outer query correlations. Subquery projections group their outer column dependencies, while implicit measure projections preserve their column names. DuckDB 1.5.5 retains the compatibility parser; native forms whose source spans cannot be represented also use that path. -The native frontend rejects `DISTINCT`, `FILTER`, `ORDER BY`, `OVER`, and `EXPORT_STATE` directly on one-argument `AGGREGATE()` calls. Their semantics are not implemented, and compatibility lowering could produce incorrect results. Define aggregation behavior in the `AS MEASURE` expression or use supported `AT` modifiers instead. DuckDB's multiargument `aggregate(list, function_name)` remains ordinary DuckDB syntax. +The native frontend supports `DISTINCT`, `FILTER`, argument `ORDER BY`, `OVER`, and `EXPORT_STATE` on one-argument `AGGREGATE()` calls. DuckDB 1.5.5 retains its compatibility frontend; these call decorations require the native frontend. DuckDB's multiargument `aggregate(list, function_name)` remains ordinary DuckDB syntax. + +```sql +SELECT AGGREGATE(DISTINCT revenue), + AGGREGATE(revenue) FILTER (WHERE region = 'US') +FROM sales; + +SELECT year, AGGREGATE(revenue) OVER ( + ORDER BY year ROWS BETWEEN 1 PRECEDING AND CURRENT ROW +) AS rolling_revenue +FROM sales_by_year; +``` + +Call decorations operate on the aggregate functions in the measure definition. For a derived measure such as `SUM(amount) / COUNT(amount)`, `DISTINCT` applies separately to both aggregate inputs, and `FILTER` restricts the base rows for both sides. Scalar arithmetic, casts, and wrappers remain intact. A call filter is combined with an existing definition filter using `AND`. Argument ordering, as in `AGGREGATE(labels ORDER BY priority DESC NULLS LAST)`, takes precedence over definition ordering; the definition's ordering remains as tie breakers. Filter and ordering expressions can reference exposed dimension aliases, including computed dimensions. + +Window partitions and `ROWS`, `RANGE`, or `GROUPS` frames select visible rows, then recompute the measure from their original base rows. This preserves averages and derived ratios when visible rows represent groups of unequal size. A window call's `FILTER` selects frame input rows, so it can reference joined relations; filters in the measure definition still apply to aggregate leaves. Frames retain DuckDB's peer, exclusion, and empty-frame behavior; repeated references introduced by joins do not duplicate the same base row. Window calls also support `AT` context modifiers, which transform the context selected by the filtered frame. + +Window argument ordering can mix source and joined input fields. When a join repeats a base row, its first occurrence under the caller's ordering supplies the joined keys; definition ordering still breaks ties. If `AT` expands the context to a base row absent from the frame, its joined keys are `NULL`, while its source keys are recomputed. + +`AGGREGATE(measure) EXPORT_STATE` exports sufficient aggregate state for later finalization. A single aggregate produces DuckDB's native state; a derived measure produces a composite state carrying its aggregate leaves and scalar formula. `yardstick_finalize(state)` accepts either form. `yardstick_combine(left, right)` combines two states with matching formulas and aggregate types, then `yardstick_finalize` evaluates the result. DuckDB's own aggregate-specific export restrictions still apply. + +```sql +SELECT yardstick_finalize(state) +FROM (SELECT AGGREGATE(revenue) EXPORT_STATE AS state FROM sales); +``` + +Combining exported states follows DuckDB's native state semantics: it does not retain a cross-shard set of distinct input values. Combining states exported with `DISTINCT` therefore does not remove duplicates shared by different shards. Recompute from the combined base rows when global distinctness is required. The native frontend also supports full and partial `CREATE VIEW` column lists for measure views. Header names apply to both dimensions and measures, while derived measures retain their declaration dependencies. Star projections are expanded against the originating session before header positions are mapped. Temporary definitions stay session-local; a single statement cannot combine temporary and permanent measure views with the same name. diff --git a/include/yardstick_ffi.h b/include/yardstick_ffi.h index 96e078d..8febea9 100644 --- a/include/yardstick_ffi.h +++ b/include/yardstick_ffi.h @@ -49,6 +49,9 @@ typedef struct { /* AT modifier chain (supports multiple: AGGREGATE(x) AT (...) AT (...)) */ YardstickAtModifier* modifiers; size_t modifier_count; + const char* call_sql; /* Native call AST, without enclosing AT suffixes */ + bool is_window; + bool has_decorations; } YardstickAggregateCall; /* List of all AGGREGATE() calls found in SQL */ @@ -59,6 +62,38 @@ typedef struct { bool native_parsed; /* Complete grammar-validated source spans */ } YardstickAggregateCallList; +/* Complete source definitions used for query-local window lineage. */ +typedef struct { + const char* key; + const char* relation_name; + const char* alias; + const char* clean_select_sql; + bool grouped; + const char* const* dimension_names; + const char* const* dimension_expressions; + size_t dimension_count; +} YardstickWindowSource; + +typedef struct { + const char* marker_name; + const char* source_key; + const char* expression_sql; + const YardstickAtModifier* modifiers; + size_t modifier_count; +} YardstickWindowCall; + +char* yardstick_decorate_measure( + const char* expression, const char* call_sql, + const char* const* dimension_names, const char* const* dimension_expressions, size_t dimension_count, + const char* const* qualifiers, size_t qualifier_count, + const char* const* binding_ctes, size_t binding_cte_count, char** error); +char* yardstick_window_marker(const char* call_sql, const char* marker_name, char** error); +char* yardstick_rewrite_measure_windows( + const char* sql, const YardstickWindowSource* sources, size_t source_count, + const YardstickWindowCall* calls, size_t call_count, + const char* const* visible_ctes, size_t visible_cte_count, + const char* const* binding_ctes, size_t binding_cte_count, char** error); + /* Grammar-owned CURRENT references, relative to the supplied expression. */ typedef struct { const char* dimension; @@ -87,11 +122,18 @@ char* yardstick_rewrite_visible_filter(const char* expression, const char* local char** error); /* Complete native SELECT scopes, including nested queries and set operands. */ +typedef struct { + uint32_t start_pos; + uint32_t end_pos; + bool recursive; +} YardstickCteDefinition; + typedef struct { uint32_t start_pos; uint32_t end_pos; const char** visible_ctes; size_t visible_cte_count; + YardstickCteDefinition* cte_definitions; } YardstickQueryScope; typedef struct { @@ -116,6 +158,7 @@ typedef struct { bool is_star; /* True if SELECT * or table.* */ bool is_measure_ref; /* True if references AGGREGATE() */ bool contains_subquery; /* Group outer dependencies instead of the subquery expression */ + bool contains_window; /* Window expressions are not implicit grouping keys */ const char* reference_column; /* Native direct column reference, decoded */ const char* reference_qualifier; /* Native direct qualifier, decoded or NULL */ const char** subquery_dimensions; /* Outer column references required by this projection */ @@ -327,7 +370,8 @@ char* yardstick_replace_range( const char* replacement ); -char* yardstick_qualify_expression(const char* expr, const char* qualifier); +/* A non-NULL dimension selects only outer references to that dimension. */ +char* yardstick_qualify_expression(const char* expr, const char* qualifier, const char* dimension); /** * Free a string allocated by yardstick functions. diff --git a/src/aggregate_decorations.cpp b/src/aggregate_decorations.cpp new file mode 100644 index 0000000..086e676 --- /dev/null +++ b/src/aggregate_decorations.cpp @@ -0,0 +1,444 @@ +#include "aggregate_decorations.hpp" + +#if YARDSTICK_GRAMMAR_EXTENSION +#include "duckdb/catalog/catalog.hpp" +#include "duckdb/catalog/entry_lookup_info.hpp" +#include "duckdb/common/exception.hpp" +#include "duckdb/common/string_util.hpp" +#include "duckdb/parser/expression/columnref_expression.hpp" +#include "duckdb/parser/expression/conjunction_expression.hpp" +#include "duckdb/parser/expression/constant_expression.hpp" +#include "duckdb/parser/expression/function_expression.hpp" +#include "duckdb/parser/expression/lambda_expression.hpp" +#include "duckdb/parser/expression/star_expression.hpp" +#include "duckdb/parser/expression/subquery_expression.hpp" +#include "duckdb/parser/parsed_expression_iterator.hpp" +#include "duckdb/parser/parser.hpp" +#include "duckdb/parser/query_node/recursive_cte_node.hpp" +#include "duckdb/parser/query_node/select_node.hpp" +#include "duckdb/parser/query_node/set_operation_node.hpp" +#include "duckdb/parser/statement/select_statement.hpp" +#include "duckdb/parser/tableref/basetableref.hpp" +#include "duckdb/parser/tableref/expressionlistref.hpp" +#include "duckdb/parser/tableref/joinref.hpp" +#include "duckdb/parser/tableref/subqueryref.hpp" +#include "duckdb/parser/tableref/table_function_ref.hpp" +#include "duckdb/planner/binder.hpp" +#include +#include + +namespace duckdb { +namespace { +using Names = std::unordered_set; + +unique_ptr ParseOne(const string &sql, const ParserOptions &options) { + auto expressions = Parser::ParseExpressionList(sql, options); + if (expressions.size() != 1) { + throw ParserException("Expected one expression in decorated AGGREGATE"); + } + return std::move(expressions[0]); +} + +struct DecorationScope { + bool unqualified_is_local = true; + bool inner_alias_shadowed = false; + Names shadowed_qualifiers; + Names shadowed_columns; + Names aliases; + Names lambda_parameters; + vector cte_scopes; +}; + +// Call decorations are written against exposed dimensions. Resolve only names +// owned by the measure view; subquery and lambda bindings keep their own scope. +class DecorationReferences { +public: + DecorationReferences(const vector> &dimensions, const vector &qualifiers, + const ParserOptions &options, bool preserve_dimension_qualifiers) + : preserve_dimension_qualifiers(preserve_dimension_qualifiers) { + for (auto &qualifier : qualifiers) local_qualifiers.insert(StringUtil::Lower(qualifier)); + for (auto &dimension : dimensions) { + auto expression = ParseOne(dimension.second, options); + if (expression->HasSubquery()) { + throw ParserException("Subquery dimensions are not supported in AGGREGATE decorations"); + } + // Resolve base-row names without substituting exposed aliases again. + QualifyDimension(expression, {}, preserve_dimension_qualifiers); + dimension_expressions.emplace(StringUtil::Lower(dimension.first), std::move(expression)); + } + } + + void Expression(unique_ptr &expression, DecorationScope scope = {}) { + if (!expression) return; + if (expression->GetExpressionClass() == ExpressionClass::COLUMN_REF) { + auto &names = expression->Cast().ColumnNames(); + auto first = StringUtil::Lower(names[0].GetIdentifierName()); + if (scope.lambda_parameters.count(first)) return; + bool local = names.size() == 1 + ? scope.unqualified_is_local && !scope.aliases.count(first) && !scope.shadowed_columns.count(first) + : local_qualifiers.count(StringUtil::Lower(names[names.size() - 2].GetIdentifierName())) && + !scope.shadowed_qualifiers.count(StringUtil::Lower(names[names.size() - 2].GetIdentifierName())); + if (local) { + if (scope.inner_alias_shadowed) { + throw ParserException("Conflicting recomputation alias in AGGREGATE decoration"); + } + auto alias = expression->GetAlias(); + auto entry = dimension_expressions.find(StringUtil::Lower(names.back().GetIdentifierName())); + if (entry == dimension_expressions.end() && preserve_dimension_qualifiers && names.size() == 1) { + // The window's consumer may provide this input through a + // join. Let its schema-aware lineage binding resolve it. + return; + } + expression = entry == dimension_expressions.end() + ? make_uniq(names.back(), Identifier("_inner")) + : entry->second->Copy(); + expression->SetAlias(std::move(alias)); + } + return; + } + if (expression->GetExpressionClass() == ExpressionClass::LAMBDA) { + auto &lambda = expression->Cast(); + if (AddLambdaBindings(lambda, scope.lambda_parameters)) { + Expression(lambda.RightMutable(), scope); + return; + } + } + if (expression->GetExpressionClass() == ExpressionClass::SUBQUERY) { + auto &subquery = expression->Cast(); + Expression(subquery.GetChildMutable(), scope); + Query(*subquery.SubqueryMutable()->node, scope); + return; + } + ParsedExpressionIterator::EnumerateChildren(*expression, + [&](unique_ptr &child) { Expression(child, scope); }); + } + +private: + bool preserve_dimension_qualifiers; + Names local_qualifiers; + std::unordered_map> dimension_expressions; + + static bool AddLambdaBindings(LambdaExpression &lambda, Names &bindings) { + // Arrow syntax also means JSON access; only the lambda keyword has + // unambiguous lexical bindings before DuckDB's binder runs. + if (lambda.GetLambdaSyntaxType() != LambdaSyntaxType::LAMBDA_KEYWORD) return false; + string error; + auto parameters = lambda.ExtractColumnRefExpressions(error); + if (!error.empty()) throw ParserException(error); + for (auto ¶meter : parameters) { + auto &names = parameter.get().Cast().ColumnNames(); + if (names.size() != 1) throw ParserException("Invalid AGGREGATE decoration lambda parameter"); + bindings.insert(StringUtil::Lower(names[0].GetIdentifierName())); + } + return true; + } + + static void QualifyDimension(unique_ptr &expression, Names bindings, + bool preserve_dimension_qualifiers) { + if (expression->GetExpressionClass() == ExpressionClass::COLUMN_REF) { + auto &names = expression->Cast().ColumnNamesMutable(); + if (!bindings.count(StringUtil::Lower(names[0].GetIdentifierName()))) { + if (preserve_dimension_qualifiers) { + // Window recomputation projects against the original FROM, + // where joined relations may expose the same column name. + names.insert(names.begin(), Identifier("_inner")); + } else { + auto column = names.back(); + names = {Identifier("_inner"), std::move(column)}; + } + } + return; + } + if (expression->GetExpressionClass() == ExpressionClass::LAMBDA) { + auto &lambda = expression->Cast(); + if (AddLambdaBindings(lambda, bindings)) { + QualifyDimension(lambda.RightMutable(), bindings, preserve_dimension_qualifiers); + return; + } + } + ParsedExpressionIterator::EnumerateChildren(*expression, + [&](unique_ptr &child) { + QualifyDimension(child, bindings, preserve_dimension_qualifiers); + }); + } + + static void Qualifiers(const TableRef &table, Names &names) { + if (!table.alias.empty()) { + names.insert(StringUtil::Lower(table.alias.GetIdentifierName())); + } else if (table.type == TableReferenceType::BASE_TABLE) { + names.insert(StringUtil::Lower(table.Cast().Table().GetIdentifierName())); + } else if (table.type == TableReferenceType::TABLE_FUNCTION) { + auto &function = table.Cast().function; + if (function->GetExpressionClass() == ExpressionClass::FUNCTION) { + names.insert(StringUtil::Lower(function->Cast().FunctionName().GetIdentifierName())); + } + } + if (table.type == TableReferenceType::JOIN) { + auto &join = table.Cast(); + Qualifiers(*join.left, names); + Qualifiers(*join.right, names); + } + } + + static bool ProjectionNames(QueryNode &query, vector &names) { + if (query.type == QueryNodeType::SET_OPERATION_NODE) { + return ProjectionNames(*query.Cast().children[0], names); + } + if (query.type != QueryNodeType::SELECT_NODE) return false; + auto &select = query.Cast(); + for (auto &expression : select.select_list) { + bool expands = false; + ParsedExpressionIterator::VisitExpressionClass(*expression, ExpressionClass::STAR, + [&](const ParsedExpression &) { expands = true; }); + ParsedExpressionIterator::VisitExpression(*expression, + [&](const FunctionExpression &function) { + expands |= StringUtil::CIEquals(function.FunctionName().GetIdentifierName(), "unnest"); + }); + if (expands) return false; + names.push_back(expression->GetName()); + } + return true; + } + + static Names NamedColumns(vector names, const vector &aliases) { + for (idx_t i = 0; i < aliases.size() && i < names.size(); ++i) names[i] = aliases[i]; + Names result; + for (auto &name : names) result.insert(StringUtil::Lower(name.GetIdentifierName())); + return result; + } + + static Names InputColumns(TableRef &table, const DecorationScope &scope) { + if (table.type == TableReferenceType::JOIN) { + // ON conditions can correlate to the outer row. They do not change + // the set of input column names; USING only removes duplicates. + auto &join = table.Cast(); + auto names = InputColumns(*join.left, scope); + auto right = InputColumns(*join.right, scope); + names.insert(right.begin(), right.end()); + return names; + } + if (table.type == TableReferenceType::SUBQUERY) { + vector names; + if (ProjectionNames(*table.Cast().subquery->node, names)) { + return NamedColumns(std::move(names), table.column_name_alias); + } + } + if (table.type == TableReferenceType::BASE_TABLE) { + auto &name = table.Cast().GetQualifiedName(); + if (name.Path().size() == 1) { + for (auto it = scope.cte_scopes.rbegin(); it != scope.cte_scopes.rend(); ++it) { + auto entry = (*it)->cte_map.map.find(name.Name()); + if (entry == (*it)->cte_map.map.end()) continue; + vector names; + if (ProjectionNames(*entry->second->query_node, names)) { + for (idx_t i = 0; i < entry->second->aliases.size() && i < names.size(); ++i) { + names[i] = entry->second->aliases[i]; + } + return NamedColumns(std::move(names), table.column_name_alias); + } + break; + } + } + } + auto context = CurrentNativeYardstickClientContext(); + if (!context) { + throw BinderException("AGGREGATE decoration column resolution requires the originating bind context"); + } + // Binding a layout-only probe discovers table/function/CTE output names + // without evaluating the filter. A FROM clause shadows only its actual + // columns, not every unqualified reference in a correlated expression. + auto probe = make_uniq(); + probe->select_list.push_back(make_uniq()); + probe->from_table = table.Copy(); + auto bound = BindNativeYardstickProbe(*probe, scope.cte_scopes); + Names names; + for (auto &name : bound.names) names.insert(StringUtil::Lower(name.GetIdentifierName())); + return names; + } + + static DecorationScope WithBindings(TableRef &table, DecorationScope scope) { + if (table.type == TableReferenceType::EMPTY_FROM) return scope; + auto columns = InputColumns(table, scope); + scope.shadowed_columns.insert(columns.begin(), columns.end()); + Qualifiers(table, scope.shadowed_qualifiers); + scope.inner_alias_shadowed |= scope.shadowed_qualifiers.count("_inner") != 0; + return scope; + } + + void Query(QueryNode &query, DecorationScope outer) { + outer.cte_scopes.push_back(&query); + for (auto &entry : query.cte_map.map) { + if (!entry.second->query_node) throw ParserException("Unsupported CTE in AGGREGATE decoration"); + Query(*entry.second->query_node, outer); + } + auto scope = outer; + switch (query.type) { + case QueryNodeType::SELECT_NODE: { + auto &select = query.Cast(); + if (select.from_table) { + scope = WithBindings(*select.from_table, outer); + Table(*select.from_table, outer); + } + for (auto &item : select.select_list) { + Expression(item, scope); + if (item->HasAlias()) scope.aliases.insert(StringUtil::Lower(item->GetAlias().GetIdentifierName())); + } + for (auto &group : select.groups.group_expressions) Expression(group, scope); + Expression(select.where_clause, scope); + Expression(select.having, scope); + Expression(select.qualify, scope); + break; + } + case QueryNodeType::SET_OPERATION_NODE: + for (auto &child : query.Cast().children) Query(*child, outer); + scope.unqualified_is_local = false; + break; + case QueryNodeType::RECURSIVE_CTE_NODE: { + auto &cte = query.Cast(); + Query(*cte.left, outer); + Query(*cte.right, outer); + scope.unqualified_is_local = false; + for (auto &key : cte.key_targets) Expression(key, scope); + break; + } + default: + throw ParserException("Unsupported query in AGGREGATE decoration"); + } + ParsedExpressionIterator::EnumerateQueryNodeModifiers(query, + [&](unique_ptr &expression) { Expression(expression, scope); }); + } + + void Table(TableRef &table, DecorationScope scope) { + switch (table.type) { + case TableReferenceType::JOIN: { + auto &join = table.Cast(); + Table(*join.left, scope); + Table(*join.right, WithBindings(*join.left, scope)); + Expression(join.condition, WithBindings(table, scope)); + break; + } + case TableReferenceType::SUBQUERY: + Query(*table.Cast().subquery->node, scope); + break; + case TableReferenceType::EXPRESSION_LIST: + for (auto &row : table.Cast().values) + for (auto &expression : row) Expression(expression, scope); + break; + case TableReferenceType::TABLE_FUNCTION: { + auto &function = table.Cast(); + Expression(function.function, scope); + if (function.subquery) Query(*function.subquery->node, scope); + break; + } + case TableReferenceType::BASE_TABLE: + case TableReferenceType::EMPTY_FROM: + break; + default: + throw ParserException("Unsupported table in AGGREGATE decoration"); + } + } +}; + +bool IsAggregate(const FunctionExpression &function) { + if (auto context = CurrentNativeYardstickClientContext()) { + EntryLookupInfo lookup(CatalogType::AGGREGATE_FUNCTION_ENTRY, function.GetQualifiedName()); + auto entry = Catalog::GetEntry(*context, lookup, OnEntryNotFound::RETURN_NULL); + return entry && entry->type == CatalogType::AGGREGATE_FUNCTION_ENTRY; + } + return IsYardstickStandardAggregate(function.FunctionName().GetIdentifierName()); +} + +class AggregateDecorator { +public: + explicit AggregateDecorator(FunctionExpression &call) : call(call) { + } + + void Expression(unique_ptr &expression) { + if (expression->GetExpressionClass() == ExpressionClass::SUBQUERY || + expression->GetExpressionClass() == ExpressionClass::WINDOW) return; + if (expression->GetExpressionClass() == ExpressionClass::FUNCTION) { + auto &function = expression->Cast(); + if (IsAggregate(function)) { + function.DistinctMutable() |= call.Distinct(); + if (call.Filter()) { + function.FilterMutable() = function.Filter() + ? make_uniq(ExpressionType::CONJUNCTION_AND, + std::move(function.FilterMutable()), call.Filter()->Copy()) + : call.Filter()->Copy(); + } + if (call.OrderBy() && !call.OrderBy()->orders.empty()) { + auto ordering = make_uniq(); + for (auto &order : call.OrderBy()->orders) { + ordering->orders.emplace_back(order.type, order.null_order, order.expression->Copy()); + } + // The caller chooses primary ordering; declaration ordering + // remains deterministic for ties in those call-level keys. + if (function.OrderBy()) { + for (auto &order : function.OrderBy()->orders) { + ordering->orders.emplace_back(order.type, order.null_order, order.expression->Copy()); + } + } + function.OrderByMutable() = std::move(ordering); + } + aggregate_count++; + if (call.ExportState()) { + if (function.ExportState()) { + throw ParserException("Cannot export an already exported measure aggregate"); + } + function.ExportStateMutable() = true; + auto field = "s" + std::to_string(states.size()); + states.emplace_back(Identifier(field), expression->Copy()); + expression = make_uniq(Identifier(field)); + } + return; + } + } + ParsedExpressionIterator::EnumerateChildren(*expression, + [&](unique_ptr &child) { Expression(child); }); + } + + idx_t aggregate_count = 0; + vector states; + +private: + FunctionExpression &call; +}; +} // namespace + +string DecorateYardstickMeasureExpression(const string &measure_expression, const string &call_sql, + const vector> &dimensions, + const vector &local_qualifiers, const ParserOptions &options, + bool preserve_dimension_qualifiers) { + auto expression = ParseOne(measure_expression, options); + auto parsed_call = ParseOne(call_sql, options); + if (parsed_call->GetExpressionClass() != ExpressionClass::FUNCTION) { + throw ParserException("Expected an AGGREGATE call for measure decorations"); + } + auto &call = parsed_call->Cast(); + if (!StringUtil::CIEquals(call.FunctionName().GetIdentifierName(), "aggregate") || + call.GetArguments().size() != 1) { + throw ParserException("Expected a single-argument AGGREGATE call for measure decorations"); + } + DecorationReferences references(dimensions, local_qualifiers, options, preserve_dimension_qualifiers); + references.Expression(call.FilterMutable()); + if (call.OrderBy()) { + for (auto &order : call.OrderByMutable()->orders) references.Expression(order.expression); + } + bool direct_aggregate = expression->GetExpressionClass() == ExpressionClass::FUNCTION && + IsAggregate(expression->Cast()); + AggregateDecorator decorator(call); + decorator.Expression(expression); + if (!decorator.aggregate_count) { + throw ParserException("Decorated AGGREGATE requires a measure containing aggregate functions"); + } + if (call.ExportState()) { + if (direct_aggregate) return decorator.states[0].GetExpression().ToString(); + vector> arguments; + arguments.push_back(ConstantExpression::String(expression->ToString())); + arguments.push_back(make_uniq(Identifier("struct_pack"), std::move(decorator.states))); + expression = make_uniq(Identifier("yardstick_state"), std::move(arguments)); + } + return expression->ToString(); +} +} // namespace duckdb +#endif diff --git a/src/aggregate_state.cpp b/src/aggregate_state.cpp new file mode 100644 index 0000000..7fa6a72 --- /dev/null +++ b/src/aggregate_state.cpp @@ -0,0 +1,231 @@ +#include "aggregate_state.hpp" + +#if YARDSTICK_GRAMMAR_EXTENSION +#include "duckdb/common/exception.hpp" +#include "duckdb/common/string_util.hpp" +#include "duckdb/function/function_binder.hpp" +#include "duckdb/function/scalar_function.hpp" +#include "duckdb/main/extension/extension_loader.hpp" +#include "duckdb/parser/expression/columnref_expression.hpp" +#include "duckdb/parser/parser.hpp" +#include "duckdb/planner/binder.hpp" +#include "duckdb/planner/expression/bound_case_expression.hpp" +#include "duckdb/planner/expression/bound_cast_expression.hpp" +#include "duckdb/planner/expression/bound_constant_expression.hpp" +#include "duckdb/planner/expression/bound_operator_expression.hpp" +#include "duckdb/planner/expression_binder/constant_binder.hpp" + +namespace duckdb { +namespace { + +// Logical-type aliases are serialized with the type, including in persistent tables and prepared statements. +// The versioned alias owns the finalization expression; each sN field retains its native aggregate state type. +constexpr const char *STATE_ALIAS_PREFIX = "yardstick_state_v1:"; + +struct StateExpressionData : FunctionData { + explicit StateExpressionData(unique_ptr expression_p) : expression(std::move(expression_p)) { + } + unique_ptr expression; + + unique_ptr Copy() const override { + return make_uniq(expression->Copy()); + } + bool Equals(const FunctionData &other) const override { + return expression->Equals(*other.Cast().expression); + } +}; + +unique_ptr BindNative(ClientContext &context, const Identifier &name, + vector> arguments) { + FunctionBinder binder(context); + ErrorData error; + auto result = binder.BindScalarFunction(Identifier::DefaultSchema(), name, std::move(arguments), error); + if (!result) { + error.Throw(); + } + return result; +} + +unique_ptr ExtractState(ClientContext &context, const Expression &state, const Identifier &name) { + vector> arguments; + // DuckDB does not implicitly cast an aliased STRUCT to struct_extract's STRUCT parameter. + // Remove only the outer alias: native aggregate-state aliases on the fields must survive. + auto struct_type = state.GetReturnType().WithAlias(""); + arguments.push_back(BoundCastExpression::AddCastToType(context, state.Copy(), struct_type)); + arguments.push_back(make_uniq(Value(name.GetIdentifierName()))); + return BindNative(context, "struct_extract", std::move(arguments)); +} + +bool IsCompositeState(const LogicalType &type) { + return type.id() == LogicalTypeId::STRUCT && type.HasAlias() && + StringUtil::StartsWith(type.GetAlias(), STATE_ALIAS_PREFIX); +} + +void ValidateFields(const LogicalType &type) { + if (type.id() != LogicalTypeId::STRUCT || StructType::GetChildTypes(type).empty()) { + throw BinderException("yardstick_state requires a nonempty STRUCT of native aggregate states"); + } + auto &fields = StructType::GetChildTypes(type); + for (idx_t i = 0; i < fields.size(); i++) { + if (fields[i].first != "s" + std::to_string(i) || !fields[i].second.IsAggregateState()) { + throw BinderException("yardstick_state fields must be native aggregate states named s0, s1, ... in order"); + } + } +} + +class StateFormulaBinder : public ConstantBinder { +public: + StateFormulaBinder(Binder &binder, ClientContext &context, const Expression &state_p) + : ConstantBinder(binder, context, "yardstick_state formula"), state(state_p) { + } + +protected: + BindResult BindExpression(unique_ptr &expression, idx_t depth, bool root_expression) override { + if (expression->GetExpressionClass() != ExpressionClass::COLUMN_REF) { + return ConstantBinder::BindExpression(expression, depth, root_expression); + } + auto &reference = expression->Cast(); + if (!reference.IsQualified()) { + for (auto &field : StructType::GetChildTypes(state.GetReturnType())) { + if (reference.GetColumnName() == field.first) { + vector> arguments; + arguments.push_back(ExtractState(context, state, field.first)); + return BindResult(BindNative(context, "finalize", std::move(arguments))); + } + } + } + throw BinderException("yardstick_state formula references unknown state field %s", reference.ToString()); + } + +private: + const Expression &state; +}; + +unique_ptr ParseFormula(const string &formula) { + auto expressions = Parser::ParseExpressionList(formula); + if (expressions.size() != 1 || !expressions[0]->GetAlias().empty()) { + throw BinderException("yardstick_state requires exactly one scalar finalization expression"); + } + return std::move(expressions[0]); +} + +unique_ptr BindFormula(ClientContext &context, const Expression &state, + unique_ptr formula) { + auto binder = Binder::CreateBinder(context); + StateFormulaBinder expression_binder(*binder, context, state); + return expression_binder.Bind(formula); +} + +unique_ptr IfNull(const Expression &input, unique_ptr when_null, + unique_ptr otherwise) { + auto condition = make_uniq(ExpressionType::OPERATOR_IS_NULL, LogicalType::BOOLEAN); + condition->GetChildrenMutable().push_back(input.Copy()); + return make_uniq(std::move(condition), std::move(when_null), std::move(otherwise)); +} + +unique_ptr ReturnExpression(BindScalarFunctionInput &input, unique_ptr expression) { + input.GetBoundFunction().SetReturnType(expression->GetReturnType()); + return make_uniq(std::move(expression)); +} + +unique_ptr ReplaceStateExpression(FunctionBindExpressionInput &input) { + return input.bind_data->Cast().expression->Copy(); +} + +unique_ptr BindState(BindScalarFunctionInput &input) { + auto &state = *input.GetArguments()[1]; + ValidateFields(state.GetReturnType()); + auto formula = ParseFormula(input.GetConstant(0, false).ToString()); + auto canonical_formula = formula->ToString(); + // Validate the complete formula now, rather than storing a state that only fails at finalization time. + BindFormula(input.GetClientContext(), state, std::move(formula)); + auto type = state.GetReturnType().WithAlias(string(STATE_ALIAS_PREFIX) + canonical_formula); + auto result = BoundCastExpression::AddCastToType(input.GetClientContext(), state.Copy(), type); + return ReturnExpression(input, std::move(result)); +} + +unique_ptr BindFinalize(BindScalarFunctionInput &input) { + auto &state = *input.GetArguments()[0]; + auto &type = state.GetReturnType(); + if (type.id() == LogicalTypeId::SQLNULL) { + return ReturnExpression(input, state.Copy()); + } + if (type.IsAggregateState()) { + vector> arguments; + arguments.push_back(state.Copy()); + return ReturnExpression(input, BindNative(input.GetClientContext(), "finalize", std::move(arguments))); + } + if (!IsCompositeState(type)) { + throw BinderException("yardstick_finalize requires a native aggregate state or yardstick_state"); + } + ValidateFields(type); + auto formula = type.GetAlias().substr(string(STATE_ALIAS_PREFIX).size()); + auto result = BindFormula(input.GetClientContext(), state, ParseFormula(formula)); + auto null_value = make_uniq(Value(result->GetReturnType())); + return ReturnExpression(input, IfNull(state, std::move(null_value), std::move(result))); +} + +unique_ptr BindCombine(BindScalarFunctionInput &input) { + auto &left = *input.GetArguments()[0]; + auto &right = *input.GetArguments()[1]; + auto &left_type = left.GetReturnType(); + auto &right_type = right.GetReturnType(); + if (left_type.id() == LogicalTypeId::SQLNULL && right_type.id() == LogicalTypeId::SQLNULL) { + return ReturnExpression(input, left.Copy()); + } + auto &type = left_type.id() == LogicalTypeId::SQLNULL ? right_type : left_type; + if (!type.IsAggregateState() && !IsCompositeState(type)) { + throw BinderException("yardstick_combine requires native aggregate states or yardstick_state values"); + } + if (left_type.id() == LogicalTypeId::SQLNULL || right_type.id() == LogicalTypeId::SQLNULL) { + return ReturnExpression(input, left_type.id() == LogicalTypeId::SQLNULL ? right.Copy() : left.Copy()); + } + if (left_type != right_type) { + throw BinderException("yardstick_combine requires matching state formulas and aggregate types"); + } + if (type.IsAggregateState()) { + vector> arguments; + arguments.push_back(left.Copy()); + arguments.push_back(right.Copy()); + return ReturnExpression(input, BindNative(input.GetClientContext(), "combine", std::move(arguments))); + } + ValidateFields(type); + vector> fields; + for (auto &field : StructType::GetChildTypes(type)) { + vector> arguments; + arguments.push_back(ExtractState(input.GetClientContext(), left, field.first)); + arguments.push_back(ExtractState(input.GetClientContext(), right, field.first)); + auto combined = BindNative(input.GetClientContext(), "combine", std::move(arguments)); + combined->SetAlias(field.first); + fields.push_back(std::move(combined)); + } + auto packed = BindNative(input.GetClientContext(), "struct_pack", std::move(fields)); + auto result = BoundCastExpression::AddCastToType(input.GetClientContext(), std::move(packed), type); + result = IfNull(right, left.Copy(), std::move(result)); + result = IfNull(left, right.Copy(), std::move(result)); + return ReturnExpression(input, std::move(result)); +} + +} // namespace + +void RegisterYardstickAggregateStateFunctions(ExtensionLoader &loader) { + ScalarFunction state("yardstick_state", {LogicalType::VARCHAR, LogicalType::ANY}, LogicalType::ANY, nullptr, BindState); + state.SetBindExpressionCallback(ReplaceStateExpression); + state.SetNullHandling(FunctionNullHandling::SPECIAL_HANDLING); + loader.RegisterFunction(state); + ScalarFunction finalize("yardstick_finalize", {LogicalType::ANY}, LogicalType::ANY, nullptr, BindFinalize); + finalize.SetBindExpressionCallback(ReplaceStateExpression); + finalize.SetNullHandling(FunctionNullHandling::SPECIAL_HANDLING); + loader.RegisterFunction(finalize); + ScalarFunction combine("yardstick_combine", {LogicalType::ANY, LogicalType::ANY}, LogicalType::ANY, nullptr, BindCombine); + combine.SetBindExpressionCallback(ReplaceStateExpression); + combine.SetNullHandling(FunctionNullHandling::SPECIAL_HANDLING); + loader.RegisterFunction(combine); +} +} // namespace duckdb +#else +namespace duckdb { +void RegisterYardstickAggregateStateFunctions(ExtensionLoader &) { +} +} // namespace duckdb +#endif diff --git a/src/frontend_peg.cpp b/src/frontend_peg.cpp index e94668c..0eb4100 100644 --- a/src/frontend_peg.cpp +++ b/src/frontend_peg.cpp @@ -18,6 +18,8 @@ #include "duckdb/parser/peg/compiled_grammar.hpp" #include "duckdb/parser/peg/transformer/peg_transformer.hpp" #include "duckdb/planner/binder.hpp" +#include "duckdb/planner/table_binding.hpp" +#include "duckdb/parser/statement/select_statement.hpp" #include #include @@ -446,16 +448,23 @@ class YardstickGrammar final : public GrammarExtension { struct YardstickGrammarInfo final : ParserExtensionInfo { shared_ptr grammar; + shared_ptr base_grammar; }; shared_ptr SelectGrammar(ParserExtensionInfo *info, const ParserOptions &options) { - if (options.compiled_grammar && options.compiled_grammar->HasGrammarChanges()) { - return options.compiled_grammar->GetRule("YardstickAtModifier") ? options.compiled_grammar : nullptr; + if (options.compiled_grammar && options.compiled_grammar->GetRule("YardstickAtModifier")) { + return options.compiled_grammar; } if (!info) { return nullptr; } - return info->Cast().grammar; + auto &yardstick = info->Cast(); + // Only replace the database's default grammar. A different active grammar + // or dialect that lacks Yardstick rules must retain its own parser. + if (options.compiled_grammar && options.compiled_grammar != yardstick.base_grammar) { + return nullptr; + } + return yardstick.grammar; } } // namespace @@ -463,6 +472,7 @@ shared_ptr SelectGrammar(ParserExtensionInfo *info, const Parse shared_ptr RegisterYardstickGrammar(DatabaseInstance &db) { GrammarExtension::Register(db, make_shared_ptr()); auto info = make_shared_ptr(); + info->base_grammar = db.GetParserCache().GetMatcher(); ClientContext context(db.shared_from_this()); // Compile without changing any connection's active_grammar_extensions. info->grammar = CompiledGrammar::Create(context, {"yardstick"}); @@ -502,6 +512,71 @@ ClientContext *CurrentNativeYardstickClientContext() { return active_bind_context; } +namespace { +thread_local const vector> *active_binding_ctes = nullptr; +} + +const vector> *CurrentNativeYardstickCteBindings() { + return active_binding_ctes; +} + +NativeYardstickCteBindScope::NativeYardstickCteBindScope(const vector> *context) + : previous(active_binding_ctes) { + active_binding_ctes = context; +} + +NativeYardstickCteBindScope::NativeYardstickCteBindScope(const vector &queries, + const ParserOptions &options) + : previous(active_binding_ctes) { + for (auto &sql : queries) { + Parser parser(options); + parser.ParseQuery(sql); + if (parser.statements.size() != 1 || parser.statements[0]->type != StatementType::SELECT_STATEMENT) { + throw ParserException("Expected a SELECT carrying native CTE definitions"); + } + definitions.push_back(parser.statements[0]->Cast().node->Copy()); + } + active_binding_ctes = &definitions; +} + +NativeYardstickCteBindScope::~NativeYardstickCteBindScope() { + active_binding_ctes = previous; +} + +BoundStatement BindNativeYardstickProbe(QueryNode &probe, const vector &local_scopes) { + auto context = CurrentNativeYardstickClientContext(); + if (!context) { + throw BinderException("Native query binding requires the originating bind context"); + } + // Mirror DuckDB's lazy CTE binder chain. Each definition retains its own + // lexical parent, so an inner name cannot change an earlier CTE's meaning. + // These plans are used only for schema inspection and are never executed. + vector> definitions; + vector> binders {Binder::CreateBinder(*context)}; + auto add_scope = [&](QueryNode &scope) { + for (auto &entry : scope.cte_map.map) { + // Binding consumes parts of the AST; every probe needs its own copy. + definitions.push_back(entry.second->Copy()); + auto &definition = *definitions.back(); + auto &parent = *binders.back(); + auto state = make_shared_ptr(parent, *definition.query_node, definition.aliases); + auto child = Binder::CreateBinder(*context, parent); + child->bind_context.AddCTEBinding( + make_uniq(BindingAlias(entry.first), state, parent.GenerateTableIndex())); + binders.push_back(std::move(child)); + } + }; + if (active_binding_ctes) { + for (auto &scope : *active_binding_ctes) { + add_scope(*scope); + } + } + for (auto *scope : local_scopes) { + add_scope(*scope); + } + return binders.back()->Bind(probe); +} + bool ParseNativeYardstickQuery(const string &sql, Parser &parser) { if (!active_parse_scope || !active_parse_scope->available || Parser::NormalizeSQLString(sql) != sql) { return false; @@ -766,6 +841,7 @@ struct NativeQueryScope { idx_t start; idx_t end; vector visible_ctes; + vector cte_definitions; }; // QueryNode does not retain a complete source location. The native parse tree @@ -776,10 +852,11 @@ bool CollectNativeQueryScopes(ParseResult &root, PEGTransformer &transformer, struct Work { ParseResult *node; vector visible_ctes; + vector cte_definitions; QueryLocation main_query; idx_t modifier_end = 0; }; - vector pending {{&root, {}, {}, 0}}; + vector pending {{&root, {}, {}, {}, 0}}; while (!pending.empty()) { auto work = std::move(pending.back()); pending.pop_back(); @@ -798,7 +875,7 @@ bool CollectNativeQueryScopes(ParseResult &root, PEGTransformer &transformer, end == work.main_query.End()) { end = MaxValue(end, work.modifier_end); } - scopes.push_back({location.Start(), end, work.visible_ctes}); + scopes.push_back({location.Start(), end, work.visible_ctes, work.cte_definitions}); } // WITH is the first optional child of SELECT/INSERT/UPDATE/DELETE. // Handle its declarations in order: a non-recursive body sees earlier @@ -828,13 +905,19 @@ bool CollectNativeQueryScopes(ParseResult &root, PEGTransformer &transformer, auto &list = declaration->Cast(); auto name = transformer.Transform(list.GetChild(0)).GetIdentifierName(); auto visible = work.visible_ctes; + auto definitions = work.cte_definitions; + auto location = declaration->GetLocation(); + YardstickCteDefinition definition {static_cast(location.Start()), + static_cast(location.End()), recursive}; if (recursive) { visible.push_back(name); + definitions.push_back(definition); } // Visiting the declaration also finds CTE bodies nested in // DML; it never infers a query from parentheses or text. - pending.push_back({declaration, std::move(visible), {}, 0}); + pending.push_back({declaration, std::move(visible), std::move(definitions), {}, 0}); work.visible_ctes.push_back(std::move(name)); + work.cte_definitions.push_back(definition); } first_child = 1; } @@ -844,7 +927,8 @@ bool CollectNativeQueryScopes(ParseResult &root, PEGTransformer &transformer, work.modifier_end = node.GetLocation().End(); } for (idx_t i = first_child; i < children.size(); i++) { - pending.push_back({&children[i].get(), work.visible_ctes, work.main_query, work.modifier_end}); + pending.push_back({&children[i].get(), work.visible_ctes, work.cte_definitions, + work.main_query, work.modifier_end}); } } return true; @@ -907,9 +991,11 @@ YardstickQueryScopeList *FindNativeYardstickQueryScopes(const char *sql_p) { scope.end_pos = static_cast(source.end); if (!source.visible_ctes.empty()) { scope.visible_ctes = new const char *[source.visible_ctes.size()] {}; + scope.cte_definitions = new YardstickCteDefinition[source.cte_definitions.size()] {}; scope.visible_cte_count = source.visible_ctes.size(); for (idx_t j = 0; j < source.visible_ctes.size(); j++) { scope.visible_ctes[j] = strdup(source.visible_ctes[j].c_str()); + scope.cte_definitions[j] = source.cte_definitions[j]; if (!scope.visible_ctes[j]) { throw std::bad_alloc(); } @@ -971,7 +1057,7 @@ YardstickAggregateCallList *FindNativeYardstickAggregates(const char *sql_p) { if (!sql_p || !active_parse_scope || !active_parse_scope->available) { return nullptr; } - string semantic_error; + bool found_decorated_call = false; try { string sql(sql_p); if (sql.size() > std::numeric_limits::max() || Parser::NormalizeSQLString(sql) != sql) { @@ -987,46 +1073,14 @@ YardstickAggregateCallList *FindNativeYardstickAggregates(const char *sql_p) { return nullptr; } - // Validate every expression before source-span adaptation can choose - // compatibility lowering for a different operand in the same query. - std::function &)> validate_expression; - validate_expression = [&](unique_ptr &expression) { - bool unsupported = false; - if (expression->GetExpressionClass() == ExpressionClass::WINDOW) { - auto &window = expression->Cast(); - unsupported = StringUtil::CIEquals(window.FunctionName().GetIdentifierName(), "aggregate") && - window.GetArguments().size() == 1; - } else if (expression->GetExpressionClass() == ExpressionClass::FUNCTION) { - auto &function = expression->Cast(); - unsupported = StringUtil::CIEquals(function.FunctionName().GetIdentifierName(), "aggregate") && - function.GetArguments().size() == 1 && - (function.Distinct() || function.Filter() || function.ExportState() || - (function.OrderBy() && !function.OrderBy()->orders.empty())); - } - if (unsupported) { - semantic_error = "Yardstick AGGREGATE does not support DISTINCT, FILTER, ORDER BY, OVER, or EXPORT_STATE; " - "define aggregation behavior in AS MEASURE or use AT modifiers"; - throw ParserException(semantic_error); - } - if (expression->GetExpressionClass() == ExpressionClass::SUBQUERY) { - auto &subquery = expression->Cast(); - ParsedExpressionIterator::EnumerateQueryNodeChildren(*subquery.SubqueryMutable()->node, - validate_expression); - } - ParsedExpressionIterator::EnumerateChildren(*expression, validate_expression); - }; - for (auto &statement : parser.statements) { - EnumerateNativeStatementExpressions(*statement, validate_expression); - } - for (auto &expression : capture.modifier_expressions) { - validate_expression(expression); - } - struct Aggregate { string measure; idx_t start; idx_t end; vector modifiers; + string call_sql; + bool is_window; + bool has_decorations; }; vector aggregates; vector used_clauses(capture.clauses.size(), false); @@ -1084,35 +1138,50 @@ YardstickAggregateCallList *FindNativeYardstickAggregates(const char *sql_p) { base = arguments[0].GetExpressionMutable().get(); } - bool is_measure_call = false; + const vector *arguments = nullptr; + bool is_window = false; + bool has_decorations = false; if (base->GetExpressionClass() == ExpressionClass::FUNCTION) { auto &function = base->Cast(); - is_measure_call = StringUtil::CIEquals(function.FunctionName().GetIdentifierName(), "aggregate") && - function.GetArguments().size() == 1; - if (is_measure_call) { - auto call_location = source_range(function); - auto &argument = function.GetArguments()[0]; - auto argument_location = source_range(argument.GetExpression()); - if (argument.HasName() || argument_location.Start() < call_location.Start() || - argument_location.End() > call_location.End()) { - throw ParserException("Unsupported Yardstick aggregate argument source"); - } - Aggregate aggregate {sql.substr(argument_location.Start(), argument_location.length), - call_location.Start(), call_location.End(), {}}; - // AST parents run from the last suffix back to the first. - // Modifier application retains the original SQL order. - for (auto suffix = suffixes.rbegin(); suffix != suffixes.rend(); ++suffix) { - auto &clause = capture.clauses[*suffix]; - if (clause.start < aggregate.end || clause.end > sql.size() || - !extend_operand(aggregate.start, aggregate.end, clause.start)) { - throw ParserException("Invalid Yardstick AT source range"); - } - aggregate.modifiers.insert(aggregate.modifiers.end(), clause.modifiers.begin(), - clause.modifiers.end()); - aggregate.end = clause.end; + if (StringUtil::CIEquals(function.FunctionName().GetIdentifierName(), "aggregate")) { + arguments = &function.GetArguments(); + has_decorations = function.Distinct() || function.Filter() || function.ExportState() || + (function.OrderBy() && !function.OrderBy()->orders.empty()); + } + } else if (base->GetExpressionClass() == ExpressionClass::WINDOW) { + auto &window = base->Cast(); + if (StringUtil::CIEquals(window.FunctionName().GetIdentifierName(), "aggregate")) { + arguments = &window.GetArguments(); + is_window = true; + has_decorations = true; + } + } + bool is_measure_call = arguments && arguments->size() == 1; + if (is_measure_call) { + found_decorated_call |= has_decorations; + auto call_location = source_range(*base); + auto &argument = (*arguments)[0]; + auto argument_location = source_range(argument.GetExpression()); + if (argument.HasName() || argument_location.Start() < call_location.Start() || + argument_location.End() > call_location.End()) { + throw ParserException("Unsupported Yardstick aggregate argument source"); + } + Aggregate aggregate {sql.substr(argument_location.Start(), argument_location.length), + call_location.Start(), call_location.End(), {}, base->ToString(), + is_window, has_decorations}; + // AST parents run from the last suffix back to the first. + // Modifier application retains the original SQL order. + for (auto suffix = suffixes.rbegin(); suffix != suffixes.rend(); ++suffix) { + auto &clause = capture.clauses[*suffix]; + if (clause.start < aggregate.end || clause.end > sql.size() || + !extend_operand(aggregate.start, aggregate.end, clause.start)) { + throw ParserException("Invalid Yardstick AT source range"); } - aggregates.push_back(std::move(aggregate)); + aggregate.modifiers.insert(aggregate.modifiers.end(), clause.modifiers.begin(), + clause.modifiers.end()); + aggregate.end = clause.end; } + aggregates.push_back(std::move(aggregate)); } if (!suffixes.empty() && !is_measure_call) { // Shorthand AT expressions still use compatibility lowering. @@ -1166,6 +1235,9 @@ YardstickAggregateCallList *FindNativeYardstickAggregates(const char *sql_p) { auto &source = aggregates[i]; auto &call = result->calls[i]; call.measure_name = duplicate(source.measure); + call.call_sql = duplicate(source.call_sql); + call.is_window = source.is_window; + call.has_decorations = source.has_decorations; call.start_pos = static_cast(source.start); call.end_pos = static_cast(source.end); if (!source.modifiers.empty()) { @@ -1185,11 +1257,11 @@ YardstickAggregateCallList *FindNativeYardstickAggregates(const char *sql_p) { } } return result.release(); - } catch (const std::exception &) { - if (!semantic_error.empty()) { + } catch (const std::exception &error) { + if (found_decorated_call) { auto *result = new YardstickAggregateCallList {}; result->native_parsed = true; - result->error = strdup(semantic_error.c_str()); + result->error = strdup(error.what()); return result; } return nullptr; diff --git a/src/include/aggregate_decorations.hpp b/src/include/aggregate_decorations.hpp new file mode 100644 index 0000000..255db9d --- /dev/null +++ b/src/include/aggregate_decorations.hpp @@ -0,0 +1,12 @@ +#pragma once + +#include "frontend_peg.hpp" + +namespace duckdb { +#if YARDSTICK_GRAMMAR_EXTENSION +string DecorateYardstickMeasureExpression(const string &measure_expression, const string &call_sql, + const vector> &dimensions, + const vector &local_qualifiers, const ParserOptions &options, + bool preserve_dimension_qualifiers = false); +#endif +} // namespace duckdb diff --git a/src/include/aggregate_state.hpp b/src/include/aggregate_state.hpp new file mode 100644 index 0000000..08df753 --- /dev/null +++ b/src/include/aggregate_state.hpp @@ -0,0 +1,10 @@ +#pragma once + +#include "yardstick_compat.hpp" + +namespace duckdb { +class ExtensionLoader; + +// Composite state helpers depend on the typed aggregate states in the native frontend target. +void RegisterYardstickAggregateStateFunctions(ExtensionLoader &loader); +} // namespace duckdb diff --git a/src/include/frontend_peg.hpp b/src/include/frontend_peg.hpp index c6ab433..b7b3c13 100644 --- a/src/include/frontend_peg.hpp +++ b/src/include/frontend_peg.hpp @@ -47,6 +47,26 @@ class NativeYardstickBindScope { }; ClientContext *CurrentNativeYardstickClientContext(); +class QueryNode; +struct BoundStatement; + +// CTE definitions are visible to schema probes only, never copied into the +// emitted query. Nested scopes restore the caller's lexical binding context. +class NativeYardstickCteBindScope { +public: + explicit NativeYardstickCteBindScope(const vector> *context); + NativeYardstickCteBindScope(const vector &queries, const ParserOptions &options); + ~NativeYardstickCteBindScope(); + NativeYardstickCteBindScope(const NativeYardstickCteBindScope &) = delete; + NativeYardstickCteBindScope &operator=(const NativeYardstickCteBindScope &) = delete; +private: + vector> definitions; + const vector> *previous; +}; + +const vector> *CurrentNativeYardstickCteBindings(); +BoundStatement BindNativeYardstickProbe(QueryNode &probe, const vector &local_scopes = {}); + // Parse through the active grammar while retaining Yardstick syntax capture. bool ParseNativeYardstickQuery(const string &sql, Parser &parser); diff --git a/src/include/measure_windows.hpp b/src/include/measure_windows.hpp new file mode 100644 index 0000000..d55a9b6 --- /dev/null +++ b/src/include/measure_windows.hpp @@ -0,0 +1,44 @@ +#pragma once + +#include "yardstick_compat.hpp" + +#if YARDSTICK_GRAMMAR_EXTENSION +#include "duckdb/common/common.hpp" +#include "duckdb/common/pair.hpp" +#include "duckdb/parser/parser_options.hpp" + +namespace duckdb { + +enum class WindowContextType { ALL, ALL_GLOBAL, SET, WHERE, VISIBLE }; + +struct WindowContextModifier { + WindowContextType type; + string dimension; + string value; +}; + +struct MeasureWindowSource { + string key; + string relation_name; + string alias; + string clean_select_sql; + bool grouped = false; + vector> dimensions; +}; + +struct MeasureWindowCall { + string marker_name; + string source_key; + string expression_sql; + vector modifiers; +}; + +// Windows select a set of original base-row identities. Repeated identities +// introduced by joins never multiply a measure, while identical original rows +// retain their independent identities. +string RewriteNativeMeasureWindows(const string &scope_sql, const vector &sources, + const vector &calls, const vector &visible_ctes, + const vector &binding_ctes, const ParserOptions &options); + +} // namespace duckdb +#endif diff --git a/src/measure_windows.cpp b/src/measure_windows.cpp new file mode 100644 index 0000000..ddc92a1 --- /dev/null +++ b/src/measure_windows.cpp @@ -0,0 +1,1369 @@ +#include "measure_windows.hpp" + +#if YARDSTICK_GRAMMAR_EXTENSION +#include "frontend_peg.hpp" +#include "duckdb/catalog/catalog.hpp" +#include "duckdb/catalog/entry_lookup_info.hpp" +#include "duckdb/common/string_util.hpp" +#include "duckdb/parser/expression/columnref_expression.hpp" +#include "duckdb/parser/expression/constant_expression.hpp" +#include "duckdb/parser/expression/function_expression.hpp" +#include "duckdb/parser/expression/lambda_expression.hpp" +#include "duckdb/parser/expression/star_expression.hpp" +#include "duckdb/parser/expression/subquery_expression.hpp" +#include "duckdb/parser/expression/window_expression.hpp" +#include "duckdb/parser/parsed_data/create_view_info.hpp" +#include "duckdb/parser/parsed_expression_iterator.hpp" +#include "duckdb/parser/parser.hpp" +#include "duckdb/parser/query_node/select_node.hpp" +#include "duckdb/parser/query_node/set_operation_node.hpp" +#include "duckdb/parser/query_node/recursive_cte_node.hpp" +#include "duckdb/parser/statement/create_statement.hpp" +#include "duckdb/parser/statement/select_statement.hpp" +#include "duckdb/parser/tableref/basetableref.hpp" +#include "duckdb/parser/tableref/joinref.hpp" +#include "duckdb/parser/tableref/subqueryref.hpp" +#include "duckdb/parser/tableref/expressionlistref.hpp" +#include "duckdb/parser/tableref/table_function_ref.hpp" +#include "duckdb/planner/binder.hpp" + +#include +#include +#include + +namespace duckdb { +namespace { + +using ExpressionMap = std::unordered_map>; +using Names = std::unordered_set; + +string Key(const string &text) { + return StringUtil::Lower(text); +} + +string Quote(const string &text) { + return "\"" + StringUtil::Replace(text, "\"", "\"\"") + "\""; +} + +unique_ptr Expression(const string &sql, const ParserOptions &options) { + auto expressions = Parser::ParseExpressionList(sql, options); + if (expressions.size() != 1) { + throw ParserException("Expected one expression while rewriting a measure window"); + } + return std::move(expressions[0]); +} + +unique_ptr ModifierExpression(const string &sql, const ParserOptions &options) { + auto references = FindNativeYardstickCurrentReferences(sql.c_str()); + if (!references) { + return Expression(sql, options); + } + string normalized = sql; + string error = references->error ? references->error : ""; + for (idx_t i = references->count; i > 0; i--) { + auto &reference = references->references[i - 1]; + normalized.replace(reference.start_pos, reference.end_pos - reference.start_pos, + "current(" + string(reference.dimension) + ")"); + } + yardstick_free_current_reference_list(references); + if (!error.empty()) { + throw ParserException(error); + } + return Expression(normalized, options); +} + +unique_ptr Query(const string &sql, const ParserOptions &options, + vector *view_aliases = nullptr) { + Parser parser(options); + parser.ParseQuery(sql); + if (parser.statements.size() != 1) { + throw ParserException("Expected one query while rewriting a measure window"); + } + auto &statement = *parser.statements[0]; + if (statement.type == StatementType::CREATE_STATEMENT) { + auto &info = statement.Cast().info->Cast(); + if (view_aliases) { + *view_aliases = info.aliases; + } + return info.query->node->Copy(); + } + return statement.Cast().node->Copy(); +} + +void Walk(unique_ptr &expression, + const std::function &)> &visit, Names lambda_parameters = {}) { + if (expression && expression->GetExpressionClass() == ExpressionClass::COLUMN_REF && + lambda_parameters.count(Key(expression->Cast().ColumnNames()[0].GetIdentifierName()))) { + return; + } + if (!expression || !visit(expression) || expression->GetExpressionClass() == ExpressionClass::SUBQUERY) { + return; + } + if (expression->GetExpressionClass() == ExpressionClass::LAMBDA) { + auto &lambda = expression->Cast(); + if (lambda.GetLambdaSyntaxType() == LambdaSyntaxType::LAMBDA_KEYWORD) { + string error; + auto parameters = lambda.ExtractColumnRefExpressions(error); + if (!error.empty()) { + throw ParserException(error); + } + for (auto ¶meter : parameters) { + lambda_parameters.insert(Key(parameter.get().Cast() + .GetColumnName().GetIdentifierName())); + } + Walk(lambda.RightMutable(), visit, std::move(lambda_parameters)); + return; + } + } + ParsedExpressionIterator::EnumerateChildren(*expression, [&](unique_ptr &child) { + Walk(child, visit, lambda_parameters); + }); +} + +void SelectExpressions(SelectNode &select, + const std::function &)> &visit) { + for (auto &expression : select.select_list) { + visit(expression); + } + for (auto &expression : select.groups.group_expressions) { + visit(expression); + } + if (select.having) { + visit(select.having); + } + if (select.qualify) { + visit(select.qualify); + } + ParsedExpressionIterator::EnumerateQueryNodeModifiers(select, visit); +} + +ExpressionMap Aliases(const SelectNode &select) { + ExpressionMap aliases; + for (auto &expression : select.select_list) { + if (expression->HasAlias()) { + auto copy = expression->Copy(); + copy->ClearAlias(); + aliases.emplace(Key(expression->GetAlias().GetIdentifierName()), std::move(copy)); + } + } + return aliases; +} + +void ExpandAliases(unique_ptr &expression, const ExpressionMap &aliases, + const Names &input_columns = {}, Names expanding = {}) { + if (!expression || expression->GetExpressionClass() == ExpressionClass::SUBQUERY) { + return; + } + if (expression->GetExpressionClass() == ExpressionClass::LAMBDA) { + auto &lambda = expression->Cast(); + if (lambda.GetLambdaSyntaxType() == LambdaSyntaxType::LAMBDA_KEYWORD) { + auto bindings = input_columns; + string error; + auto parameters = lambda.ExtractColumnRefExpressions(error); + if (!error.empty()) { + throw ParserException(error); + } + for (auto ¶meter : parameters) { + bindings.insert(Key(parameter.get().Cast().GetColumnName().GetIdentifierName())); + } + ExpandAliases(lambda.RightMutable(), aliases, bindings, expanding); + return; + } + } + if (expression->GetExpressionClass() == ExpressionClass::COLUMN_REF) { + auto &column = expression->Cast(); + if (!column.IsQualified()) { + auto key = Key(column.GetColumnName().GetIdentifierName()); + auto match = aliases.find(key); + if (match != aliases.end() && !input_columns.count(key) && !expanding.count(key)) { + expanding.insert(key); + auto alias = expression->GetAlias(); + expression = match->second->Copy(); + expression->SetAlias(alias); + ExpandAliases(expression, aliases, input_columns, std::move(expanding)); + return; + } + } + } + ParsedExpressionIterator::EnumerateChildren(*expression, [&](unique_ptr &child) { + ExpandAliases(child, aliases, input_columns, expanding); + }); +} + +bool HasAggregate(unique_ptr &expression) { + bool found = false; + Walk(expression, [&](unique_ptr &node) { + if (node->GetExpressionClass() == ExpressionClass::WINDOW) { + return false; + } + if (node->GetExpressionClass() == ExpressionClass::FUNCTION) { + auto &function = node->Cast(); + if (auto context = CurrentNativeYardstickClientContext()) { + EntryLookupInfo lookup(CatalogType::AGGREGATE_FUNCTION_ENTRY, function.GetQualifiedName()); + auto entry = Catalog::GetEntry(*context, lookup, OnEntryNotFound::RETURN_NULL); + found |= entry && entry->type == CatalogType::AGGREGATE_FUNCTION_ENTRY; + } else { + found |= IsYardstickStandardAggregate(function.FunctionName().GetIdentifierName()); + } + } + return true; + }); + return found; +} + +Names InputColumns(SelectNode &select) { + Names result; + if (!select.from_table || select.from_table->type == TableReferenceType::EMPTY_FROM) { + return result; + } + auto context = CurrentNativeYardstickClientContext(); + if (!context) { + throw BinderException("Measure window binding requires the originating bind context"); + } + auto probe = select.Copy(); + auto &layout = probe->Cast(); + layout.select_list.clear(); + layout.select_list.push_back(make_uniq()); + layout.groups = GroupByNode(); + layout.having.reset(); + layout.qualify.reset(); + layout.where_clause.reset(); + layout.modifiers.clear(); + layout.aggregate_handling = AggregateHandling::STANDARD_HANDLING; + auto bound = BindNativeYardstickProbe(*probe); + for (auto &name : bound.names) { + result.insert(Key(name.GetIdentifierName())); + } + return result; +} + +void BaseQualifiers(const TableRef &table, Names &names) { + if (table.type == TableReferenceType::BASE_TABLE && table.alias.empty()) { + names.insert(Key(table.Cast().Table().GetIdentifierName())); + } else if (table.type == TableReferenceType::JOIN) { + auto &join = table.Cast(); + BaseQualifiers(*join.left, names); + BaseQualifiers(*join.right, names); + } +} + +bool IsGrouped(SelectNode &select) { + if (!select.groups.group_expressions.empty() || !select.groups.grouping_sets.empty() || + select.aggregate_handling == AggregateHandling::FORCE_AGGREGATES) { + return true; + } + for (auto &expression : select.select_list) { + if (HasAggregate(expression)) { + return true; + } + } + return select.having && HasAggregate(select.having); +} + +void ExpandStars(SelectNode &select) { + auto contains_star = [](unique_ptr &expression) { + bool found = false; + Walk(expression, [&](unique_ptr &node) { + if (node->GetExpressionClass() == ExpressionClass::WINDOW || + (node->GetExpressionClass() == ExpressionClass::FUNCTION && + IsYardstickStandardAggregate(node->Cast().FunctionName().GetIdentifierName()))) { + return false; + } + found |= node->GetExpressionClass() == ExpressionClass::STAR; + return true; + }); + return found; + }; + bool has_star = false; + for (auto &projection : select.select_list) { + has_star |= contains_star(projection); + } + if (!has_star) { + return; + } + auto context = CurrentNativeYardstickClientContext(); + if (!context) { + throw BinderException("Measure window star expansion requires the originating bind context"); + } + auto probe = select.Copy(); + auto &layout = probe->Cast(); + layout.groups = GroupByNode(); + layout.having.reset(); + layout.qualify.reset(); + layout.where_clause.reset(); + layout.modifiers.clear(); + layout.aggregate_handling = AggregateHandling::STANDARD_HANDLING; + string prefix = "__ys_star_projection_"; + auto source_sql = select.ToString(); + while (source_sql.find(prefix) != string::npos) { + prefix += "_"; + } + std::unordered_map placeholders; + for (idx_t i = 0; i < layout.select_list.size(); i++) { + if (!contains_star(layout.select_list[i])) { + auto name = prefix + std::to_string(i); + auto placeholder = ConstantExpression::FromValue(Value()); + placeholder->SetAlias(Identifier(name)); + layout.select_list[i] = std::move(placeholder); + placeholders.emplace(name, i); + } + } + auto bound = BindNativeYardstickProbe(*probe); + Names base_qualifiers; + if (select.from_table) { + BaseQualifiers(*select.from_table, base_qualifiers); + } + vector> expanded; + for (auto &projection : bound.extra_info.original_expressions) { + auto found = placeholders.find(projection->GetAlias().GetIdentifierName()); + if (projection->GetExpressionClass() == ExpressionClass::CONSTANT && found != placeholders.end()) { + expanded.push_back(std::move(select.select_list[found->second])); + } else { + Walk(projection, [&](unique_ptr &node) { + if (node->GetExpressionClass() != ExpressionClass::COLUMN_REF) { + return true; + } + auto &names = node->Cast().ColumnNamesMutable(); + if (names.size() < 2) { + return false; + } + auto qualifier = names[names.size() - 2].GetIdentifierName(); + string visible_name; + auto qualifier_key = Key(qualifier); + for (auto &candidate : base_qualifiers) { + if ((qualifier_key == candidate || StringUtil::EndsWith(qualifier_key, "." + candidate)) && + candidate.size() > visible_name.size()) { + visible_name = candidate; + } + } + if (!visible_name.empty()) { + auto column = names.back(); + names = {Identifier(visible_name), std::move(column)}; + } + return false; + }); + expanded.push_back(std::move(projection)); + } + } + if (expanded.size() != bound.names.size()) { + throw BinderException("Unable to expand measure window projection columns"); + } + select.select_list = std::move(expanded); +} + +unique_ptr Relation(const string &name) { + auto relation = make_uniq(); + relation->SetTable(Identifier(name)); + return std::move(relation); +} + +unique_ptr Subquery(unique_ptr query, const string &alias) { + auto statement = make_uniq(); + statement->node = std::move(query); + return make_uniq(std::move(statement), Identifier(alias)); +} + +struct SourcePlan { + const MeasureWindowSource *spec; + string base_name; + string lineage_name; + string id_name; + unique_ptr base; + unique_ptr view; + vector> columns; + std::unordered_map column_index; + ExpressionMap dimensions; + Names qualifiers; + Names input_columns; +}; + +struct ReferenceScope { + bool unqualified_is_outer = true; + Names local_qualifiers; + Names local_columns; + Names aliases; + Names lambda_parameters; + vector cte_scopes; +}; + +void RelationQualifiers(const TableRef &table, Names &names) { + if (!table.alias.empty()) { + names.insert(Key(table.alias.GetIdentifierName())); + } else if (table.type == TableReferenceType::BASE_TABLE) { + names.insert(Key(table.Cast().Table().GetIdentifierName())); + } else if (table.type == TableReferenceType::TABLE_FUNCTION) { + auto &function = table.Cast().function; + if (function->GetExpressionClass() == ExpressionClass::FUNCTION) { + names.insert(Key(function->Cast().FunctionName().GetIdentifierName())); + } + } + if (table.type == TableReferenceType::JOIN) { + auto &join = table.Cast(); + RelationQualifiers(*join.left, names); + RelationQualifiers(*join.right, names); + } +} + +// Only references captured from the surrounding base-row scope are changed. +// Lambda parameters and nested query relations retain their own namespaces. +class CapturedReferences { +public: + using Visitor = std::function &)>; + + CapturedReferences(const Names &qualifiers_p, Visitor visitor_p) + : qualifiers(qualifiers_p), visitor(std::move(visitor_p)) { + } + + void Expression(unique_ptr &expression, ReferenceScope scope = {}) { + if (!expression) { + return; + } + if (expression->GetExpressionClass() == ExpressionClass::COLUMN_REF) { + auto &names = expression->Cast().ColumnNames(); + auto first = Key(names[0].GetIdentifierName()); + if (scope.lambda_parameters.count(first)) { + return; + } + bool captured = names.size() == 1 + ? scope.unqualified_is_outer && !scope.aliases.count(first) && !scope.local_columns.count(first) + : (scope.unqualified_is_outer || qualifiers.count(first)) && !scope.local_qualifiers.count(first); + if (captured) { + visitor(expression); + } + return; + } + if (expression->GetExpressionClass() == ExpressionClass::FUNCTION && + StringUtil::CIEquals(expression->Cast().FunctionName().GetIdentifierName(), "current")) { + return; + } + if (expression->GetExpressionClass() == ExpressionClass::LAMBDA) { + auto &lambda = expression->Cast(); + if (lambda.GetLambdaSyntaxType() == LambdaSyntaxType::LAMBDA_KEYWORD) { + string error; + auto parameters = lambda.ExtractColumnRefExpressions(error); + if (!error.empty()) { + throw ParserException(error); + } + for (auto ¶meter : parameters) { + scope.lambda_parameters.insert(Key(parameter.get().Cast() + .GetColumnName().GetIdentifierName())); + } + Expression(lambda.RightMutable(), std::move(scope)); + return; + } + } + if (expression->GetExpressionClass() == ExpressionClass::SUBQUERY) { + auto &subquery = expression->Cast(); + Expression(subquery.GetChildMutable(), scope); + Query(*subquery.SubqueryMutable()->node, scope); + return; + } + ParsedExpressionIterator::EnumerateChildren(*expression, [&](unique_ptr &child) { + Expression(child, scope); + }); + } + +private: + ReferenceScope WithBindings(TableRef &table, ReferenceScope scope) { + if (table.type != TableReferenceType::EMPTY_FROM) { + auto probe = make_uniq(); + probe->select_list.push_back(make_uniq()); + probe->from_table = table.Copy(); + auto context = CurrentNativeYardstickClientContext(); + if (!context) { + throw BinderException("Measure window correlation requires the originating bind context"); + } + auto bound = BindNativeYardstickProbe(*probe, scope.cte_scopes); + for (auto &name : bound.names) { + scope.local_columns.insert(Key(name.GetIdentifierName())); + } + RelationQualifiers(table, scope.local_qualifiers); + } + return scope; + } + + void Query(QueryNode &query, ReferenceScope scope) { + scope.cte_scopes.push_back(&query); + for (auto &entry : query.cte_map.map) { + if (entry.second->query_node) { + Query(*entry.second->query_node, scope); + } + } + if (query.type == QueryNodeType::SELECT_NODE) { + auto &select = query.Cast(); + if (select.from_table) { + Table(*select.from_table, scope); + scope = WithBindings(*select.from_table, std::move(scope)); + } + for (auto &projection : select.select_list) { + Expression(projection, scope); + if (projection->HasAlias()) { + scope.aliases.insert(Key(projection->GetAlias().GetIdentifierName())); + } + } + for (auto &group : select.groups.group_expressions) { + Expression(group, scope); + } + Expression(select.where_clause, scope); + Expression(select.having, scope); + Expression(select.qualify, scope); + } else if (query.type == QueryNodeType::SET_OPERATION_NODE) { + for (auto &child : query.Cast().children) { + Query(*child, scope); + } + scope.unqualified_is_outer = false; + } else if (query.type == QueryNodeType::RECURSIVE_CTE_NODE) { + auto &cte = query.Cast(); + Query(*cte.left, scope); + Query(*cte.right, scope); + scope.unqualified_is_outer = false; + } + ParsedExpressionIterator::EnumerateQueryNodeModifiers(query, [&](unique_ptr &expression) { + Expression(expression, scope); + }); + } + + void Table(TableRef &table, ReferenceScope scope) { + if (table.type == TableReferenceType::JOIN) { + auto &join = table.Cast(); + Table(*join.left, scope); + Table(*join.right, WithBindings(*join.left, scope)); + Expression(join.condition, WithBindings(table, scope)); + } else if (table.type == TableReferenceType::SUBQUERY) { + Query(*table.Cast().subquery->node, scope); + } else if (table.type == TableReferenceType::EXPRESSION_LIST) { + for (auto &row : table.Cast().values) { + for (auto &expression : row) { + Expression(expression, scope); + } + } + } else if (table.type == TableReferenceType::TABLE_FUNCTION) { + auto &function = table.Cast(); + Expression(function.function, scope); + if (function.subquery) { + Query(*function.subquery->node, scope); + } + } + } + + const Names &qualifiers; + Visitor visitor; +}; + +unique_ptr BaseColumn(const ParsedExpression &expression, const SourcePlan &plan) { + auto result = expression.Copy(); + result->ClearAlias(); + auto &names = result->Cast().ColumnNamesMutable(); + if (names.size() > 1 && StringUtil::CIEquals(names[0].GetIdentifierName(), "_inner")) { + names.erase(names.begin()); + } else if (names.size() == 2 && + (StringUtil::CIEquals(names[0].GetIdentifierName(), plan.spec->relation_name) || + StringUtil::CIEquals(names[0].GetIdentifierName(), plan.spec->alias))) { + names.erase(names.begin()); + } + if (names.size() == 1) { + auto key = Key(names[0].GetIdentifierName()); + auto dimension = plan.dimensions.find(key); + if (!plan.input_columns.count(key) && dimension != plan.dimensions.end()) { + return dimension->second->Copy(); + } + } + return result; +} + +void NormalizeModifierReferences(unique_ptr &expression, const SourcePlan &plan) { + CapturedReferences references(plan.qualifiers, [&](unique_ptr &node) { + auto &column = node->Cast(); + auto match = plan.dimensions.find(Key(column.GetColumnName().GetIdentifierName())); + auto replacement = match == plan.dimensions.end() ? BaseColumn(*node, plan) : match->second->Copy(); + CapturedReferences base_references(plan.qualifiers, [&](unique_ptr &base) { + auto &names = base->Cast().ColumnNamesMutable(); + names.insert(names.begin(), Identifier("_inner")); + }); + base_references.Expression(replacement); + node = std::move(replacement); + }); + references.Expression(expression); +} + +bool IsSourceReference(const ParsedExpression &expression, const SourcePlan &plan, bool call_scope = false) { + auto &names = expression.Cast().ColumnNames(); + if (names.size() > 1) { + auto first = Key(names[0].GetIdentifierName()); + if (call_scope) { + // Decorated source keys carry _inner. A defining FROM alias can be + // reused by a different relation in the consumer's namespace. + return first == "_inner" || first == Key(plan.spec->relation_name) || + first == Key(plan.spec->alias); + } + return plan.qualifiers.count(first) || plan.input_columns.count(first) || plan.dimensions.count(first); + } + auto name = Key(names[0].GetIdentifierName()); + return plan.input_columns.count(name) || plan.dimensions.count(name); +} + +void CollectColumns(unique_ptr &expression, SourcePlan &plan, bool call_scope = false) { + CapturedReferences references(plan.qualifiers, [&](unique_ptr &node) { + if (!IsSourceReference(*node, plan, call_scope)) { + return; + } + auto base = BaseColumn(*node, plan); + auto key = Key(base->ToString()); + if (!plan.column_index.count(key)) { + plan.column_index.emplace(key, plan.columns.size()); + plan.columns.push_back(std::move(base)); + } + }); + references.Expression(expression); +} + +void RebindColumns(unique_ptr &expression, const SourcePlan &plan, + const string &qualifier = "") { + CapturedReferences references(plan.qualifiers, [&](unique_ptr &node) { + auto base = BaseColumn(*node, plan); + auto match = plan.column_index.find(Key(base->ToString())); + if (match != plan.column_index.end()) { + auto alias = node->GetAlias(); + auto name = Identifier("__ys_c" + std::to_string(match->second)); + node = qualifier.empty() ? make_uniq(name) + : make_uniq(name, Identifier(qualifier)); + node->SetAlias(alias); + } + }); + references.Expression(expression); +} + +SourcePlan BuildSource(const MeasureWindowSource &source, const vector &calls, + const std::unordered_map &caller_order_counts, + idx_t index, Names &cte_names, const ParserOptions &options) { + auto consumer_ctes = CurrentNativeYardstickCteBindings(); + // A stored measure's defining query does not inherit consumer CTEs. + NativeYardstickCteBindScope defining_scope(nullptr); + SourcePlan plan; + plan.spec = &source; + auto prefix = "__ys_window_source_" + std::to_string(index); + plan.lineage_name = prefix + "_lineage"; + plan.id_name = prefix + "_id"; + vector view_aliases; + plan.view = Query(source.clean_select_sql, options, &view_aliases); + for (auto &entry : plan.view->cte_map.map) { + cte_names.insert(Key(entry.first.GetIdentifierName())); + } + plan.base_name = prefix + "_base"; + idx_t suffix = 0; + while (!cte_names.insert(Key(plan.base_name)).second) { + plan.base_name = prefix + "_base_" + std::to_string(++suffix); + } + auto &view = plan.view->Cast(); + if (view.from_table) { + RelationQualifiers(*view.from_table, plan.qualifiers); + } + plan.qualifiers.insert("_inner"); + plan.qualifiers.insert(Key(source.relation_name)); + plan.qualifiers.insert(Key(source.alias)); + ExpandStars(view); + auto input_columns = InputColumns(view); + plan.input_columns = input_columns; + auto aliases = Aliases(view); + for (auto &dimension : source.dimensions) { + plan.dimensions.emplace(Key(dimension.first), Expression(dimension.second, options)); + } + // Resolve output aliases before rebinding the defining query to generated + // base columns. Keep names stable even for originally unaliased dimensions. + for (auto &expression : view.select_list) { + auto name = expression->GetName(); + ExpandAliases(expression, aliases, input_columns); + expression->SetAlias(name); + } + for (auto &expression : view.groups.group_expressions) { + ExpandAliases(expression, aliases, input_columns); + } + if (view.having) { + ExpandAliases(view.having, aliases, input_columns); + } + if (view.qualify) { + ExpandAliases(view.qualify, aliases, input_columns); + } + for (auto &modifier : view.modifiers) { + if (modifier->type == ResultModifierType::ORDER_MODIFIER) { + for (auto &order : modifier->Cast().orders) { + auto &expression = order.expression; + bool output_alias = expression->GetExpressionClass() == ExpressionClass::COLUMN_REF && + !expression->Cast().IsQualified() && + aliases.count(Key(expression->Cast().GetColumnName().GetIdentifierName())); + // A bare ORDER BY name chooses an output alias; names inside + // an ORDER BY expression retain normal input precedence. + ExpandAliases(expression, aliases, output_alias ? Names {} : input_columns); + } + } else if (modifier->type == ResultModifierType::DISTINCT_MODIFIER) { + for (auto &expression : modifier->Cast().distinct_on_targets) { + bool output_alias = expression->GetExpressionClass() == ExpressionClass::COLUMN_REF && + !expression->Cast().IsQualified() && + aliases.count(Key(expression->Cast().GetColumnName().GetIdentifierName())); + ExpandAliases(expression, aliases, output_alias ? Names {} : input_columns); + } + } + } + // Legacy metadata stores only explicitly aliased dimensions. Recover the + // complete dimension set from the defining projection for AT contexts. + for (idx_t i = 0; i < view.select_list.size(); i++) { + auto &projection = view.select_list[i]; + if (HasAggregate(projection)) { + continue; + } + bool window = false; + Walk(projection, [&](unique_ptr &node) { + window |= node->GetExpressionClass() == ExpressionClass::WINDOW; + return !window; + }); + if (window) { + continue; + } + bool references_base = false; + CapturedReferences references(plan.qualifiers, [&](unique_ptr &) { references_base = true; }); + references.Expression(projection); + if (references_base) { + auto name = i < view_aliases.size() ? view_aliases[i] : projection->GetName(); + auto expression = projection->Copy(); + expression->ClearAlias(); + plan.dimensions[Key(name.GetIdentifierName())] = std::move(expression); + } + } + SelectExpressions(view, [&](unique_ptr &expression) { CollectColumns(expression, plan); }); + for (auto &dimension : plan.dimensions) { + CollectColumns(dimension.second, plan); + } + for (auto &call : calls) { + NativeYardstickCteBindScope consumer_scope(consumer_ctes); + if (call.source_key != source.key) { + continue; + } + auto expression = Expression(call.expression_sql, options); + auto order_count = caller_order_counts.at(call.marker_name); + if (order_count) { + Walk(expression, [&](unique_ptr &node) { + if (node->GetExpressionClass() != ExpressionClass::FUNCTION) { + return true; + } + auto &function = node->Cast(); + if (function.OrderBy() && function.OrderBy()->orders.size() >= order_count) { + auto &orders = function.OrderByMutable()->orders; + for (idx_t i = 0; i < order_count; i++) { + CollectColumns(orders[i].expression, plan, true); + } + // Only the caller's prefix uses the consumer namespace. + // Declaration arguments, filters and order ties retain the + // defining FROM namespace, even when aliases are reused. + orders.erase(orders.begin(), orders.begin() + order_count); + } + return true; + }); + } + CollectColumns(expression, plan); + for (auto &modifier : call.modifiers) { + if (modifier.type == WindowContextType::WHERE) { + auto condition = ModifierExpression(modifier.value, options); + NormalizeModifierReferences(condition, plan); + CollectColumns(condition, plan); + } + } + } + auto base = make_uniq(); + base->from_table = std::move(view.from_table); + base->where_clause = std::move(view.where_clause); + base->cte_map = std::move(view.cte_map); + base->sample = std::move(view.sample); + auto identity = Expression("row_number() OVER ()", options); + identity->SetAlias(Identifier(plan.id_name)); + base->select_list.push_back(std::move(identity)); + for (idx_t i = 0; i < plan.columns.size(); i++) { + auto column = plan.columns[i]->Copy(); + column->SetAlias(Identifier("__ys_c" + std::to_string(i))); + base->select_list.push_back(std::move(column)); + } + plan.base = std::move(base); + bool grouped = source.grouped || IsGrouped(view); + // Adding provenance to SELECT DISTINCT would otherwise prevent duplicate + // elimination. Equal visible rows form one lineage group instead. + if (!grouped) { + for (auto it = view.modifiers.begin(); it != view.modifiers.end();) { + if ((*it)->type == ResultModifierType::DISTINCT_MODIFIER && + (*it)->Cast().distinct_on_targets.empty()) { + GroupingSet set; + for (idx_t i = 0; i < view.select_list.size(); i++) { + auto group = view.select_list[i]->Copy(); + group->ClearAlias(); + view.groups.group_expressions.push_back(std::move(group)); + set.insert(ProjectionIndex(i)); + } + view.groups.grouping_sets.push_back(std::move(set)); + it = view.modifiers.erase(it); + grouped = true; + } else { + ++it; + } + } + } + SelectExpressions(view, [&](unique_ptr &expression) { RebindColumns(expression, plan); }); + for (idx_t i = 0; i < view_aliases.size() && i < view.select_list.size(); i++) { + view.select_list[i]->SetAlias(view_aliases[i]); + } + view.from_table = Relation(plan.base_name); + auto lineage = Expression((grouped ? "list(" : "list_value(") + Quote(plan.id_name) + ")", options); + lineage->SetAlias(Identifier(plan.lineage_name)); + view.select_list.push_back(std::move(lineage)); + return plan; +} + +bool ReplaceSource(unique_ptr &relation, const SourcePlan &plan) { + if (!relation) { + return false; + } + if (relation->type == TableReferenceType::JOIN) { + auto &join = relation->Cast(); + bool left = ReplaceSource(join.left, plan); + bool right = ReplaceSource(join.right, plan); + return left || right; + } + if (relation->type != TableReferenceType::BASE_TABLE) { + return false; + } + auto &table = relation->Cast(); + auto alias = relation->alias.empty() ? table.Table().GetIdentifierName() : relation->alias.GetIdentifierName(); + auto expected_alias = plan.spec->alias.empty() ? plan.spec->relation_name : plan.spec->alias; + if (!StringUtil::CIEquals(alias, expected_alias)) { + return false; + } + auto replacement = Subquery(plan.view->Copy(), alias); + replacement->sample = std::move(relation->sample); + replacement->column_name_alias = std::move(relation->column_name_alias); + relation = std::move(replacement); + return true; +} + +class WindowRewrite { +public: + WindowRewrite(SelectNode &owner_p, vector &sources_p, + const vector &calls_p, const ParserOptions &options_p, + const Names &input_columns_p) + : owner(owner_p), sources(sources_p), calls(calls_p), options(options_p), grouped(IsGrouped(owner_p)), + aliases(Aliases(owner_p)), input_columns(input_columns_p) { + } + + unique_ptr Run() { + auto output = make_uniq(); + auto projections = std::move(owner.select_list); + auto qualify = std::move(owner.qualify); + output->modifiers = std::move(owner.modifiers); + + if (owner.aggregate_handling == AggregateHandling::FORCE_AGGREGATES) { + // DuckDB's GROUP BY ALL binder cannot infer groups from the new + // list window. Infer them from the user's original projection. + owner.aggregate_handling = AggregateHandling::STANDARD_HANDLING; + owner.groups = GroupByNode(); + GroupingSet grouping; + for (auto &projection : projections) { + bool window = false; + Walk(projection, [&](unique_ptr &node) { + window |= node->GetExpressionClass() == ExpressionClass::WINDOW; + return !window; + }); + if (window || HasAggregate(projection)) { + continue; + } + auto group = projection->Copy(); + group->ClearAlias(); + grouping.insert(ProjectionIndex(owner.groups.group_expressions.size())); + owner.groups.group_expressions.push_back(std::move(group)); + } + owner.groups.grouping_sets.push_back(std::move(grouping)); + } + + for (auto &group : owner.groups.group_expressions) { + // SQL ordinal grouping is relative to the original projection. + if (group->GetExpressionClass() == ExpressionClass::CONSTANT) { + auto text = group->ToString(); + if (!text.empty() && text.find_first_not_of("0123456789") == string::npos) { + auto ordinal = std::stoull(text); + if (ordinal > 0 && ordinal <= projections.size()) { + group = projections[ordinal - 1]->Copy(); + group->ClearAlias(); + } + } + } + ExpandAliases(group, aliases, input_columns); + } + if (owner.having) { + ExpandAliases(owner.having, aliases, input_columns); + } + if (owner.where_clause) { + ExpandAliases(owner.where_clause, aliases, input_columns); + } + for (auto &projection : projections) { + auto name = projection->GetName(); + projection->ClearAlias(); + ExpandAliases(projection, aliases, input_columns); + Rewrite(projection); + projection->SetAlias(name); + output->select_list.push_back(std::move(projection)); + } + if (qualify) { + ExpandAliases(qualify, aliases, input_columns); + Rewrite(qualify); + output->where_clause = std::move(qualify); + } + for (auto &modifier : output->modifiers) { + if (modifier->type == ResultModifierType::ORDER_MODIFIER) { + for (auto &order : modifier->Cast().orders) { + RewriteModifier(order.expression); + } + } else if (modifier->type == ResultModifierType::DISTINCT_MODIFIER) { + for (auto &target : modifier->Cast().distinct_on_targets) { + RewriteModifier(target); + } + } + } + output->cte_map = std::move(owner.cte_map); + output->from_table = Subquery(owner.Copy(), "__ys_window_stage"); + return std::move(output); + } + +private: + const MeasureWindowCall *Call(const ParsedExpression &expression) const { + if (expression.GetExpressionClass() != ExpressionClass::WINDOW) { + return nullptr; + } + auto &window = expression.Cast(); + for (auto &call : calls) { + if (StringUtil::CIEquals(window.FunctionName().GetIdentifierName(), call.marker_name)) { + return &call; + } + } + return nullptr; + } + + bool ContainsCall(unique_ptr &expression) const { + bool found = false; + Walk(expression, [&](unique_ptr &node) { + found = found || Call(*node); + return !found; + }); + return found; + } + + string Project(unique_ptr expression) { + auto name = "__ys_window_value_" + std::to_string(owner.select_list.size()); + expression->SetAlias(Identifier(name)); + owner.select_list.push_back(std::move(expression)); + return name; + } + + void RewriteModifier(unique_ptr &expression) { + // Output aliases and ordinal references are resolved by the final query. + if (expression->GetExpressionClass() == ExpressionClass::CONSTANT) { + return; + } + if (expression->GetExpressionClass() == ExpressionClass::COLUMN_REF) { + auto &column = expression->Cast(); + if (!column.IsQualified() && aliases.count(Key(column.GetColumnName().GetIdentifierName()))) { + return; + } + } + ExpandAliases(expression, aliases, input_columns); + Rewrite(expression); + } + + string DimensionSQL(const SourcePlan &source, const string &dimension, const string &qualifier) const { + auto expression = Expression(dimension, options); + if (expression->GetExpressionClass() == ExpressionClass::COLUMN_REF) { + auto key = Key(expression->Cast().GetColumnName().GetIdentifierName()); + auto match = source.dimensions.find(key); + if (match != source.dimensions.end()) { + expression = match->second->Copy(); + } + } + RebindColumns(expression, source, qualifier); + return expression->ToString(); + } + + string Context(const SourcePlan &source, const MeasureWindowCall &call, const string &frame, + const string &visible_frame, const string &payload_ids = "") const { + const string candidate = "__ys_candidate"; + const string selected = "__ys_selected"; + auto frame_ids = payload_ids.empty() ? "SELECT unnest(flatten(" + Quote(frame) + "))" : payload_ids; + auto membership = candidate + "." + Quote(source.id_name) + " IN (" + frame_ids + ")"; + std::unordered_set removed; + struct SetOverride { + string dimension; + string value; + }; + std::unordered_map sets; + string condition; + bool global = false; + bool expand = call.modifiers.size() > 1; + bool has_set = false; + for (auto &modifier : call.modifiers) { + has_set |= modifier.type == WindowContextType::SET; + } + bool visible = false; + auto dimension_key = [&](const string &dimension) { + auto expression = Expression(dimension, options); + if (expression->GetExpressionClass() == ExpressionClass::COLUMN_REF) { + return Key(expression->Cast().GetColumnName().GetIdentifierName()); + } + return Key(expression->ToString()); + }; + auto resolve_current = [&](unique_ptr &expression, bool bare_dimensions) { + Walk(expression, [&](unique_ptr &node) { + string dimension; + if (node->GetExpressionClass() == ExpressionClass::FUNCTION) { + auto &function = node->Cast(); + if (StringUtil::CIEquals(function.FunctionName().GetIdentifierName(), "current") && + function.GetArguments().size() == 1) { + dimension = function.GetArguments()[0].GetExpression().ToString(); + } + } else if (bare_dimensions && node->GetExpressionClass() == ExpressionClass::COLUMN_REF) { + auto &column = node->Cast(); + if (source.dimensions.count(Key(column.GetColumnName().GetIdentifierName()))) { + dimension = node->ToString(); + } + } + if (dimension.empty()) { + return true; + } + auto dim = DimensionSQL(source, dimension, selected); + node = Expression("(SELECT CASE WHEN count(DISTINCT " + dim + + ") + max(CASE WHEN " + dim + " IS NULL THEN 1 ELSE 0 END) = 1 THEN first(" + dim + + ") ELSE NULL END FROM " + Quote(source.base_name) + " " + selected + " WHERE " + + selected + "." + Quote(source.id_name) + " IN (" + frame_ids + "))", options); + return false; + }); + }; + for (auto it = call.modifiers.rbegin(); it != call.modifiers.rend(); ++it) { + auto &modifier = *it; + switch (modifier.type) { + case WindowContextType::ALL_GLOBAL: + global = true; + condition.clear(); + visible = false; + removed.clear(); + sets.clear(); + break; + case WindowContextType::ALL: + removed.insert(dimension_key(modifier.dimension)); + expand = true; + break; + case WindowContextType::SET: + if (!global && !removed.count(dimension_key(modifier.dimension))) { + sets[dimension_key(modifier.dimension)] = {modifier.dimension, modifier.value}; + expand = true; + } + break; + case WindowContextType::WHERE: + if (!global) { + auto expression = ModifierExpression(modifier.value, options); + NormalizeModifierReferences(expression, source); + RebindColumns(expression, source, candidate); + resolve_current(expression, false); + condition = expression->ToString(); + visible = false; + } + break; + case WindowContextType::VISIBLE: + if (!global && !has_set) { + condition.clear(); + visible = true; + } + break; + } + } + if (global) { + return "true"; + } + if (!condition.empty() && call.modifiers.size() == 1) { + return condition; + } + if (!expand) { + return membership; + } + vector correlations; + for (auto &dimension : source.dimensions) { + auto base_expression = dimension.second->ToString(); + bool replaced = false; + for (auto &entry : sets) { + replaced |= entry.first == dimension_key(dimension.first) || + entry.first == dimension_key(base_expression); + } + if (!removed.count(dimension_key(dimension.first)) && + !removed.count(dimension_key(base_expression)) && !replaced) { + correlations.push_back(DimensionSQL(source, dimension.first, candidate) + + " IS NOT DISTINCT FROM " + DimensionSQL(source, dimension.first, selected)); + } + } + bool needs_selected_row = !correlations.empty(); + for (auto &entry : sets) { + auto value = ModifierExpression(entry.second.value, options); + // CURRENT values use the single value of the frame dimension. A + // multi-valued or empty frame has NULL as its current value. + resolve_current(value, true); + correlations.push_back(DimensionSQL(source, entry.second.dimension, candidate) + " IS NOT DISTINCT FROM " + + value->ToString()); + } + if (!condition.empty()) { + correlations.push_back(condition); + } + if (visible && !visible_frame.empty()) { + correlations.push_back(candidate + "." + Quote(source.id_name) + + " IN (SELECT unnest(flatten(" + Quote(visible_frame) + ")))"); + } + if (!needs_selected_row) { + return correlations.empty() ? "true" : StringUtil::Join(correlations, " AND "); + } + correlations.push_back(selected + "." + Quote(source.id_name) + " IN (" + frame_ids + ")"); + return "EXISTS (SELECT 1 FROM " + Quote(source.base_name) + " " + selected + " WHERE " + + StringUtil::Join(correlations, " AND ") + ")"; + } + + void Rewrite(unique_ptr &expression) { + if (auto call = Call(*expression)) { + SourcePlan *source = nullptr; + for (auto &candidate : sources) { + if (candidate.spec->key == call->source_key) { + source = &candidate; + break; + } + } + if (!source) { + throw InternalException("Missing source for measure window %s", call->marker_name); + } + auto window = expression->Copy(); + auto &list = window->Cast(); + auto measure = Expression(call->expression_sql, options); + const auto order_count = list.ArgOrders().size(); + vector> caller_keys; + Walk(measure, [&](unique_ptr &node) { + if (!caller_keys.empty() || node->GetExpressionClass() != ExpressionClass::FUNCTION) { + return caller_keys.empty(); + } + auto &function = node->Cast(); + if (order_count && function.OrderBy() && function.OrderBy()->orders.size() >= order_count) { + for (idx_t i = 0; i < order_count; i++) { + caller_keys.push_back(function.OrderBy()->orders[i].expression->Copy()); + } + return false; + } + return true; + }); + vector external_keys; + vector payload_fields; + bool external_order = false; + for (idx_t i = 0; i < caller_keys.size(); i++) { + bool external = false; + CapturedReferences references(source->qualifiers, [&](unique_ptr &node) { + if (IsSourceReference(*node, *source, true)) { + return; + } + external = true; + auto field = "__ys_ext_" + std::to_string(payload_fields.size()); + auto input = node->Copy(); + ExpandAliases(input, aliases, input_columns); + payload_fields.push_back(Quote(field) + " := " + input->ToString()); + node = Expression("__ys_payload." + Quote(field), options); + }); + // Grouped consumer keys are evaluated at the window input grain. + // Source-only keys instead remain expressions over original rows. + auto raw_key = list.ArgOrders()[i].expression->Copy(); + ExpandAliases(raw_key, aliases, input_columns); + if (HasAggregate(raw_key)) { + external = true; + auto field = "__ys_ext_" + std::to_string(payload_fields.size()); + payload_fields.push_back(Quote(field) + " := " + raw_key->ToString()); + caller_keys[i] = Expression("__ys_payload." + Quote(field), options); + } else { + references.Expression(caller_keys[i]); + } + external_keys.push_back(external); + external_order |= external; + } + vector occurrence_orders; + for (idx_t i = 0; i < caller_keys.size(); i++) { + auto &original = list.ArgOrders()[i]; + OrderByNode order(original.type, original.null_order, + Expression("__ys_key_" + std::to_string(i), options)); + occurrence_orders.push_back(order.ToString()); + } + list.SetFunctionName("list"); + list.DistinctMutable() = false; + list.ArgOrdersMutable().clear(); + list.GetArgumentsMutable().clear(); + auto alias = source->spec->alias.empty() ? source->spec->relation_name : source->spec->alias; + auto lineage_sql = Quote(alias) + "." + Quote(source->lineage_name); + if (grouped) { + lineage_sql = "flatten(list(" + lineage_sql + "))"; + } + auto frame_value = lineage_sql; + if (external_order) { + frame_value = "struct_pack(ids := " + lineage_sql + ", " + + StringUtil::Join(payload_fields, ", ") + ")"; + } + list.GetArgumentsMutable().emplace_back(Expression(frame_value, options)); + if (list.Filter()) { + ExpandAliases(list.FilterMutable(), aliases, input_columns); + } + for (auto &partition : list.PartitionsMutable()) { + ExpandAliases(partition, aliases, input_columns); + } + for (auto &order : list.OrderByMutable()) { + ExpandAliases(order.expression, aliases, input_columns); + } + if (list.StartExpr()) { + ExpandAliases(list.StartExprMutable(), aliases, input_columns); + } + if (list.EndExpr()) { + ExpandAliases(list.EndExprMutable(), aliases, input_columns); + } + auto frame = Project(std::move(window)); + string visible_frame; + for (auto &modifier : call->modifiers) { + if (modifier.type == WindowContextType::VISIBLE) { + auto existing = visible_frames.find(source->spec->key); + if (existing == visible_frames.end()) { + visible_frame = Project(Expression("list(" + lineage_sql + ") OVER ()", options)); + visible_frames.emplace(source->spec->key, visible_frame); + } else { + visible_frame = existing->second; + } + break; + } + } + string order_cte; + string order_join; + string frame_ids; + if (external_order) { + frame_ids = "SELECT unnest(__ys_payload.ids) FROM unnest(" + Quote(frame) + + ") __ys_frames(__ys_payload)"; + vector key_projections; + for (idx_t i = 0; i < caller_keys.size(); i++) { + RebindColumns(caller_keys[i], *source, "__ys_candidate"); + key_projections.push_back(caller_keys[i]->ToString() + " AS __ys_key_" + std::to_string(i)); + } + order_cte = "WITH __ys_occurrences AS MATERIALIZED (SELECT __ys_candidate." + + Quote(source->id_name) + ", " + StringUtil::Join(key_projections, ", ") + + " FROM unnest(" + Quote(frame) + ") __ys_frames(__ys_payload), " + + "unnest(__ys_payload.ids) __ys_ids(id) JOIN " + Quote(source->base_name) + + " __ys_candidate ON __ys_candidate." + Quote(source->id_name) + " = __ys_ids.id), " + + "__ys_order_keys AS (SELECT * FROM __ys_occurrences QUALIFY row_number() OVER " + + "(PARTITION BY " + Quote(source->id_name) + " ORDER BY " + + StringUtil::Join(occurrence_orders, ", ") + ") = 1) "; + // Context expansion can introduce rows without frame occurrences. + // Their joined-input keys are NULL; source keys still recompute. + order_join = " LEFT JOIN __ys_order_keys USING (" + Quote(source->id_name) + ")"; + Walk(measure, [&](unique_ptr &node) { + if (node->GetExpressionClass() != ExpressionClass::FUNCTION) { + return true; + } + auto &function = node->Cast(); + if (function.OrderBy() && function.OrderBy()->orders.size() >= order_count) { + for (idx_t i = 0; i < order_count; i++) { + if (external_keys[i]) { + function.OrderByMutable()->orders[i].expression = + Expression("__ys_order_keys.__ys_key_" + std::to_string(i), options); + } + } + } + return true; + }); + } + RebindColumns(measure, *source, "__ys_candidate"); + expression = Expression("(" + order_cte + "SELECT " + measure->ToString() + " FROM " + + Quote(source->base_name) + " __ys_candidate" + order_join + " WHERE " + + Context(*source, *call, frame, visible_frame, frame_ids) + ")", options); + return; + } + if (!ContainsCall(expression)) { + auto name = Project(std::move(expression)); + expression = make_uniq(Identifier(name)); + return; + } + ParsedExpressionIterator::EnumerateChildren(*expression, [&](unique_ptr &child) { + Rewrite(child); + }); + } + + SelectNode &owner; + vector &sources; + const vector &calls; + const ParserOptions &options; + bool grouped; + ExpressionMap aliases; + Names input_columns; + std::unordered_map visible_frames; +}; + +} // namespace + +string RewriteNativeMeasureWindows(const string &scope_sql, const vector &sources, + const vector &calls, const vector &visible_ctes, + const vector &binding_ctes, const ParserOptions &options) { + if (calls.empty()) { + return scope_sql; + } + NativeYardstickCteBindScope binding_scope(binding_ctes, options); + auto query = Query(scope_sql, options); + auto &owner = query->Cast(); + ExpandStars(owner); + auto input_columns = InputColumns(owner); + std::unordered_map caller_order_counts; + for (auto &call : calls) { + caller_order_counts.emplace(call.marker_name, 0); + } + SelectExpressions(owner, [&](unique_ptr &expression) { + Walk(expression, [&](unique_ptr &node) { + if (node->GetExpressionClass() == ExpressionClass::WINDOW) { + auto &window = node->Cast(); + auto entry = caller_order_counts.find(window.FunctionName().GetIdentifierName()); + if (entry != caller_order_counts.end()) { + entry->second = window.ArgOrders().size(); + } + } + return true; + }); + }); + vector plans; + Names cte_names; + for (auto &name : visible_ctes) { + cte_names.insert(Key(name)); + } + for (auto &entry : owner.cte_map.map) { + cte_names.insert(Key(entry.first.GetIdentifierName())); + } + for (idx_t i = 0; i < sources.size(); i++) { + plans.push_back(BuildSource(sources[i], calls, caller_order_counts, i, cte_names, options)); + } + for (auto &plan : plans) { + if (!ReplaceSource(owner.from_table, plan)) { + throw BinderException("Cannot locate measure window source %s in the query", plan.spec->relation_name); + } + } + WindowRewrite rewrite(owner, plans, calls, options, input_columns); + auto output = rewrite.Run(); + for (auto &plan : plans) { + auto cte = make_uniq(); + cte->query_node = std::move(plan.base); + cte->materialized = CTEMaterialize::CTE_MATERIALIZE_ALWAYS; + output->cte_map.map.insert(Identifier(plan.base_name), std::move(cte)); + } + if (!visible_ctes.empty()) { + // This SELECT is spliced into a statement with an enclosing WITH. + // Keep the generated WITH inside a subquery rather than emitting two + // adjacent WITH clauses at the same query level. + auto wrapper = make_uniq(); + wrapper->select_list.push_back(make_uniq()); + wrapper->from_table = Subquery(std::move(output), "__ys_window_result"); + return wrapper->ToString(); + } + return output->ToString(); +} + +} // namespace duckdb +#endif diff --git a/src/yardstick_extension.cpp b/src/yardstick_extension.cpp index 645f029..8de3f2b 100644 --- a/src/yardstick_extension.cpp +++ b/src/yardstick_extension.cpp @@ -3,6 +3,7 @@ #include "yardstick_extension.hpp" #include "yardstick_parser_extension.hpp" #include "frontend_peg.hpp" +#include "aggregate_state.hpp" #include "duckdb/parser/parser.hpp" #include "duckdb/parser/parser_extension.hpp" #include "duckdb/parser/statement/extension_statement.hpp" @@ -38,7 +39,7 @@ extern "C" { void yardstick_free_create_view_info(YardstickCreateViewInfo* info); char* yardstick_replace_range(const char* sql, uint32_t start, uint32_t end, const char* replacement); char* yardstick_apply_replacements(const char* sql, const YardstickReplacement* replacements, size_t count); - char* yardstick_qualify_expression(const char* expr, const char* qualifier); + char* yardstick_qualify_expression(const char* expr, const char* qualifier, const char* dimension); void yardstick_free_string(char* ptr); char* yardstick_expand_aggregate_call( const char* measure_name, @@ -89,7 +90,7 @@ extern "C" { void (*free_create_view_info)(YardstickCreateViewInfo*), char* (*replace_range)(const char*, uint32_t, uint32_t, const char*), char* (*apply_replacements)(const char*, const YardstickReplacement*, size_t), - char* (*qualify_expression)(const char*, const char*), + char* (*qualify_expression)(const char*, const char*, const char*), char* (*inline_order_by_subquery_aliases)(const char*), void (*free_string)(char*), char* (*expand_aggregate_call)(const char*, const char*, const YardstickAtModifier*, size_t, const char*, const char*, const char*, const char* const*, size_t), @@ -99,7 +100,13 @@ extern "C" { int32_t (*expressions_equal)(const char*, const char*), YardstickQueryScopeList* (*find_query_scopes)(const char*), void (*free_query_scopes)(YardstickQueryScopeList*), - char* (*rewrite_visible_filter)(const char*, const char*, const char* const*, const char* const*, size_t, char**) + char* (*rewrite_visible_filter)(const char*, const char*, const char* const*, const char* const*, size_t, char**), + char* (*decorate_measure)(const char*, const char*, const char* const*, const char* const*, size_t, + const char* const*, size_t, const char* const*, size_t, char**), + char* (*window_marker)(const char*, const char*, char**), + char* (*rewrite_measure_windows)(const char*, const YardstickWindowSource*, size_t, + const YardstickWindowCall*, size_t, const char* const*, size_t, + const char* const*, size_t, char**) ); } @@ -145,12 +152,23 @@ struct YardstickQueryData : public TableFunctionData { bool done = false; }; +#if YARDSTICK_GRAMMAR_EXTENSION +struct DeferredMeasureFunctionInfo : public TableFunctionInfo { + explicit DeferredMeasureFunctionInfo(shared_ptr parser_info_p) + : parser_info(std::move(parser_info_p)) { + } + shared_ptr parser_info; +}; +#endif + static unique_ptr YardstickQueryBind(ClientContext &context, TableFunctionBindInput &input, vector &return_types, vector &names) { #if YARDSTICK_GRAMMAR_EXTENSION - NativeYardstickParseScope native_scope(nullptr, context.GetParserOptions()); + NativeYardstickBindScope bind_scope(context); + auto &info = input.info->Cast(); + NativeYardstickParseScope native_scope(info.parser_info.get(), context.GetParserOptions()); #endif auto data = make_uniq(); data->original_sql = input.inputs[0].GetValue(); @@ -1795,6 +1813,32 @@ static vector> DeferMeasureColumnListBatch(const string if (StartsWithSemantic(sql, semantic_stripped)) { sql = std::move(semantic_stripped); } + auto inspect_calls = [&](const string &scope_sql) { + bool inspect_nested_scopes = false; + if (auto *calls = FindNativeYardstickAggregates(scope_sql.c_str())) { + for (size_t index = 0; index < calls->count; index++) { + // Decorations need the originating catalog for aggregate + // classification and star expansion before adding lineage. + requires_binding |= calls->calls[index].has_decorations; + inspect_nested_scopes |= calls->calls[index].modifier_count != 0; + } + yardstick_free_aggregate_list(calls); + } else { + inspect_nested_scopes = true; + } + return inspect_nested_scopes; + }; + if (inspect_calls(sql) && !requires_binding) { + // AT modifier expressions can own nested queries that are absent + // from the enclosing call's marker AST. Inspect their native spans. + if (auto *scopes = FindNativeYardstickQueryScopes(sql.c_str())) { + for (size_t index = 0; index < scopes->count && !requires_binding; index++) { + auto &scope = scopes->scopes[index]; + inspect_calls(sql.substr(scope.start_pos, scope.end_pos - scope.start_pos)); + } + yardstick_free_query_scopes(scopes); + } + } if (!StartsWithCreateViewStatement(sql)) { continue; } @@ -2236,13 +2280,6 @@ static BoundStatement BindDeferredMeasureStatement(ClientContext &context, Binde } } -struct DeferredMeasureFunctionInfo : public TableFunctionInfo { - explicit DeferredMeasureFunctionInfo(shared_ptr parser_info_p) - : parser_info(std::move(parser_info_p)) { - } - shared_ptr parser_info; -}; - static unique_ptr DeferredMeasureSelectBindReplace(ClientContext &context, TableFunctionBindInput &input) { auto &info = input.info->Cast(); NativeYardstickBindScope bind_scope(context); @@ -2360,7 +2397,8 @@ BoundStatement yardstick_bind(ClientContext &context, Binder &binder, //============================================================================= static void LoadInternal(ExtensionLoader &loader) { - // Initialize parser FFI function pointers in Rust (must be done first) + RegisterYardstickAggregateStateFunctions(loader); + // Initialize parser FFI function pointers before registering query entry points. yardstick_init_parser_ffi( yardstick_find_aggregates, yardstick_free_aggregate_list, @@ -2382,7 +2420,10 @@ static void LoadInternal(ExtensionLoader &loader) { yardstick_expressions_equal, yardstick_find_query_scopes, yardstick_free_query_scopes, - yardstick_rewrite_visible_filter + yardstick_rewrite_visible_filter, + yardstick_decorate_measure, + yardstick_window_marker, + yardstick_rewrite_measure_windows ); auto &db = loader.GetDatabaseInstance(); @@ -2413,9 +2454,15 @@ static void LoadInternal(ExtensionLoader &loader) { // Register table function for AGGREGATE() expansion TableFunction query_func("yardstick", {LogicalType::VARCHAR}, YardstickQueryFunction, YardstickQueryBind); +#if YARDSTICK_GRAMMAR_EXTENSION + query_func.function_info = make_shared_ptr(parser.parser_info); +#endif loader.RegisterFunction(query_func); TableFunction query_func_with_warnings("yardstick", {LogicalType::VARCHAR, LogicalType::VARCHAR}, YardstickQueryFunction, YardstickQueryBind); +#if YARDSTICK_GRAMMAR_EXTENSION + query_func_with_warnings.function_info = make_shared_ptr(parser.parser_info); +#endif loader.RegisterFunction(query_func_with_warnings); #if YARDSTICK_GRAMMAR_EXTENSION diff --git a/src/yardstick_parser_ffi.cpp b/src/yardstick_parser_ffi.cpp index 4541250..06d5d2b 100644 --- a/src/yardstick_parser_ffi.cpp +++ b/src/yardstick_parser_ffi.cpp @@ -10,6 +10,8 @@ #include "yardstick_ffi.h" #include "yardstick_compat.hpp" #include "frontend_peg.hpp" +#include "aggregate_decorations.hpp" +#include "measure_windows.hpp" #if YARDSTICK_GRAMMAR_EXTENSION #include "duckdb/parser/peg/compiled_grammar.hpp" #include "duckdb/parser/statement/insert_statement.hpp" @@ -1290,7 +1292,7 @@ extern "C" int32_t yardstick_current_where_is_single_valued(const char* predicat const char* qualifier) { #if YARDSTICK_GRAMMAR_EXTENSION auto *options = CurrentNativeYardstickParserOptions(); - if (!options || !options->compiled_grammar || !options->compiled_grammar->HasGrammarChanges()) { + if (!options || !options->compiled_grammar || !options->compiled_grammar->GetRule("YardstickAtModifier")) { return -1; } try { @@ -1335,6 +1337,7 @@ extern "C" void yardstick_free_query_scopes(YardstickQueryScopeList* list) { free(const_cast(list->scopes[i].visible_ctes[j])); } delete[] list->scopes[i].visible_ctes; + delete[] list->scopes[i].cte_definitions; } delete[] list->scopes; delete list; @@ -1415,7 +1418,7 @@ extern "C" YardstickAggregateCallList* yardstick_find_aggregates(const char* sql // Convert to C structs if (!aggregates.empty()) { result->count = aggregates.size(); - result->calls = new YardstickAggregateCall[result->count]; + result->calls = new YardstickAggregateCall[result->count] {}; for (size_t i = 0; i < aggregates.size(); i++) { auto& info = aggregates[i]; @@ -1455,6 +1458,7 @@ extern "C" void yardstick_free_aggregate_list(YardstickAggregateCallList* list) for (size_t i = 0; i < list->count; i++) { auto& call = list->calls[i]; free(const_cast(call.measure_name)); + free(const_cast(call.call_sql)); for (size_t j = 0; j < call.modifier_count; j++) { free(const_cast(call.modifiers[j].dimension)); free(const_cast(call.modifiers[j].value)); @@ -1466,6 +1470,125 @@ extern "C" void yardstick_free_aggregate_list(YardstickAggregateCallList* list) delete list; } +extern "C" char* yardstick_decorate_measure( + const char* expression, const char* call_sql, const char* const* dimension_names, + const char* const* dimension_expressions, size_t dimension_count, + const char* const* qualifiers, size_t qualifier_count, + const char* const* binding_ctes, size_t binding_cte_count, char** error) { + if (error) *error = nullptr; +#if YARDSTICK_GRAMMAR_EXTENSION + try { + vector> dimensions; + for (size_t i = 0; i < dimension_count; i++) { + dimensions.emplace_back(dimension_names[i], dimension_expressions[i]); + } + vector local_qualifiers; + for (size_t i = 0; i < qualifier_count; i++) local_qualifiers.emplace_back(qualifiers[i]); + auto options = YardstickParserOptions(); + vector cte_queries; + for (size_t i = 0; i < binding_cte_count; i++) cte_queries.emplace_back(binding_ctes[i]); + NativeYardstickCteBindScope binding_scope(cte_queries, options); + auto calls = Parser::ParseExpressionList(call_sql, options); + if (calls.size() != 1) throw ParserException("Expected one AGGREGATE call"); + // Window FILTER selects input rows in the lineage stage. DISTINCT and + // argument ordering apply to the aggregate leaves being recomputed. + const bool is_window = calls[0]->GetExpressionClass() == ExpressionClass::WINDOW; + if (is_window) { + auto &window = calls[0]->Cast(); + vector> arguments; + for (auto &argument : window.GetArguments()) arguments.push_back(argument.GetExpression().Copy()); + auto orders = make_uniq(); + for (auto &order : window.ArgOrders()) { + orders->orders.emplace_back(order.type, order.null_order, order.expression->Copy()); + } + calls[0] = make_uniq(Identifier("aggregate"), std::move(arguments), + nullptr, std::move(orders), window.Distinct()); + } + return safe_strdup(DecorateYardstickMeasureExpression( + expression, calls[0]->ToString(), dimensions, local_qualifiers, options, is_window)); + } catch (const std::exception &exception) { + if (error) *error = safe_strdup(exception.what()); + } +#endif + return nullptr; +} + +extern "C" char* yardstick_window_marker(const char* call_sql, const char* marker_name, char** error) { + if (error) *error = nullptr; +#if YARDSTICK_GRAMMAR_EXTENSION + try { + auto expressions = Parser::ParseExpressionList(call_sql, YardstickParserOptions()); + if (expressions.size() != 1 || expressions[0]->GetExpressionClass() != ExpressionClass::WINDOW) { + throw ParserException("Expected one windowed AGGREGATE call"); + } + auto &window = expressions[0]->Cast(); + window.SetFunctionName(marker_name); + window.GetArgumentsMutable().clear(); + window.GetArgumentsMutable().emplace_back(ConstantExpression::FromValue(Value::INTEGER(0))); + window.DistinctMutable() = false; + return safe_strdup(window.ToString()); + } catch (const std::exception &exception) { + if (error) *error = safe_strdup(exception.what()); + } +#endif + return nullptr; +} + +extern "C" char* yardstick_rewrite_measure_windows( + const char* sql, const YardstickWindowSource* sources, size_t source_count, + const YardstickWindowCall* calls, size_t call_count, + const char* const* visible_ctes, size_t visible_cte_count, + const char* const* binding_ctes, size_t binding_cte_count, char** error) { + if (error) *error = nullptr; +#if YARDSTICK_GRAMMAR_EXTENSION + try { + vector source_info; + for (size_t i = 0; i < source_count; i++) { + auto &source = sources[i]; + MeasureWindowSource info {source.key, source.relation_name, source.alias, source.clean_select_sql, + source.grouped, {}}; + for (size_t j = 0; j < source.dimension_count; j++) { + info.dimensions.emplace_back(source.dimension_names[j], source.dimension_expressions[j]); + } + source_info.push_back(std::move(info)); + } + vector call_info; + for (size_t i = 0; i < call_count; i++) { + auto &call = calls[i]; + MeasureWindowCall info {call.marker_name, call.source_key, call.expression_sql, {}}; + for (size_t j = 0; j < call.modifier_count; j++) { + auto &modifier = call.modifiers[j]; + WindowContextType type; + switch (modifier.type) { + case YARDSTICK_AT_ALL_GLOBAL: type = WindowContextType::ALL_GLOBAL; break; + case YARDSTICK_AT_ALL_DIM: type = WindowContextType::ALL; break; + case YARDSTICK_AT_SET: type = WindowContextType::SET; break; + case YARDSTICK_AT_WHERE: type = WindowContextType::WHERE; break; + case YARDSTICK_AT_VISIBLE: type = WindowContextType::VISIBLE; break; + default: continue; + } + info.modifiers.push_back({type, modifier.dimension ? modifier.dimension : "", + modifier.value ? modifier.value : ""}); + } + call_info.push_back(std::move(info)); + } + vector cte_names; + for (size_t i = 0; i < visible_cte_count; i++) { + cte_names.push_back(visible_ctes[i]); + } + vector cte_queries; + for (size_t i = 0; i < binding_cte_count; i++) { + cte_queries.push_back(binding_ctes[i]); + } + return safe_strdup(RewriteNativeMeasureWindows(sql, source_info, call_info, cte_names, + cte_queries, YardstickParserOptions())); + } catch (const std::exception &exception) { + if (error) *error = safe_strdup(exception.what()); + } +#endif + return nullptr; +} + //============================================================================= // FFI Implementation: yardstick_parse_select //============================================================================= @@ -1493,7 +1616,9 @@ static void NativeRelationQualifiers(TableRef &ref, std::unordered_set & } } -static vector NativeSubqueryDimensions(ParsedExpression &projection, SelectNode &outer) { +static vector NativeSubqueryDimensions( + ParsedExpression &projection, SelectNode &outer, + const std::function &rewrite = {}) { struct Scope { bool unqualified_outer = true; bool nested = false; @@ -1520,6 +1645,9 @@ static vector NativeSubqueryDimensions(ParsedExpression &projection, Sel if (std::find(dimensions.begin(), dimensions.end(), sql) == dimensions.end()) { dimensions.push_back(std::move(sql)); } + if (rewrite) { + rewrite(expr.Cast()); + } } } else if (expr.GetExpressionClass() == ExpressionClass::SUBQUERY) { query(*expr.Cast().SubqueryMutable()->node, scope); @@ -1789,6 +1917,7 @@ extern "C" YardstickSelectInfo* yardstick_parse_select(const char* sql) { #endif item.is_aggregate = ExpressionContainsAggregate(expr.get()); + item.contains_window = expr->IsWindow(); item.is_star = expr->GetExpressionClass() == ExpressionClass::STAR; item.is_measure_ref = ExpressionContainsMeasureRef(expr.get()); @@ -2579,10 +2708,44 @@ extern "C" char* yardstick_replace_range( return safe_strdup(result); } -extern "C" char* yardstick_qualify_expression(const char* expr_str, const char* qualifier) { +extern "C" char* yardstick_qualify_expression(const char* expr_str, const char* qualifier, + const char* dimension) { if (!expr_str || !qualifier) return nullptr; try { + if (dimension) { +#if YARDSTICK_GRAMMAR_EXTENSION + if (!CurrentNativeYardstickParserOptions()) return nullptr; + auto options = YardstickParserOptions(); + auto dimensions = Parser::ParseExpressionList(dimension, options); + if (dimensions.size() != 1 || dimensions[0]->GetExpressionClass() != ExpressionClass::COLUMN_REF) { + return nullptr; + } + auto column = dimensions[0]->Cast().GetColumnName(); + auto qualified = Parser::ParseExpressionList(string(qualifier) + "." + dimension, options); + if (qualified.size() != 1 || qualified[0]->GetExpressionClass() != ExpressionClass::COLUMN_REF) { + return nullptr; + } + auto qualified_names = qualified[0]->Cast().ColumnNames(); + Parser parser(options); + parser.ParseQuery(string("SELECT ") + expr_str + " FROM " + qualifier); + if (parser.statements.size() != 1 || parser.statements[0]->type != StatementType::SELECT_STATEMENT) { + return nullptr; + } + auto &query = parser.statements[0]->Cast().node; + if (query->type != QueryNodeType::SELECT_NODE) return nullptr; + auto &select = query->Cast(); + if (select.select_list.size() != 1) return nullptr; + NativeSubqueryDimensions(*select.select_list[0], select, [&](ColumnRefExpression &reference) { + if (!reference.IsQualified() && reference.GetColumnName() == column) { + reference.ColumnNamesMutable() = qualified_names; + } + }); + return safe_strdup(select.select_list[0]->ToString()); +#else + return nullptr; +#endif + } auto expressions = Parser::ParseExpressionList(expr_str, YardstickParserOptions()); if (expressions.empty()) { return safe_strdup(expr_str); diff --git a/test/sql/native_aggregate_decorations.test b/test/sql/native_aggregate_decorations.test new file mode 100644 index 0000000..a1bc29f --- /dev/null +++ b/test/sql/native_aggregate_decorations.test @@ -0,0 +1,239 @@ +# name: test/sql/native_aggregate_decorations.test +# description: Call decorations recompute base and derived measures over their base rows. +# group: [yardstick] + +require-env YARDSTICK_NATIVE_PEG 1 + +require yardstick + +statement ok +CREATE TABLE decoration_sales(region VARCHAR, amount INTEGER, weight INTEGER, label VARCHAR, priority INTEGER); + +statement ok +INSERT INTO decoration_sales VALUES + ('a', 10, 1, 'first', 2), ('a', 10, 3, 'second', 1), + ('a', 20, 2, 'third', NULL), ('b', 40, 4, 'fourth', 1), + ('b', NULL, 5, NULL, NULL); + +statement ok +CREATE VIEW decoration_v AS +SELECT region AS area, amount * 2 AS doubled, priority AS ranking, + SUM(amount) AS MEASURE revenue, + AVG(amount) AS MEASURE average_amount, + SUM(amount * weight)::DOUBLE / SUM(weight) AS MEASURE weighted, + SUM(amount)::DOUBLE / COUNT(amount) AS MEASURE ratio, + COUNT(*) AS MEASURE row_count, + SUM(amount) FILTER (WHERE amount >= 20) AS MEASURE large_revenue, + STRING_AGG(label, ',' ORDER BY label) AS MEASURE labels, + LIST(amount) AS MEASURE amounts +FROM decoration_sales; + +statement ok +CREATE TABLE decoration_scope_rows(id INTEGER, g INTEGER, x INTEGER); + +statement ok +INSERT INTO decoration_scope_rows VALUES (1, 1, 10), (2, 1, 20), (3, 2, 10), (4, 2, 30); + +statement ok +CREATE VIEW decoration_scope_v AS +SELECT id, g AS area, x, SUM(x) AS MEASURE total FROM decoration_scope_rows; + +foreach parser_mode true false + +statement ok +SET heap_based_parser=${parser_mode}; + +query RRR +SELECT AGGREGATE(DISTINCT revenue), AGGREGATE(DISTINCT average_amount), AGGREGATE(DISTINCT ratio) +FROM decoration_v; +---- +70.0 23.333333333333332 23.333333333333332 + +# FILTER is inside every aggregate leaf, including both sides of ratios. +# Unequal source row counts distinguish this from averaging grouped averages. +query RRR +SELECT AGGREGATE(average_amount) FILTER (WHERE area = 'a'), + AGGREGATE(weighted) FILTER (WHERE area = 'a'), + AGGREGATE(ratio) FILTER (WHERE area = 'a') +FROM decoration_v; +---- +13.333333333333334 13.333333333333334 13.333333333333334 + +# Resolve exposed aliases and computed dimensions before base-row filtering. +query RR +SELECT AGGREGATE(revenue) FILTER (WHERE v.doubled >= 40), + AGGREGATE(large_revenue) FILTER (WHERE v.area = 'a') +FROM decoration_v v; +---- +60.0 20.0 + +query RR +SELECT AGGREGATE(revenue) FILTER (WHERE FALSE), + AGGREGATE(ratio) FILTER (WHERE FALSE) +FROM decoration_v; +---- +NULL NULL + +# Caller ordering is primary; the declaration's label order breaks ties. +query TT +SELECT AGGREGATE(labels ORDER BY ranking ASC NULLS FIRST), + AGGREGATE(amounts ORDER BY ranking DESC NULLS LAST, doubled DESC) +FROM decoration_v; +---- +third,fourth,second,first [10, 40, 10, 20, NULL] + +query T +SELECT AGGREGATE(labels ORDER BY v.ranking DESC NULLS LAST) FILTER (WHERE v.area = 'a') +FROM decoration_v v; +---- +first,second,third + +# Correlated references resolve to the outer view; nested aliases retain ownership. +query R +SELECT AGGREGATE(revenue) FILTER (WHERE EXISTS ( + SELECT 1 FROM (VALUES ('a')) allowed(area) WHERE allowed.area = v.area +)) FROM decoration_v v; +---- +40.0 + +query R +SELECT AGGREGATE(revenue) FILTER (WHERE EXISTS ( + SELECT 1 FROM (VALUES ('a')) v(area) WHERE v.area = 'a' +)) FROM decoration_v v; +---- +80.0 + +query R +SELECT AGGREGATE(revenue) FILTER (WHERE list_contains(list_transform(['a'], lambda area: area), v.area)) +FROM decoration_v v; +---- +40.0 + +# An unrelated nested FROM column does not hide an unqualified outer dimension. +# The alias differs from its base expression, exposing accidental view-row binding. +query R +SELECT AGGREGATE(total) FILTER (WHERE EXISTS ( + SELECT 1 FROM range(1) a(i) WHERE area = 1 +)) FROM decoration_scope_v; +---- +30.0 + +query R +SELECT DISTINCT AGGREGATE(total) FILTER (WHERE EXISTS ( + SELECT 1 FROM range(1) a(i) WHERE area = 1 +)) OVER () FROM decoration_scope_v; +---- +30.0 + +query R +SELECT AGGREGATE(total) FILTER (WHERE EXISTS ( + WITH permitted AS (SELECT i FROM range(1) r(i)) + SELECT 1 FROM permitted WHERE area = 1 +)) FROM decoration_scope_v; +---- +30.0 + +query R +SELECT AGGREGATE(total) FILTER (WHERE EXISTS ( + SELECT 1 FROM decoration_scope_rows a WHERE area = 1 AND a.id = 1 +)) FROM decoration_scope_v; +---- +30.0 + +query R +SELECT AGGREGATE(total) FILTER (WHERE EXISTS ( + SELECT 1 FROM range(1) a(i) JOIN range(1) b(j) ON area = 1 +)) FROM decoration_scope_v; +---- +30.0 + +query R +SELECT AGGREGATE(total) FILTER (WHERE EXISTS ( + SELECT 1 FROM (SELECT area AS permitted_area) a WHERE permitted_area = 1 +)) FROM decoration_scope_v; +---- +30.0 + +# An actual local column of the same name must still shadow the view dimension. +query R +SELECT AGGREGATE(total) FILTER (WHERE EXISTS ( + SELECT 1 FROM range(1) a(area) WHERE area = 0 +)) FROM decoration_scope_v; +---- +70.0 + +query R +SELECT AGGREGATE(total) FILTER (WHERE EXISTS ( + SELECT 1 FROM (VALUES (0)) a(area) WHERE EXISTS ( + SELECT 1 FROM range(1) b(i) WHERE area = 0 + ) +)) FROM decoration_scope_v; +---- +70.0 + +query R +SELECT AGGREGATE(total) FILTER (WHERE EXISTS ( + WITH permitted(area) AS (SELECT i FROM range(1) r(i)) + SELECT 1 FROM permitted WHERE area = 0 +)) FROM decoration_scope_v; +---- +70.0 + +# Native state export and composite formula states both round-trip. +query R +SELECT finalize(state) FROM (SELECT AGGREGATE(revenue) EXPORT_STATE AS state FROM decoration_v); +---- +80.0 + +query R +SELECT yardstick_finalize(state) +FROM (SELECT AGGREGATE(ratio) EXPORT_STATE AS state FROM decoration_v); +---- +20.0 + +# The explicit query function uses the same catalog-aware decoration lowering. +query R +SELECT * FROM yardstick('SELECT AGGREGATE(revenue) FILTER (WHERE area = ''a'') FROM decoration_v'); +---- +40.0 + +query R +SELECT * FROM yardstick('SELECT DISTINCT AGGREGATE(revenue) OVER () AS total FROM decoration_v'); +---- +80.0 + +query I +SELECT AGGREGATE(row_count) FILTER (WHERE FALSE) FROM decoration_v; +---- +0 + +query I +SELECT AGGREGATE(row_count) FILTER (WHERE TRUE) FROM decoration_v WHERE FALSE; +---- +0 + +query I +SELECT yardstick_finalize(state) +FROM (SELECT AGGREGATE(row_count) FILTER (WHERE TRUE) EXPORT_STATE AS state + FROM decoration_v WHERE FALSE); +---- +0 + +query I +SELECT AGGREGATE(row_count) FILTER (WHERE TRUE) AT (ALL) FROM decoration_v WHERE FALSE; +---- +5 + +# Native DISTINCT states combine their already accumulated statistics; they do +# not retain a set of values for deduplication between independent shards. +query R +WITH states AS ( + SELECT area, AGGREGATE(DISTINCT revenue) EXPORT_STATE AS state + FROM decoration_v GROUP BY area +) +SELECT yardstick_finalize(yardstick_combine(a.state, b.state)) +FROM states a, states b WHERE a.area = 'a' AND b.area = 'b'; +---- +70.0 + +endloop diff --git a/test/sql/native_aggregate_errors.test b/test/sql/native_aggregate_errors.test index fa54741..fe6f6f8 100644 --- a/test/sql/native_aggregate_errors.test +++ b/test/sql/native_aggregate_errors.test @@ -1,5 +1,5 @@ # name: test/sql/native_aggregate_errors.test -# description: Unsupported measure-call decorations fail instead of losing semantics in compatibility lowering. +# description: Decorations retain semantics through statement scopes and preserve invalid-input errors. # group: [yardstick] require-env YARDSTICK_NATIVE_PEG 1 @@ -26,115 +26,139 @@ foreach parser_mode true false statement ok SET heap_based_parser=${parser_mode}; -statement error +query R SELECT AGGREGATE(DISTINCT revenue) FROM aggregate_error_v; ---- -Yardstick AGGREGATE does not support +40.0 -statement error +query R SELECT AGGREGATE(revenue ORDER BY amount) FROM aggregate_error_v; ---- -Yardstick AGGREGATE does not support +50.0 -statement error +query R SELECT AGGREGATE(revenue) FILTER (WHERE amount > 10) FROM aggregate_error_v; ---- -Yardstick AGGREGATE does not support +30.0 -statement error +query R SELECT AGGREGATE(revenue) OVER () FROM aggregate_error_v; ---- -Yardstick AGGREGATE does not support +50.0 +50.0 +50.0 -statement error +query R rowsort SELECT AGGREGATE(revenue) OVER (PARTITION BY region) FROM aggregate_error_v; ---- -Yardstick AGGREGATE does not support +20.0 +20.0 +30.0 -statement error -SELECT AGGREGATE(revenue) EXPORT_STATE FROM aggregate_error_v; +query R +SELECT yardstick_finalize(state) +FROM (SELECT AGGREGATE(revenue) EXPORT_STATE AS state FROM aggregate_error_v); ---- -Yardstick AGGREGATE does not support +50.0 -# Trivia, nested scopes, built-in calls, and AT must not hide the error. -statement error +# Trivia, nested scopes, built-in calls, and AT retain decoration semantics. +query R SELECT "AGGREGATE"/* comment */(revenue) FILTER (WHERE FALSE) FROM aggregate_error_v; ---- -Yardstick AGGREGATE does not support +NULL -statement error +query IRR SELECT aggregate([1, 2], 'sum'), AGGREGATE(revenue), AGGREGATE(revenue) FILTER (WHERE FALSE) FROM aggregate_error_v; ---- -Yardstick AGGREGATE does not support +3 50.0 NULL -statement error +query RR SELECT revenue AT (ALL), AGGREGATE(revenue) FILTER (WHERE FALSE) FROM aggregate_error_v; ---- -Yardstick AGGREGATE does not support +50.0 NULL -statement error +query RR SELECT AGGREGATE(revenue) FILTER (WHERE FALSE), revenue AT (ALL) FROM aggregate_error_v; ---- -Yardstick AGGREGATE does not support +NULL 50.0 -statement error +query R SELECT (SELECT AGGREGATE(revenue) FILTER (WHERE FALSE) FROM aggregate_error_v); ---- -Yardstick AGGREGATE does not support +NULL -statement error +query R rowsort WITH totals AS (SELECT AGGREGATE(revenue) OVER (PARTITION BY region) AS total FROM aggregate_error_v) SELECT * FROM totals; ---- -Yardstick AGGREGATE does not support +20.0 +20.0 +30.0 -statement error +query R rowsort SELECT AGGREGATE(revenue) FILTER (WHERE FALSE) FROM aggregate_error_v UNION ALL SELECT AGGREGATE(revenue) FROM aggregate_error_v; ---- -Yardstick AGGREGATE does not support +50.0 +NULL -statement error +query R SELECT AGGREGATE(DISTINCT revenue) AT (ALL) FROM aggregate_error_v; ---- -Yardstick AGGREGATE does not support +40.0 -statement error +query R SELECT AGGREGATE(revenue ORDER BY amount) AT (ALL) FROM aggregate_error_v; ---- -Yardstick AGGREGATE does not support +50.0 -statement error +query R SELECT AGGREGATE(revenue) FILTER (WHERE FALSE) AT (ALL) FROM aggregate_error_v; ---- -Yardstick AGGREGATE does not support +NULL -statement error +query R SELECT AGGREGATE(revenue) OVER () AT (ALL) FROM aggregate_error_v; ---- -Yardstick AGGREGATE does not support +50.0 +50.0 +50.0 -statement error +statement ok INSERT INTO aggregate_error_results SELECT AGGREGATE(revenue) FILTER (WHERE FALSE) FROM aggregate_error_v; ----- -Yardstick AGGREGATE does not support -# Modifier subqueries retain their own expression validation. -statement error +# Modifier subqueries lower their own decorated calls before the outer context. +query R SELECT AGGREGATE(revenue) AT (WHERE amount = ( SELECT AGGREGATE(revenue) FILTER (WHERE FALSE) FROM aggregate_error_v )) FROM aggregate_error_v; ---- -Yardstick AGGREGATE does not support +NULL -statement error +query R SELECT AGGREGATE(revenue) AT (SET amount = ( SELECT AGGREGATE(revenue) OVER () FROM aggregate_error_v LIMIT 1 )) FROM aggregate_error_v; ---- -Yardstick AGGREGATE does not support +NULL + +# The inner amount is local; the sibling amount belongs to the consumer group. +query IR +SELECT amount, AGGREGATE(revenue) AT ( + SET amount = (SELECT max(amount) FROM aggregate_error_v) + amount - 30 +) FROM aggregate_error_v ORDER BY amount; +---- +10 20.0 +30 30.0 + +query IR +SELECT amount, AGGREGATE(revenue) AT (SET amount = amount) +FROM main.aggregate_error_v ORDER BY amount; +---- +10 20.0 +30 30.0 # Ordinary aggregate decorations in a measure definition remain valid. query RR @@ -152,6 +176,17 @@ SELECT AGGREGATE(revenue) AS export_state FROM aggregate_error_v; ---- 50.0 +# Decorations must still bind their referenced columns. +statement error +SELECT AGGREGATE(revenue) FILTER (WHERE missing_filter_column > 0) FROM aggregate_error_v; +---- +missing_filter_column + +statement error +SELECT AGGREGATE(revenue ORDER BY missing_order_column) FROM aggregate_error_v; +---- +missing_order_column + # DuckDB retains ownership of invalid decorations on its multiargument scalar. statement error SELECT aggregate(DISTINCT [1, 2], 'sum'); @@ -168,4 +203,4 @@ endloop query I SELECT COUNT(*) FROM aggregate_error_results; ---- -0 +2 diff --git a/test/sql/native_aggregate_state_helpers.test b/test/sql/native_aggregate_state_helpers.test new file mode 100644 index 0000000..014c0d9 --- /dev/null +++ b/test/sql/native_aggregate_state_helpers.test @@ -0,0 +1,169 @@ +# name: test/sql/native_aggregate_state_helpers.test +# description: Composite exported states retain leaf statistics, formulas, and types. +# group: [yardstick] + +require-env YARDSTICK_NATIVE_PEG 1 + +require yardstick + +# Match native DISTINCT state semantics: overlap between shards is not deduplicated. +query II +WITH data(shard, value) AS (VALUES (1, 10), (1, 20), (2, 10), (2, 30)), + states AS (SELECT shard, sum(DISTINCT value) EXPORT_STATE AS state FROM data GROUP BY shard) +SELECT yardstick_finalize(yardstick_combine(a.state, b.state)), + (SELECT sum(DISTINCT value) FROM data) +FROM states a, states b WHERE a.shard = 1 AND b.shard = 2; +---- +70 60 + +query R +SELECT yardstick_finalize(avg(i) EXPORT_STATE) FROM range(6) t(i); +---- +2.5 + +query I +WITH a AS (SELECT sum(i) EXPORT_STATE s FROM range(2) t(i)), + b AS (SELECT sum(i) EXPORT_STATE s FROM range(2, 6) t(i)) +SELECT yardstick_finalize(yardstick_combine(a.s, b.s)) FROM a, b; +---- +15 + +# Unequal shards distinguish combining sufficient statistics from averaging finalized ratios. +statement ok +CREATE TABLE composite_state_shards AS +SELECT i < 2 AS shard, + yardstick_state('s0 / s1', struct_pack(s0 := sum(i) EXPORT_STATE, s1 := count(i) EXPORT_STATE)) AS state +FROM range(6) t(i) GROUP BY shard; + +query R +SELECT yardstick_finalize(yardstick_combine(a.state, b.state)) +FROM composite_state_shards a, composite_state_shards b +WHERE a.shard AND NOT b.shard; +---- +2.5 + +query R rowsort +SELECT yardstick_finalize(state) FROM composite_state_shards; +---- +0.5 +3.5 + +statement ok +PREPARE finalize_composite AS SELECT yardstick_finalize(state) FROM composite_state_shards ORDER BY shard; + +query R +EXECUTE finalize_composite; +---- +3.5 +0.5 + +query I +SELECT count(*) FROM composite_state_shards WHERE starts_with(typeof(state), 'yardstick_state_v1:'); +---- +2 + +# Round-trip the type and alias through a persistent DuckDB table and reattach it. +statement ok +ATTACH '__TEST_DIR__/yardstick_composite_states.db' AS composite_state_storage; + +statement ok +CREATE TABLE composite_state_storage.saved_states AS SELECT * FROM composite_state_shards; + +statement ok +DETACH composite_state_storage; + +statement ok +ATTACH '__TEST_DIR__/yardstick_composite_states.db' AS composite_state_storage; + +query R +SELECT yardstick_finalize(yardstick_combine(a.state, b.state)) +FROM composite_state_storage.saved_states a, composite_state_storage.saved_states b +WHERE a.shard AND NOT b.shard; +---- +2.5 + +statement ok +DETACH composite_state_storage; + +query R rowsort +SELECT yardstick_finalize(yardstick_combine(state, NULL)) FROM composite_state_shards; +---- +0.5 +3.5 + +query R rowsort +SELECT yardstick_finalize(yardstick_combine(NULL, state)) FROM composite_state_shards; +---- +0.5 +3.5 + +# Typed NULLs must remain NULL rather than becoming a non-NULL struct of NULL leaves. +query II +WITH states AS (SELECT CASE WHEN false THEN state END AS missing, state FROM composite_state_shards) +SELECT count(yardstick_finalize(missing)), count(yardstick_combine(missing, missing)) FROM states; +---- +0 0 + +query R rowsort +WITH states AS (SELECT CASE WHEN false THEN state END AS missing, state FROM composite_state_shards) +SELECT yardstick_finalize(yardstick_combine(missing, state)) FROM states; +---- +0.5 +3.5 + +query II +SELECT yardstick_finalize(NULL), yardstick_combine(NULL, NULL); +---- +NULL NULL + +query I +SELECT yardstick_finalize(yardstick_state('coalesce(s0, 42)', struct_pack(s0 := sum(i) EXPORT_STATE))) +FROM range(0) t(i); +---- +42 + +query R +SELECT yardstick_finalize(yardstick_state('case when s1 = 0 then NULL else round(s0::DOUBLE / s1, 1) end', + struct_pack(s0 := sum(i) EXPORT_STATE, s1 := count(i) EXPORT_STATE))) +FROM range(6) t(i); +---- +2.5 + +statement error +SELECT yardstick_combine(yardstick_state('s0 / s1', struct_pack(s0 := sum(i) EXPORT_STATE, s1 := count(i) EXPORT_STATE)), + yardstick_state('s0 + s1', struct_pack(s0 := sum(i) EXPORT_STATE, s1 := count(i) EXPORT_STATE))) +FROM range(6) t(i); +---- +matching state formulas and aggregate types + +statement error +SELECT yardstick_combine(yardstick_state('s0', struct_pack(s0 := sum(i) EXPORT_STATE)), + yardstick_state('s0', struct_pack(s0 := count(i) EXPORT_STATE))) +FROM range(6) t(i); +---- +matching state formulas and aggregate types + +statement error +SELECT yardstick_state('s1', struct_pack(s0 := sum(i) EXPORT_STATE)) FROM range(6) t(i); +---- +unknown state field + +statement error +SELECT yardstick_state('s0', struct_pack(s0 := 42)); +---- +native aggregate states named s0, s1 + +statement error +SELECT yardstick_state('s0', struct_pack(total := sum(i) EXPORT_STATE)) FROM range(6) t(i); +---- +native aggregate states named s0, s1 + +statement error +SELECT yardstick_finalize(struct_pack(s0 := sum(i) EXPORT_STATE)) FROM range(6) t(i); +---- +requires a native aggregate state or yardstick_state + +statement error +SELECT yardstick_state('(select 42)', struct_pack(s0 := sum(i) EXPORT_STATE)) FROM range(6) t(i); +---- +cannot contain subqueries diff --git a/test/sql/native_aggregate_windows.test b/test/sql/native_aggregate_windows.test new file mode 100644 index 0000000..965fa3a --- /dev/null +++ b/test/sql/native_aggregate_windows.test @@ -0,0 +1,712 @@ +# name: test/sql/native_aggregate_windows.test +# description: Measure windows preserve SQL frames and original base-row identity. +# group: [yardstick] + +require-env YARDSTICK_NATIVE_PEG 1 + +require yardstick + +statement ok +CREATE TABLE measure_window_rows(id INTEGER, yr INTEGER, region VARCHAR, amount INTEGER, units INTEGER); + +statement ok +INSERT INTO measure_window_rows VALUES + (1, 2022, 'A', 10, 1), (2, 2022, 'A', 30, 3), + (3, 2023, 'A', 100, 2), (4, 2024, 'B', 200, 4), + (5, 2024, 'B', 200, 4), (6, 2025, NULL, 60, 2); + +statement ok +CREATE TABLE measure_window_left(id INTEGER, amount INTEGER); + +statement ok +INSERT INTO measure_window_left VALUES (1, 10), (2, 20); + +statement ok +CREATE TABLE measure_window_right(id INTEGER); + +statement ok +INSERT INTO measure_window_right VALUES (10), (20); + +foreach parser_mode true false + +statement ok +SET heap_based_parser=${parser_mode}; + +statement ok +CREATE OR REPLACE VIEW measure_window_raw AS +SELECT id, yr, region, + SUM(amount) AS MEASURE revenue, + AVG(amount) AS MEASURE mean_amount, + SUM(amount)::DOUBLE / SUM(units) AS MEASURE unit_price, + COUNT(*) AS MEASURE row_count, + LIST(amount) AS MEASURE amounts +FROM measure_window_rows; + +statement ok +CREATE OR REPLACE VIEW measure_window_grouped AS +SELECT yr, region, + SUM(amount) AS MEASURE revenue, + AVG(amount) AS MEASURE mean_amount, + SUM(amount)::DOUBLE / SUM(units) AS MEASURE unit_price, + COUNT(DISTINCT amount) AS MEASURE distinct_amounts +FROM measure_window_rows +GROUP BY yr, region; + +# ROWS counts visible rows, while evaluation recomputes the measure over the +# original rows selected by the frame. +query II +SELECT id, AGGREGATE(revenue) OVER (ORDER BY id ROWS BETWEEN 1 PRECEDING AND CURRENT ROW) AS running +FROM measure_window_raw ORDER BY id; +---- +1 10 +2 40 +3 130 +4 300 +5 400 +6 260 + +# GROUPS retains peers; NULL partition keys are ordinary partition values. +query II +SELECT id, AGGREGATE(revenue) OVER (PARTITION BY region ORDER BY yr GROUPS CURRENT ROW) +FROM measure_window_raw ORDER BY id; +---- +1 40 +2 40 +3 100 +4 400 +5 400 +6 60 + +query II +SELECT id, AGGREGATE(revenue) OVER (ORDER BY yr RANGE BETWEEN 1 PRECEDING AND CURRENT ROW) +FROM measure_window_raw ORDER BY id; +---- +1 40 +2 40 +3 140 +4 500 +5 500 +6 460 + +query II +SELECT id, AGGREGATE(revenue) OVER (ORDER BY id ROWS BETWEEN (id % 2) PRECEDING AND CURRENT ROW) +FROM measure_window_raw ORDER BY id; +---- +1 10 +2 30 +3 130 +4 200 +5 400 +6 60 + +query II +SELECT id, AGGREGATE(revenue) OVER (ORDER BY yr GROUPS CURRENT ROW EXCLUDE TIES) +FROM measure_window_raw ORDER BY id; +---- +1 10 +2 30 +3 100 +4 200 +5 200 +6 60 + +# Empty frames use the defining aggregate's empty-input behavior. +query III +SELECT id, + AGGREGATE(row_count) OVER (ORDER BY id ROWS CURRENT ROW EXCLUDE CURRENT ROW), + AGGREGATE(revenue) OVER (ORDER BY id ROWS CURRENT ROW EXCLUDE CURRENT ROW) +FROM measure_window_raw ORDER BY id; +---- +1 0 NULL +2 0 NULL +3 0 NULL +4 0 NULL +5 0 NULL +6 0 NULL + +# Original identical values remain two independent rows. +query I +SELECT AGGREGATE(revenue) OVER () AS total +FROM measure_window_raw LIMIT 1; +---- +600 + +# DISTINCT belongs to the defining aggregate leaves, separately from identity +# deduplication used to make measures immune to join fanout. +query I +SELECT AGGREGATE(DISTINCT revenue) OVER () AS total +FROM measure_window_raw LIMIT 1; +---- +400 + +query I +SELECT AGGREGATE(revenue) FILTER (WHERE region = 'A') OVER () AS total +FROM measure_window_raw LIMIT 1; +---- +140 + +# A window call FILTER selects input rows before measure evaluation. It may +# reference an outer joined relation, and leaves output row count unchanged. +query III +SELECT v.id, j.copy, + AGGREGATE(v.revenue) FILTER (WHERE j.copy = 1) + OVER (ORDER BY v.id, j.copy ROWS UNBOUNDED PRECEDING) +FROM measure_window_raw v CROSS JOIN range(2) j(copy) +WHERE v.id <= 2 ORDER BY v.id, j.copy; +---- +1 0 NULL +1 1 10 +2 0 10 +2 1 40 + +query II +SELECT AGGREGATE(row_count) FILTER (WHERE false) OVER (), + AGGREGATE(revenue) FILTER (WHERE false) OVER () +FROM measure_window_raw LIMIT 1; +---- +0 NULL + +query I +SELECT AGGREGATE(revenue) FILTER (WHERE false) OVER () AT (ALL) +FROM measure_window_raw LIMIT 1; +---- +600 + +# A lambda parameter belongs to its lambda, even when a source dimension has +# the same name. Nested relations likewise retain their own local columns. +query I +SELECT AGGREGATE(revenue) FILTER ( + WHERE list_contains(list_transform([1], lambda yr: yr), 1) +) OVER () FROM measure_window_raw LIMIT 1; +---- +600 + +query I +SELECT AGGREGATE(revenue) FILTER ( + WHERE EXISTS (SELECT 1 FROM (VALUES ('A')) a(region) WHERE a.region = measure_window_raw.region) +) OVER () FROM measure_window_raw LIMIT 1; +---- +140 + +# Unqualified names capture the outer source only when the nested relation does +# not define that name. +query I +SELECT AGGREGATE(revenue) FILTER ( + WHERE EXISTS (SELECT 1 FROM (VALUES (1)) a(dummy) WHERE yr = 2022) +) OVER () FROM measure_window_raw LIMIT 1; +---- +40 + +query I +SELECT AGGREGATE(revenue) FILTER ( + WHERE EXISTS (SELECT 1 FROM (VALUES ('A')) a(region) WHERE region = 'A') +) OVER () FROM measure_window_raw LIMIT 1; +---- +600 + +query T +SELECT AGGREGATE(amounts ORDER BY id DESC) OVER () AS ordered_amounts +FROM measure_window_raw LIMIT 1; +---- +[60, 200, 200, 100, 30, 10] + +# Grouped view rows carry different numbers of base rows: neither averaging +# averages nor adding derived ratios produces these results. +query IRR +SELECT yr, + AGGREGATE(mean_amount) OVER (ORDER BY yr ROWS BETWEEN 1 PRECEDING AND CURRENT ROW), + AGGREGATE(unit_price) OVER (ORDER BY yr ROWS BETWEEN 1 PRECEDING AND CURRENT ROW) +FROM measure_window_grouped ORDER BY yr; +---- +2022 20.000 10.000 +2023 46.667 23.333 +2024 166.667 50.000 +2025 153.333 46.000 + +query II +SELECT yr, AGGREGATE(distinct_amounts) OVER (ORDER BY yr ROWS BETWEEN 1 PRECEDING AND CURRENT ROW) +FROM measure_window_grouped ORDER BY yr; +---- +2022 2 +2023 3 +2024 2 +2025 2 + +query I +SELECT AGGREGATE(revenue) FILTER (WHERE region = 'A') OVER () +FROM measure_window_grouped LIMIT 1; +---- +140 + +query TI +SELECT region, AGGREGATE(revenue) FILTER (WHERE count(*) > 2) OVER () +FROM measure_window_raw GROUP BY region ORDER BY region NULLS LAST; +---- +A 140 +B 140 +NULL 140 + +# A multiplying join repeats a reference to the same original rows. It changes +# the SQL frame's row count, but does not inflate a measure's base-row set. +query III +SELECT v.yr, j.copy, + AGGREGATE(v.revenue) OVER (ORDER BY v.yr, j.copy ROWS UNBOUNDED PRECEDING) +FROM measure_window_grouped v CROSS JOIN range(2) j(copy) +ORDER BY v.yr, j.copy; +---- +2022 0 40 +2022 1 40 +2023 0 140 +2023 1 140 +2024 0 540 +2024 1 540 +2025 0 600 +2025 1 600 + +# A grouped consumer first forms its groups, then applies the SQL window. +query TI +SELECT region, AGGREGATE(revenue) OVER (ORDER BY region NULLS LAST ROWS UNBOUNDED PRECEDING) +FROM measure_window_raw GROUP BY region ORDER BY region NULLS LAST; +---- +A 140 +B 540 +NULL 600 + +query I +SELECT AGGREGATE(revenue) OVER () FROM measure_window_raw GROUP BY (); +---- +600 + +query TII +SELECT region, COUNT(*) AS visible_rows, + AGGREGATE(revenue) OVER (ORDER BY region NULLS LAST ROWS UNBOUNDED PRECEDING) +FROM measure_window_raw GROUP BY region ORDER BY region NULLS LAST; +---- +A 3 140 +B 2 540 +NULL 1 600 + +query TI +SELECT region, AGGREGATE(revenue) OVER () +FROM measure_window_raw GROUP BY ALL ORDER BY region NULLS LAST; +---- +A 600 +B 600 +NULL 600 + +# Aggregate classification comes from DuckDB's catalog, including aggregates +# beyond Yardstick's original built-in name list. +query IR +SELECT AGGREGATE(revenue) OVER (), product(id) FROM measure_window_raw; +---- +600 720.000 + +# Input names take priority inside expressions and window specifications. +# A bare final ORDER BY alias still denotes the output column. +query II +SELECT -id AS id, AGGREGATE(revenue) OVER (ORDER BY id ROWS UNBOUNDED PRECEDING) +FROM measure_window_raw ORDER BY id; +---- +-6 600 +-5 540 +-4 340 +-3 140 +-2 40 +-1 10 + +# Reusing a window-result alias must happen after the window stage. +query III +SELECT AGGREGATE(revenue) OVER () AS total, total + 1 AS plus, plus + 1 AS again +FROM measure_window_raw LIMIT 1; +---- +600 601 602 + +query II +SELECT id, AGGREGATE(revenue) OVER () AS total +FROM measure_window_raw ORDER BY total + id DESC LIMIT 2; +---- +6 600 +5 600 + +statement ok +CREATE OR REPLACE VIEW measure_window_shadow AS +SELECT id + 1 AS id, yr, region, SUM(amount) AS MEASURE revenue +FROM measure_window_rows; + +query II +SELECT id, AGGREGATE(revenue) OVER (ORDER BY id ROWS CURRENT ROW) +FROM measure_window_shadow ORDER BY id; +---- +2 10 +3 30 +4 100 +5 200 +6 200 +7 60 + +query I +SELECT AGGREGATE(revenue) FILTER (WHERE id >= 6) OVER () FROM measure_window_shadow LIMIT 1; +---- +260 + +# The defining view's final ordering chooses its output alias before LIMIT. +# Reconstructing lineage must select the same source rows as the actual view. +statement ok +CREATE OR REPLACE VIEW measure_window_limited AS +SELECT -id AS id, amount, SUM(amount) AS MEASURE revenue +FROM measure_window_rows ORDER BY id LIMIT 2; + +query II +SELECT id, AGGREGATE(revenue) OVER () FROM measure_window_limited ORDER BY id; +---- +-6 260 +-5 260 + +statement ok +CREATE OR REPLACE VIEW measure_window_alias_chain AS +SELECT id AS key_id, key_id + 1 AS shifted, SUM(amount) AS MEASURE revenue +FROM measure_window_rows; + +query II +SELECT shifted, AGGREGATE(revenue) FILTER (WHERE shifted > 5) OVER () +FROM measure_window_alias_chain ORDER BY shifted LIMIT 1; +---- +2 260 + +statement ok +CREATE OR REPLACE VIEW measure_window_nested AS +SELECT id, SUM((SELECT amount FROM (VALUES (1)) a(dummy))) AS MEASURE revenue +FROM measure_window_rows; + +query I +SELECT AGGREGATE(revenue) OVER () FROM measure_window_nested LIMIT 1; +---- +600 + +# Schema-backed star expansion happens before hidden provenance is introduced. +query IITI +SELECT * EXCLUDE (revenue, mean_amount, unit_price, row_count, amounts), + AGGREGATE(revenue) OVER () AS total +FROM measure_window_raw ORDER BY id; +---- +1 2022 A 600 +2 2022 A 600 +3 2023 A 600 +4 2024 B 600 +5 2024 B 600 +6 2025 NULL 600 + +statement ok +CREATE OR REPLACE VIEW measure_window_bonus AS +SELECT id, SUM(amount * 2) AS MEASURE bonus FROM measure_window_rows; + +query III +SELECT v.id, + AGGREGATE(v.revenue) OVER (ORDER BY v.id ROWS CURRENT ROW), + AGGREGATE(b.bonus) OVER (ORDER BY v.id ROWS BETWEEN 1 PRECEDING AND CURRENT ROW) +FROM measure_window_raw v JOIN measure_window_bonus b ON v.id = b.id +ORDER BY v.id; +---- +1 10 20 +2 30 80 +3 100 260 +4 200 600 +5 200 800 +6 60 520 + +# Decorations on a view with a joined defining relation retain which input +# owns an identically named column. +statement ok +CREATE OR REPLACE VIEW measure_window_joined AS +SELECT a.id AS aid, b.id AS bid, SUM(a.amount) AS MEASURE joined_revenue, + LIST(a.amount) AS MEASURE joined_amounts +FROM measure_window_left a CROSS JOIN measure_window_right b; + +query II +SELECT AGGREGATE(joined_revenue) FILTER (WHERE aid = 1) OVER (), + AGGREGATE(joined_revenue) FILTER (WHERE bid = 10) OVER () +FROM measure_window_joined LIMIT 1; +---- +20 30 + +query T +SELECT AGGREGATE(joined_amounts ORDER BY bid DESC, aid DESC) OVER () +FROM measure_window_joined LIMIT 1; +---- +[20, 10, 20, 10] + +# Named windows, projection aliases, ordinary windows, QUALIFY and final LIMIT +# keep their original SQL evaluation order. +query III +SELECT id AS row_id, + AGGREGATE(revenue) OVER w AS running, + row_number() OVER w AS position +FROM measure_window_raw +WINDOW w AS (ORDER BY id ROWS UNBOUNDED PRECEDING) +QUALIFY running >= 140 +ORDER BY row_id DESC LIMIT 2; +---- +6 600 6 +5 540 5 + +query II +SELECT yr, AGGREGATE(revenue) OVER (ORDER BY yr ROWS CURRENT ROW) AT (ALL) +FROM measure_window_grouped ORDER BY yr; +---- +2022 600 +2023 600 +2024 600 +2025 600 + +query II +SELECT yr, AGGREGATE(revenue) OVER (ORDER BY yr ROWS CURRENT ROW) AT (ALL yr) +FROM measure_window_grouped ORDER BY yr; +---- +2022 140 +2023 140 +2024 400 +2025 60 + +query II +SELECT yr, AGGREGATE(revenue) OVER (ORDER BY yr ROWS CURRENT ROW) AT (SET yr = CURRENT yr - 1) +FROM measure_window_grouped ORDER BY yr; +---- +2022 NULL +2023 40 +2024 NULL +2025 NULL + +query II +SELECT yr, AGGREGATE(revenue) OVER (ORDER BY yr ROWS CURRENT ROW) AT (WHERE yr = CURRENT yr - 1) +FROM measure_window_grouped ORDER BY yr; +---- +2022 NULL +2023 40 +2024 100 +2025 400 + +# ALL may expand a frame; VISIBLE retains the enclosing query's visible set. +query II +SELECT yr, AGGREGATE(revenue) OVER (ORDER BY yr ROWS CURRENT ROW) AT (VISIBLE ALL yr) +FROM measure_window_grouped WHERE yr = 2022 ORDER BY yr; +---- +2022 40 + +query II +SELECT yr, AGGREGATE(revenue) OVER (ORDER BY yr ROWS CURRENT ROW) AT (ALL yr) +FROM measure_window_grouped WHERE yr = 2022 ORDER BY yr; +---- +2022 140 + +# Joined-input argument ordering travels with each frame occurrence. Repeated +# source IDs choose their first ordered occurrence before fanout is removed. +statement ok +CREATE OR REPLACE VIEW measure_window_ordered AS +SELECT id, LIST(amount ORDER BY id DESC) AS MEASURE amounts FROM measure_window_rows; + +query T +SELECT DISTINCT AGGREGATE(amounts ORDER BY j.copy) OVER () +FROM measure_window_ordered r +JOIN (VALUES (1, 4), (1, 0), (2, 2), (3, 1)) j(id, copy) ON r.id = j.id; +---- +[10, 100, 30] + +# Equal caller keys retain the declaration's order as the tie breaker. +query T +SELECT DISTINCT AGGREGATE(amounts ORDER BY j.copy) OVER () +FROM measure_window_ordered r +JOIN (VALUES (1, 0), (1, 0), (2, 0), (3, 0)) j(id, copy) ON r.id = j.id; +---- +[100, 30, 10] + +query T +SELECT DISTINCT AGGREGATE(amounts ORDER BY j.copy, r.id DESC) OVER () +FROM measure_window_ordered r +JOIN (VALUES (1, 0), (2, 0), (3, 0)) j(id, copy) ON r.id = j.id; +---- +[100, 30, 10] + +query T +SELECT DISTINCT AGGREGATE(amounts ORDER BY j.copy + amount) OVER () +FROM measure_window_ordered r +JOIN (VALUES (1, 50), (2, 0), (3, -90)) j(id, copy) ON r.id = j.id; +---- +[100, 30, 10] + +# AT expansion has no joined-input key for rows absent from the frame. Those +# keys are NULL; source-owned declaration ordering still breaks their ties. +query T +SELECT AGGREGATE(amounts ORDER BY j.copy NULLS LAST) OVER () AT (ALL) +FROM measure_window_ordered r JOIN (VALUES (1, 0)) j(id, copy) ON r.id = j.id; +---- +[10, 60, 200, 200, 100, 30] + +query T +SELECT DISTINCT AGGREGATE(amounts ORDER BY copy DESC) OVER () +FROM measure_window_ordered r +JOIN (VALUES (1, 0), (2, 2), (3, 1)) j(id, copy) ON r.id = j.id; +---- +[30, 100, 10] + +query T +SELECT DISTINCT AGGREGATE(amounts ORDER BY SUM(j.copy) DESC) OVER () +FROM measure_window_ordered r +JOIN (VALUES (1, 1), (1, 2), (2, 4), (3, 0)) j(id, copy) ON r.id = j.id +GROUP BY r.id; +---- +[30, 10, 100] + +# The consumer may reuse a defining FROM alias for an unrelated joined input. +statement ok +CREATE OR REPLACE VIEW measure_window_alias_order AS +SELECT t.id, LIST(t.amount ORDER BY t.id DESC) AS MEASURE amounts FROM measure_window_rows t; + +query T +SELECT DISTINCT AGGREGATE(amounts ORDER BY t.copy) OVER () +FROM measure_window_alias_order r +JOIN (VALUES (1, 2), (2, 1), (3, 0)) t(id, copy) ON r.id = t.id; +---- +[100, 30, 10] + +query T +SELECT DISTINCT AGGREGATE(amounts ORDER BY t.id ASC) OVER () +FROM measure_window_alias_order r CROSS JOIN (VALUES (5), (2)) t(id); +---- +[60, 200, 200, 100, 30, 10] + +# Generated lineage CTEs must not shadow user CTEs, including case-insensitive +# collisions and names that already use the generated fallback suffix. +query III +WITH "__YS_WINDOW_SOURCE_0_BASE" AS (SELECT 7 AS n), + __ys_window_source_0_base_1 AS (SELECT 9 AS n) +SELECT AGGREGATE(revenue) OVER (), a.n, b.n +FROM measure_window_raw +CROSS JOIN "__YS_WINDOW_SOURCE_0_BASE" a +CROSS JOIN __ys_window_source_0_base_1 b LIMIT 1; +---- +600 7 9 + +# Enclosing CTEs remain available to consumers of the rewritten window query. +query III +WITH "__YS_WINDOW_SOURCE_0_BASE" AS (SELECT 7 AS n), + __ys_window_source_0_base_1 AS (SELECT 9 AS n), + totals AS (SELECT AGGREGATE(revenue) OVER () AS total FROM measure_window_raw LIMIT 1) +SELECT total, a.n, b.n FROM totals +CROSS JOIN "__YS_WINDOW_SOURCE_0_BASE" a +CROSS JOIN __ys_window_source_0_base_1 b; +---- +600 7 9 + +# Equivalent SET spellings retain the same precedence as repeated identical +# spellings. Keep the original dimension SQL for binding quoted identifiers. +query III +SELECT + AGGREGATE(revenue) OVER () AT (SET yr = 2022) AT (SET yr = 2023), + AGGREGATE(revenue) OVER () AT (SET yr = 2022) AT (SET v.yr = 2023), + AGGREGATE(revenue) OVER () AT (SET v."YR" = 2022) AT (SET yr = 2023) +FROM measure_window_grouped v WHERE region = 'A' LIMIT 1; +---- +40 40 40 + +query III +SELECT + AGGREGATE(revenue) OVER () AT (SET yr = 2023) AT (SET yr = 2022), + AGGREGATE(revenue) OVER () AT (SET v.yr = 2023) AT (SET yr = 2022), + AGGREGATE(revenue) OVER () AT (SET yr = 2023) AT (SET v."YR" = 2022) +FROM measure_window_grouped v WHERE region = 'A' LIMIT 1; +---- +100 100 100 + +# Schema probes resolve enclosing CTE definitions and their column aliases. +query ITI +WITH chosen(row_id, label) AS (VALUES (1, 'a'), (2, 'a')) +SELECT c.*, AGGREGATE(revenue) OVER () +FROM measure_window_raw r JOIN chosen c ON r.id = c.row_id +ORDER BY c.row_id; +---- +1 a 40 +2 a 40 + +# Earlier producers are lowered before their consumers inspect their schema. +query II +WITH totals AS (SELECT AGGREGATE(revenue) AS total FROM measure_window_raw), + copied AS (SELECT total FROM totals) +SELECT c.total, AGGREGATE(revenue) OVER () +FROM measure_window_raw CROSS JOIN copied c LIMIT 1; +---- +600 600 + +# A dependency retains the CTE visible at its definition, even when a nested +# scope shadows that name before consuming the dependency. +query II +WITH chosen AS (SELECT 1 AS id), dependency AS (SELECT * FROM chosen) +SELECT * FROM ( + WITH chosen AS (SELECT 2 AS id) + SELECT AGGREGATE(revenue) OVER () AS total, c.id + FROM measure_window_raw r JOIN dependency d ON r.id = d.id + CROSS JOIN chosen c +) nested; +---- +10 2 + +query I +WITH RECURSIVE chosen(id) AS (VALUES (1) UNION ALL SELECT id + 1 FROM chosen WHERE id < 2) +SELECT DISTINCT AGGREGATE(revenue) OVER () +FROM measure_window_raw r JOIN chosen c ON r.id = c.id; +---- +40 + +# Lazy schema binding must not force an unused invalid definition. +query I +WITH unused AS (SELECT * FROM measure_window_missing_relation), chosen AS (SELECT 1 AS id) +SELECT AGGREGATE(revenue) OVER () +FROM measure_window_raw r JOIN chosen c ON r.id = c.id; +---- +10 + +# Correlated FILTER subqueries use the same enclosing CTE context. +query I +WITH chosen AS (SELECT 1 AS id) +SELECT DISTINCT AGGREGATE(revenue) FILTER ( + WHERE EXISTS (SELECT 1 FROM chosen c WHERE c.id = r.id) +) OVER () FROM measure_window_raw r; +---- +10 + +query I +WITH chosen AS (SELECT 1 AS id) +SELECT DISTINCT AGGREGATE(revenue) OVER () AT ( + WHERE EXISTS (SELECT 1 FROM chosen c WHERE c.id = r.id) +) FROM measure_window_raw r; +---- +10 + +query T +WITH chosen(id, priority) AS (VALUES (1, 2), (2, 1), (3, 0)) +SELECT DISTINCT AGGREGATE(amounts ORDER BY ( + SELECT priority FROM chosen c WHERE c.id = r.id +)) OVER () FROM measure_window_raw r WHERE id <= 3; +---- +[100, 30, 10] + +# Binding definitions must not leak into runtime SQL and evaluate a shared +# materialized CTE again at a window or scalar-subquery reference. +statement ok +CREATE OR REPLACE SEQUENCE measure_window_cte_sequence START 1; + +query III +WITH token AS MATERIALIZED (SELECT nextval('measure_window_cte_sequence') AS value) +SELECT AGGREGATE(revenue) OVER (), t.value, (SELECT value FROM token) +FROM measure_window_raw CROSS JOIN token t LIMIT 1; +---- +600 1 1 + +query I +SELECT nextval('measure_window_cte_sequence'); +---- +2 + +endloop + +statement ok +SET heap_based_parser=true; diff --git a/test/sql/native_combined_grammar.test b/test/sql/native_combined_grammar.test index 85392d1..53c9baa 100644 --- a/test/sql/native_combined_grammar.test +++ b/test/sql/native_combined_grammar.test @@ -22,13 +22,26 @@ statement ok INSERT INTO combined_sales VALUES (2022, 100), (2023, 225); statement ok -SET active_grammar_extensions=['yardstick', 'yardstick_test_grammar']; +CREATE VIEW combined_plain_v AS SELECT SUM(amount) AS MEASURE revenue FROM combined_sales; foreach parser_mode true false statement ok SET heap_based_parser=${parser_mode}; +# A selected grammar without Yardstick rules must not be replaced by the +# extension's private grammar merely because Yardstick is loaded. +statement ok +SET active_grammar_extensions=['yardstick_test_grammar']; + +query I +SELECT yardstick_test_year!(); +---- +2023 + +statement ok +SET active_grammar_extensions=['yardstick', 'yardstick_test_grammar']; + query I SELECT yardstick_test_year!(); ---- @@ -79,6 +92,11 @@ SELECT yardstick_test_year!(); statement ok RESET active_grammar_extensions; +query I +SELECT AGGREGATE(revenue) FROM combined_plain_v; +---- +325 + statement error SELECT yardstick_test_year!(); ---- diff --git a/yardstick-rs/src/parser_ffi.rs b/yardstick-rs/src/parser_ffi.rs index b7f4002..29c546e 100644 --- a/yardstick-rs/src/parser_ffi.rs +++ b/yardstick-rs/src/parser_ffi.rs @@ -60,6 +60,9 @@ pub struct YardstickAggregateCall { pub end_pos: u32, pub modifiers: *mut YardstickAtModifier, pub modifier_count: usize, + pub call_sql: *const c_char, + pub is_window: bool, + pub has_decorations: bool, } /// List of AGGREGATE() calls found in SQL @@ -72,6 +75,43 @@ pub struct YardstickAggregateCallList { pub native_parsed: bool, } +#[repr(C)] +pub struct YardstickWindowSource { + key: *const c_char, + relation_name: *const c_char, + alias: *const c_char, + clean_select_sql: *const c_char, + grouped: bool, + dimension_names: *const *const c_char, + dimension_expressions: *const *const c_char, + dimension_count: usize, +} + +#[repr(C)] +pub struct YardstickWindowCall { + marker_name: *const c_char, + source_key: *const c_char, + expression_sql: *const c_char, + modifiers: *const YardstickAtModifier, + modifier_count: usize, +} + +pub struct WindowSource { + pub key: String, + pub relation_name: String, + pub alias: String, + pub clean_select_sql: String, + pub grouped: bool, + pub dimensions: std::collections::HashMap, +} + +pub struct WindowCall { + pub marker_name: String, + pub source_key: String, + pub expression_sql: String, + pub modifiers: Vec, +} + /// Information about a single SELECT item #[repr(C)] #[derive(Debug)] @@ -84,6 +124,7 @@ pub struct YardstickSelectItem { pub is_star: bool, pub is_measure_ref: bool, pub contains_subquery: bool, + pub contains_window: bool, pub reference_column: *const c_char, pub reference_qualifier: *const c_char, pub subquery_dimensions: *const *const c_char, @@ -100,12 +141,21 @@ pub struct YardstickTableRef { pub schema_qualified: bool, } +#[repr(C)] +#[derive(Clone, Copy)] +pub struct YardstickCteDefinition { + pub start_pos: u32, + pub end_pos: u32, + pub recursive: bool, +} + #[repr(C)] pub struct YardstickQueryScope { pub start_pos: u32, pub end_pos: u32, pub visible_ctes: *const *const c_char, pub visible_cte_count: usize, + pub cte_definitions: *const YardstickCteDefinition, } #[repr(C)] @@ -118,10 +168,31 @@ pub struct QueryScope { pub start: usize, pub end: usize, pub visible_ctes: Vec, + pub cte_definitions: Vec, } thread_local! { static QUERY_CTES: RefCell> = const { RefCell::new(Vec::new()) }; + static QUERY_BINDING_CTES: RefCell> = const { RefCell::new(Vec::new()) }; +} + +/// Binding-only definitions retain their declaration order and lexical scope. +pub struct BindingCteScopeGuard(Vec); + +impl BindingCteScopeGuard { + pub fn enter(ctes: &[String]) -> Self { + Self(QUERY_BINDING_CTES.with(|current| { + let previous = current.borrow().clone(); + current.borrow_mut().extend_from_slice(ctes); + previous + })) + } +} + +impl Drop for BindingCteScopeGuard { + fn drop(&mut self) { + QUERY_BINDING_CTES.with(|current| current.replace(std::mem::take(&mut self.0))); + } } /// Carry enclosing CTE visibility when a native query body is lowered alone. @@ -261,6 +332,9 @@ pub fn find_query_scopes(sql: &str) -> Option> { start: scope.start_pos as usize, end: scope.end_pos as usize, visible_ctes, + cte_definitions: (0..scope.visible_cte_count) + .map(|cte| *scope.cte_definitions.add(cte)) + .collect(), }); } free(list); @@ -418,9 +492,21 @@ type FnParseCreateView = unsafe extern "C" fn(*const c_char) -> *mut YardstickCr type FnFreeCreateViewInfo = unsafe extern "C" fn(*mut YardstickCreateViewInfo); type FnReplaceRange = unsafe extern "C" fn(*const c_char, u32, u32, *const c_char) -> *mut c_char; type FnApplyReplacements = unsafe extern "C" fn(*const c_char, *const YardstickReplacement, usize) -> *mut c_char; -type FnQualifyExpression = unsafe extern "C" fn(*const c_char, *const c_char) -> *mut c_char; +type FnQualifyExpression = unsafe extern "C" fn(*const c_char, *const c_char, *const c_char) -> *mut c_char; type FnInlineOrderBySubqueryAliases = unsafe extern "C" fn(*const c_char) -> *mut c_char; type FnFreeString = unsafe extern "C" fn(*mut c_char); +type FnDecorateMeasure = unsafe extern "C" fn( + *const c_char, *const c_char, *const *const c_char, *const *const c_char, usize, + *const *const c_char, usize, *const *const c_char, usize, *mut *mut c_char, +) -> *mut c_char; +type FnWindowMarker = unsafe extern "C" fn(*const c_char, *const c_char, *mut *mut c_char) -> *mut c_char; +type FnRewriteMeasureWindows = unsafe extern "C" fn( + *const c_char, *const YardstickWindowSource, usize, *const YardstickWindowCall, usize, + *const *const c_char, usize, *const *const c_char, usize, *mut *mut c_char, +) -> *mut c_char; +static FN_DECORATE_MEASURE: AtomicPtr<()> = AtomicPtr::new(ptr::null_mut()); +static FN_WINDOW_MARKER: AtomicPtr<()> = AtomicPtr::new(ptr::null_mut()); +static FN_REWRITE_MEASURE_WINDOWS: AtomicPtr<()> = AtomicPtr::new(ptr::null_mut()); type FnExpandAggregateCall = unsafe extern "C" fn( *const c_char, *const c_char, *const YardstickAtModifier, usize, *const c_char, *const c_char, *const c_char, *const *const c_char, usize @@ -466,6 +552,9 @@ pub extern "C" fn yardstick_init_parser_ffi( find_query_scopes: FnFindQueryScopes, free_query_scopes: FnFreeQueryScopes, rewrite_visible_filter: FnRewriteVisibleFilter, + decorate_measure: FnDecorateMeasure, + window_marker: FnWindowMarker, + rewrite_measure_windows: FnRewriteMeasureWindows, ) { FN_FIND_AGGREGATES.store(find_aggregates as *mut (), Ordering::SeqCst); FN_FREE_AGGREGATE_LIST.store(free_aggregate_list as *mut (), Ordering::SeqCst); @@ -488,6 +577,236 @@ pub extern "C" fn yardstick_init_parser_ffi( FN_FIND_QUERY_SCOPES.store(find_query_scopes as *mut (), Ordering::SeqCst); FN_FREE_QUERY_SCOPES.store(free_query_scopes as *mut (), Ordering::SeqCst); FN_REWRITE_VISIBLE_FILTER.store(rewrite_visible_filter as *mut (), Ordering::SeqCst); + FN_DECORATE_MEASURE.store(decorate_measure as *mut (), Ordering::SeqCst); + FN_WINDOW_MARKER.store(window_marker as *mut (), Ordering::SeqCst); + FN_REWRITE_MEASURE_WINDOWS.store(rewrite_measure_windows as *mut (), Ordering::SeqCst); +} + +unsafe fn owned_rewrite_result(result: *mut c_char, error: *mut c_char) -> Result { + if !error.is_null() { + let message = CStr::from_ptr(error).to_string_lossy().into_owned(); + yardstick_free_string(error); + if !result.is_null() { + yardstick_free_string(result); + } + return Err(message); + } + if result.is_null() { + return Err("Native aggregate rewrite is unavailable".to_string()); + } + let sql = CStr::from_ptr(result).to_string_lossy().into_owned(); + yardstick_free_string(result); + Ok(sql) +} + +pub fn decorate_measure( + expression: &str, + call_sql: &str, + dimensions: &std::collections::HashMap, + qualifiers: &[String], +) -> Result { + let function = FN_DECORATE_MEASURE.load(Ordering::SeqCst); + if function.is_null() { + return Err("Native aggregate rewrite is unavailable".to_string()); + } + let string = |value: &str| CString::new(value).map_err(|error| error.to_string()); + let expression = string(expression)?; + let call_sql = string(call_sql)?; + let entries: Vec<_> = dimensions.iter().collect(); + let names = entries + .iter() + .map(|(name, _)| string(name)) + .collect::, _>>()?; + let values = entries + .iter() + .map(|(_, value)| string(value)) + .collect::, _>>()?; + let qualifiers = qualifiers + .iter() + .map(|value| string(value)) + .collect::, _>>()?; + let names: Vec<_> = names.iter().map(|value| value.as_ptr()).collect(); + let values: Vec<_> = values.iter().map(|value| value.as_ptr()).collect(); + let qualifier_ptrs: Vec<_> = qualifiers.iter().map(|value| value.as_ptr()).collect(); + let binding_strings = QUERY_BINDING_CTES.with(|ctes| { + ctes.borrow().iter().map(|query| string(query)).collect::, _>>() + })?; + let binding_queries: Vec<_> = binding_strings.iter().map(|query| query.as_ptr()).collect(); + unsafe { + let function: FnDecorateMeasure = std::mem::transmute(function); + let mut error = ptr::null_mut(); + let result = function( + expression.as_ptr(), + call_sql.as_ptr(), + names.as_ptr(), + values.as_ptr(), + names.len(), + qualifier_ptrs.as_ptr(), + qualifier_ptrs.len(), + binding_queries.as_ptr(), + binding_queries.len(), + &mut error, + ); + owned_rewrite_result(result, error) + } +} + +pub fn window_marker(call_sql: &str, marker: &str) -> Result { + let function = FN_WINDOW_MARKER.load(Ordering::SeqCst); + if function.is_null() { + return Err("Native window rewrite is unavailable".to_string()); + } + let call_sql = CString::new(call_sql).map_err(|error| error.to_string())?; + let marker = CString::new(marker).map_err(|error| error.to_string())?; + unsafe { + let function: FnWindowMarker = std::mem::transmute(function); + let mut error = ptr::null_mut(); + let result = function(call_sql.as_ptr(), marker.as_ptr(), &mut error); + owned_rewrite_result(result, error) + } +} + +pub fn rewrite_measure_windows( + sql: &str, + sources: &[WindowSource], + calls: &[WindowCall], +) -> Result { + let function = FN_REWRITE_MEASURE_WINDOWS.load(Ordering::SeqCst); + if function.is_null() { + return Err("Native window rewrite is unavailable".to_string()); + } + let string = |value: &str| CString::new(value).map_err(|error| error.to_string()); + let sql = string(sql)?; + struct SourceStrings { + key: CString, + relation: CString, + alias: CString, + query: CString, + names: Vec, + expressions: Vec, + } + let cte_strings = QUERY_CTES.with(|ctes| { + ctes.borrow().iter().map(|name| string(name)).collect::, _>>() + })?; + let cte_names: Vec<_> = cte_strings.iter().map(|name| name.as_ptr()).collect(); + let binding_strings = QUERY_BINDING_CTES.with(|ctes| { + ctes.borrow().iter().map(|query| string(query)).collect::, _>>() + })?; + let binding_queries: Vec<_> = binding_strings.iter().map(|query| query.as_ptr()).collect(); + let mut source_strings = Vec::new(); + for source in sources { + let entries: Vec<_> = source.dimensions.iter().collect(); + source_strings.push(SourceStrings { + key: string(&source.key)?, + relation: string(&source.relation_name)?, + alias: string(&source.alias)?, + query: string(&source.clean_select_sql)?, + names: entries + .iter() + .map(|(name, _)| string(name)) + .collect::>()?, + expressions: entries + .iter() + .map(|(_, expression)| string(expression)) + .collect::>()?, + }); + } + let dimension_names: Vec> = source_strings + .iter() + .map(|source| source.names.iter().map(|name| name.as_ptr()).collect()) + .collect(); + let dimension_expressions: Vec> = source_strings + .iter() + .map(|source| { + source + .expressions + .iter() + .map(|expression| expression.as_ptr()) + .collect() + }) + .collect(); + let source_info: Vec<_> = source_strings + .iter() + .enumerate() + .map(|(index, source)| YardstickWindowSource { + key: source.key.as_ptr(), + relation_name: source.relation.as_ptr(), + alias: source.alias.as_ptr(), + clean_select_sql: source.query.as_ptr(), + grouped: sources[index].grouped, + dimension_names: dimension_names[index].as_ptr(), + dimension_expressions: dimension_expressions[index].as_ptr(), + dimension_count: source.names.len(), + }) + .collect(); + struct CallStrings { + marker: CString, + source: CString, + expression: CString, + dimensions: Vec, + values: Vec, + } + let mut call_strings = Vec::new(); + for call in calls { + call_strings.push(CallStrings { + marker: string(&call.marker_name)?, + source: string(&call.source_key)?, + expression: string(&call.expression_sql)?, + dimensions: call + .modifiers + .iter() + .map(|modifier| string(modifier.dimension.as_deref().unwrap_or(""))) + .collect::>()?, + values: call + .modifiers + .iter() + .map(|modifier| string(modifier.value.as_deref().unwrap_or(""))) + .collect::>()?, + }); + } + let modifiers: Vec> = calls + .iter() + .zip(&call_strings) + .map(|(call, strings)| { + call.modifiers + .iter() + .enumerate() + .map(|(index, modifier)| YardstickAtModifier { + at_type: modifier.modifier_type.clone().into(), + dimension: strings.dimensions[index].as_ptr(), + value: strings.values[index].as_ptr(), + }) + .collect() + }) + .collect(); + let call_info: Vec<_> = call_strings + .iter() + .enumerate() + .map(|(index, call)| YardstickWindowCall { + marker_name: call.marker.as_ptr(), + source_key: call.source.as_ptr(), + expression_sql: call.expression.as_ptr(), + modifiers: modifiers[index].as_ptr(), + modifier_count: modifiers[index].len(), + }) + .collect(); + unsafe { + let function: FnRewriteMeasureWindows = std::mem::transmute(function); + let mut error = ptr::null_mut(); + let result = function( + sql.as_ptr(), + source_info.as_ptr(), + source_info.len(), + call_info.as_ptr(), + call_info.len(), + cte_names.as_ptr(), + cte_names.len(), + binding_queries.as_ptr(), + binding_queries.len(), + &mut error, + ); + owned_rewrite_result(result, error) + } } // Helper macros to call function pointers @@ -669,6 +988,9 @@ pub struct AggregateCall { pub start_pos: u32, pub end_pos: u32, pub modifiers: Vec, + pub call_sql: Option, + pub is_window: bool, + pub has_decorations: bool, } /// Safe wrapper for SELECT item information @@ -682,6 +1004,7 @@ pub struct SelectItem { pub is_star: bool, pub is_measure_ref: bool, pub contains_subquery: bool, + pub contains_window: bool, pub reference_column: Option, pub reference_qualifier: Option, pub subquery_dimensions: Vec, @@ -869,6 +1192,9 @@ pub(crate) fn find_aggregates_with_source( start_pos: call.start_pos, end_pos: call.end_pos, modifiers, + call_sql: c_str_to_string(call.call_sql), + is_window: call.is_window, + has_decorations: call.has_decorations, }); } @@ -922,6 +1248,7 @@ pub fn parse_select(sql: &str) -> Result { is_star: item.is_star, is_measure_ref: item.is_measure_ref, contains_subquery: item.contains_subquery, + contains_window: item.contains_window, reference_column: c_str_to_string(item.reference_column), reference_qualifier: c_str_to_string(item.reference_qualifier), subquery_dimensions: (0..item.subquery_dimension_count) @@ -1169,7 +1496,7 @@ pub fn qualify_expression(expr: &str, qualifier: &str) -> Result unsafe { let f: FnQualifyExpression = std::mem::transmute(fn_ptr); - let result_ptr = f(expr_ptr.as_ptr(), qualifier_ptr.as_ptr()); + let result_ptr = f(expr_ptr.as_ptr(), qualifier_ptr.as_ptr(), ptr::null()); if result_ptr.is_null() { return Err("Failed to qualify expression".to_string()); } @@ -1179,6 +1506,26 @@ pub fn qualify_expression(expr: &str, qualifier: &str) -> Result } } +pub fn qualify_outer_dimension(expr: &str, qualifier: &str, dimension: &str) -> Option { + let function = FN_QUALIFY_EXPRESSION.load(Ordering::SeqCst); + if function.is_null() { + return None; + } + let expr = CString::new(expr).ok()?; + let qualifier = CString::new(qualifier).ok()?; + let dimension = CString::new(dimension).ok()?; + unsafe { + let function: FnQualifyExpression = std::mem::transmute(function); + let result = function(expr.as_ptr(), qualifier.as_ptr(), dimension.as_ptr()); + if result.is_null() { + return None; + } + let rewritten = c_str_to_string(result); + yardstick_free_string(result); + rewritten + } +} + pub fn inline_order_by_subquery_aliases(sql: &str) -> Option { let fn_ptr = FN_INLINE_ORDER_BY_SUBQUERY_ALIASES.load(Ordering::SeqCst); if fn_ptr.is_null() { diff --git a/yardstick-rs/src/sql/measures.rs b/yardstick-rs/src/sql/measures.rs index 7bf43b9..11c5c38 100644 --- a/yardstick-rs/src/sql/measures.rs +++ b/yardstick-rs/src/sql/measures.rs @@ -3535,6 +3535,9 @@ fn expand_derived_measure_expr(expr: &str, measure_view: &MeasureView) -> String /// Qualify dimension reference in expression for correlated subquery /// "year - 1" with table "sales" and dim "year" -> "sales.year - 1" pub fn qualify_outer_reference(expr: &str, table_name: &str, dim: &str) -> String { + if let Some(qualified) = parser_ffi::qualify_outer_dimension(expr, table_name, dim) { + return qualified; + } // Parse expression into tokens and replace matching identifiers let mut result = String::new(); let mut chars = expr.chars().peekable(); @@ -5164,7 +5167,7 @@ pub fn expand_aggregate(sql: &str) -> AggregateExpandResult { fn extract_dimension_columns_from_select_info(info: &SelectInfo) -> Vec { info.items .iter() - .filter(|item| !item.is_aggregate && !item.is_star && !item.is_measure_ref) + .filter(|item| !item.is_aggregate && !item.is_star && !item.is_measure_ref && !item.contains_window) .filter(|item| !is_literal_constant(&item.expression_sql)) .map(|item| { // Use alias if present, otherwise expression @@ -6717,7 +6720,24 @@ fn validate_set_expression_requirements( continue; } let dim_name = dim.split('.').next_back().unwrap_or(dim).trim(); - if expr_mentions_identifier_outside_current(expr, dim_name) + let probe_sql = match default_qualifier { + Some(qualifier) => format!("SELECT {expr} FROM {qualifier}"), + None => format!("SELECT {expr}"), + }; + let mentions_dimension = parser_ffi::parse_select(&probe_sql) + .ok() + .filter(|info| info.native_parsed) + .and_then(|info| info.items.into_iter().next()) + .filter(|item| item.contains_subquery) + .map(|item| { + // A scalar subquery owns its input columns. Only its + // outer dependencies require the consumer's grouping. + item.subquery_dimensions.iter().any(|dependency| { + expr_mentions_identifier_outside_current(dependency, dim_name) + }) + }) + .unwrap_or_else(|| expr_mentions_identifier_outside_current(expr, dim_name)); + if mentions_dimension && !dimension_in_group_by(dim, group_by_cols, default_qualifier) { return Some(format!( @@ -7466,18 +7486,6 @@ fn warning_for_at_all_ungrouped_where_with_qualifiers( /// Expand AGGREGATE() with AT modifiers in SQL pub fn expand_aggregate_with_at(sql: &str) -> AggregateExpandResult { - // Validate the complete statement before rewriting individual query scopes. - // This also covers calls in statement wrappers and nested expressions. - if let Err(error) = parser_ffi::find_aggregates_with_source(sql) { - if error.native_parsed { - return AggregateExpandResult { - had_aggregate: true, - expanded_sql: sql.to_string(), - error: Some(error.message), - warnings: Vec::new(), - }; - } - } if let Some(scopes) = parser_ffi::find_query_scopes(sql) { return expand_native_query_scopes(sql, &scopes); } @@ -7517,13 +7525,55 @@ fn expand_native_query_scopes(sql: &str, scopes: &[parser_ffi::QueryScope]) -> A let mut lowered: Vec = vec![String::new(); scopes.len()]; let mut had_aggregate = false; let mut warnings = Vec::new(); - for index in (0..scopes.len()).rev() { + // Lower children before parents, but keep sibling declarations in source + // order: a consumer's binding probes need its earlier CTEs already lowered. + let mut pending: Vec<_> = roots.iter().rev().map(|&index| (index, false)).collect(); + let mut order = Vec::with_capacity(scopes.len()); + while let Some((index, visited)) = pending.pop() { + if visited { + order.push(index); + } else { + pending.push((index, true)); + pending.extend(children[index].iter().rev().map(|&child| (child, false))); + } + } + for index in order { let scope = &scopes[index]; let mut query = sql[scope.start..scope.end].to_string(); for &child in children[index].iter().rev() { query.replace_range(scopes[child].start - scope.start..scopes[child].end - scope.start, &lowered[child]); } let _scope = parser_ffi::QueryScopeGuard::enter(&scope.visible_ctes); + let mut binding_ctes = Vec::new(); + for definition in &scope.cte_definitions { + let start = definition.start_pos as usize; + let end = definition.end_pos as usize; + let Some(declaration) = sql.get(start..end) else { + return AggregateExpandResult { + had_aggregate: true, expanded_sql: sql.to_string(), + error: Some("Invalid native CTE source range".to_string()), warnings, + }; + }; + let mut declaration = declaration.to_string(); + // Replace only the outermost completed scopes in this declaration; + // each replacement already contains its lowered descendants. + let mut replacements = Vec::new(); + let mut previous_end = start; + for (cte_index, cte_scope) in scopes.iter().enumerate() { + if cte_scope.start >= previous_end && cte_scope.end <= end && + !lowered[cte_index].is_empty() { + replacements.push(cte_index); + previous_end = cte_scope.end; + } + } + for cte_index in replacements.into_iter().rev() { + declaration.replace_range(scopes[cte_index].start - start..scopes[cte_index].end - start, + &lowered[cte_index]); + } + let recursive = if definition.recursive { "RECURSIVE " } else { "" }; + binding_ctes.push(format!("WITH {recursive}{declaration} SELECT 1")); + } + let _binding_scope = parser_ffi::BindingCteScopeGuard::enter(&binding_ctes); let expanded = expand_aggregate_query(&query, true); had_aggregate |= expanded.had_aggregate; for warning in expanded.warnings { @@ -7545,6 +7595,134 @@ fn expand_native_query_scopes(sql: &str, scopes: &[parser_ffi::QueryScope]) -> A AggregateExpandResult { had_aggregate, expanded_sql, error: None, warnings } } +fn decorated_recomputation_alias(mut sql: String, base_relation_sql: &str) -> String { + let suffix = format!(" FROM {})", base_relation_for_subquery(base_relation_sql)); + if sql.ends_with(&suffix) { + sql.truncate(sql.len() - 1); + sql.push_str(" _inner)"); + } + sql +} + +fn wrap_decorated_scalar_recomputation(sql: &str) -> String { + // The outer relation can be empty even though recomputing COUNT (or an + // exported state) over that context produces a value. Retain that result. + format!("COALESCE(ANY_VALUE({sql}), {sql})") +} + +fn expand_window_measure_query( + sql: &str, + native_calls: &[parser_ffi::AggregateCall], +) -> AggregateExpandResult { + let expanded = (|| -> std::result::Result { + let from = extract_from_clause_info(sql); + let primary = from + .primary_table + .as_ref() + .ok_or("Windowed AGGREGATE requires a measure relation")?; + let mut sources: Vec = Vec::new(); + let mut windows = Vec::new(); + let mut replacements = Vec::new(); + for call in native_calls.iter().filter(|call| call.is_window) { + let (qualifier, measure) = parse_simple_measure_ref(&call.measure_name) + .ok_or("Windowed AGGREGATE requires a measure reference")?; + let relation = qualifier + .as_ref() + .and_then(|qualifier| { + from.tables.values().find(|table| { + normalize_identifier_name(&table.effective_name) == *qualifier + }) + }) + .unwrap_or(primary); + let resolved = resolve_measure_source(&measure, &relation.name); + let view = get_measure_view(&resolved.source_view) + .ok_or_else(|| format!("Unknown window measure {}", call.measure_name))?; + let source_alias = if relation.name.eq_ignore_ascii_case(&resolved.source_view) { + relation.effective_name.clone() + } else { + find_alias_for_view(&from, &resolved.source_view) + .unwrap_or(&resolved.source_view) + .to_string() + }; + let source_key = source_alias.clone(); + if !sources.iter().any(|source| source.key == source_key) { + let select_sql = + extract_view_query(&view.base_query).unwrap_or(view.base_query.clone()); + sources.push(parser_ffi::WindowSource { + key: source_key.clone(), + relation_name: resolved.source_view.clone(), + alias: source_alias.clone(), + grouped: has_top_level_group_by(&select_sql), + clean_select_sql: view.base_query.clone(), + dimensions: resolved.dimension_exprs.clone(), + }); + } + let mut marker_index = windows.len(); + let marker = loop { + let candidate = format!("__yardstick_window_call_{marker_index}"); + if !sql.contains(&candidate) + && !windows + .iter() + .any(|call: &parser_ffi::WindowCall| call.marker_name == candidate) + { + break candidate; + } + marker_index += 1; + }; + let call_sql = call + .call_sql + .as_deref() + .ok_or("Missing native window expression")?; + let expression = resolved + .derived_expr + .as_deref() + .unwrap_or(&resolved.expression); + let expression_sql = parser_ffi::decorate_measure( + expression, + call_sql, + &resolved.dimension_exprs, + &[source_alias, resolved.source_view.clone()], + )?; + replacements.push(( + call.start_pos as usize, + call.end_pos as usize, + parser_ffi::window_marker(call_sql, &marker)?, + )); + windows.push(parser_ffi::WindowCall { + marker_name: marker, + source_key, + expression_sql, + modifiers: call.modifiers.clone(), + }); + } + replacements.sort_by(|left, right| right.0.cmp(&left.0)); + let mut marked_sql = sql.to_string(); + for (start, end, replacement) in replacements { + marked_sql.replace_range(start..end, &replacement); + } + // Ordinary measures bind against the original source and grouping before + // window staging introduces generated projections and lineage columns. + let ordinary = expand_aggregate_query(&marked_sql, true); + if let Some(error) = ordinary.error { + return Err(error); + } + let expanded_sql = + parser_ffi::rewrite_measure_windows(&ordinary.expanded_sql, &sources, &windows)?; + Ok(AggregateExpandResult { + had_aggregate: true, + expanded_sql, + error: None, + warnings: ordinary.warnings, + }) + })(); + expanded.unwrap_or_else(|error| AggregateExpandResult { + had_aggregate: true, + expanded_sql: sql.to_string(), + error: Some(error), + warnings: Vec::new(), + }) +} + fn expand_aggregate_query(sql: &str, native_scope: bool) -> AggregateExpandResult { let _filter_scope = FilterRewriteGuard::enter(); let mut expanded = expand_aggregate_query_impl(sql, native_scope); @@ -7597,6 +7775,13 @@ fn expand_aggregate_query_impl(sql: &str, native_scope: bool) -> AggregateExpand } // Check if we need the full expansion path (AT modifiers or non-decomposable measures) + if native_scope { + if let Ok((calls, true)) = parser_ffi::find_aggregates_with_source(&sql) { + if calls.iter().any(|call| call.is_window) { + return expand_window_measure_query(&sql, &calls); + } + } + } let has_aggregate = has_aggregate_function(&sql); // If no AGGREGATE function at all, nothing to do @@ -7610,6 +7795,13 @@ fn expand_aggregate_query_impl(sql: &str, native_scope: bool) -> AggregateExpand } had_aggregate = true; + let native_calls = match parser_ffi::find_aggregates_with_source(&sql) { + Ok((calls, _)) => calls, + Err(error) if error.native_parsed => return AggregateExpandResult { + had_aggregate: true, expanded_sql: sql, error: Some(error.message), warnings, + }, + Err(_) => Vec::new(), + }; let at_patterns = parse_aggregate_modifiers(&sql); // Keep full expansion path even without AT to handle non-decomposable measures safely @@ -7714,6 +7906,8 @@ fn expand_aggregate_query_impl(sql: &str, native_scope: bool) -> AggregateExpand let mut patterns = at_patterns; patterns.sort_by(|a, b| b.2.cmp(&a.2)); for (measure_name, modifiers, start, end) in patterns { + let decorated_call = native_calls.iter() + .find(|call| call.start_pos as usize == start && call.has_decorations && !call.is_window); let measure_lookup_name = strip_measure_qualifier(&measure_name); // Look up which view contains this measure (for JOIN support) let resolved = resolve_measure_source(&measure_lookup_name, &primary_table_name); @@ -7805,6 +7999,18 @@ fn expand_aggregate_query_impl(sql: &str, native_scope: bool) -> AggregateExpand .derived_expr .clone() .unwrap_or_else(|| resolved.expression.clone()); + let expression_for_eval = if let Some(call) = decorated_call { + let qualifiers = allowed_qualifiers.iter().cloned().collect::>(); + match parser_ffi::decorate_measure( + &expression_for_eval, call.call_sql.as_deref().unwrap_or_default(), + &resolved.dimension_exprs, &qualifiers, + ) { + Ok(expression) => expression, + Err(error) => return AggregateExpandResult { + had_aggregate: true, expanded_sql: result_sql, error: Some(error), warnings, + }, + } + } else { expression_for_eval }; let is_window_measure = is_window_expression(&expression_for_eval) || is_window_expression(&resolved.expression); @@ -7833,8 +8039,15 @@ fn expand_aggregate_query_impl(sql: &str, native_scope: bool) -> AggregateExpand } }; let eval_sql = wrap_window_rows_as_single_value(&row_eval_sql, &measure_lookup_name); + let eval_sql = if decorated_call.is_some() { + decorated_recomputation_alias(eval_sql, &base_relation_sql) + } else { eval_sql }; if original_dim_cols.is_empty() { - format!("MAX({eval_sql})") + if decorated_call.is_some() { + wrap_decorated_scalar_recomputation(&eval_sql) + } else { + format!("MAX({eval_sql})") + } } else { eval_sql } @@ -7848,8 +8061,15 @@ fn expand_aggregate_query_impl(sql: &str, native_scope: bool) -> AggregateExpand &modifiers, &resolved.dimension_exprs, ); + let eval_sql = if decorated_call.is_some() { + decorated_recomputation_alias(eval_sql, &base_relation_sql) + } else { eval_sql }; if original_dim_cols.is_empty() { - format!("MAX({eval_sql})") + if decorated_call.is_some() { + wrap_decorated_scalar_recomputation(&eval_sql) + } else { + format!("MAX({eval_sql})") + } } else { eval_sql } @@ -7868,10 +8088,13 @@ fn expand_aggregate_query_impl(sql: &str, native_scope: bool) -> AggregateExpand } // Also expand plain AGGREGATE() calls (without AT modifiers) using text replacement + let plain_native_calls = parser_ffi::find_aggregates(&result_sql).unwrap_or_default(); let mut plain_calls = extract_all_aggregate_calls(&result_sql); plain_calls.sort_by(|a, b| b.1.cmp(&a.1)); // Sort by position descending for (measure_name, start, end) in plain_calls { + let decorated_call = plain_native_calls.iter() + .find(|call| call.start_pos as usize == start && call.has_decorations && !call.is_window); let mut replacement_end = end; let mut use_default_context = false; let suffix = &result_sql[end..]; @@ -7952,6 +8175,18 @@ fn expand_aggregate_query_impl(sql: &str, native_scope: bool) -> AggregateExpand .derived_expr .clone() .unwrap_or_else(|| resolved.expression.clone()); + let expression_for_eval = if let Some(call) = decorated_call { + let qualifiers = allowed_qualifiers.iter().cloned().collect::>(); + match parser_ffi::decorate_measure( + &expression_for_eval, call.call_sql.as_deref().unwrap_or_default(), + &resolved.dimension_exprs, &qualifiers, + ) { + Ok(expression) => expression, + Err(error) => return AggregateExpandResult { + had_aggregate: true, expanded_sql: result_sql, error: Some(error), warnings, + }, + } + } else { expression_for_eval }; let is_window_measure = is_window_expression(&expression_for_eval) || is_window_expression(&resolved.expression); @@ -7982,8 +8217,15 @@ fn expand_aggregate_query_impl(sql: &str, native_scope: bool) -> AggregateExpand &resolved.dimension_exprs, ) }; + let eval_sql = if decorated_call.is_some() { + decorated_recomputation_alias(eval_sql, &base_relation_sql) + } else { eval_sql }; if original_dim_cols.is_empty() { - format!("MAX({eval_sql})") + if decorated_call.is_some() { + wrap_decorated_scalar_recomputation(&eval_sql) + } else { + format!("MAX({eval_sql})") + } } else { eval_sql } @@ -8318,7 +8560,7 @@ fn extract_dimension_columns_from_select(sql: &str) -> Vec { if let Ok(info) = parser_ffi::parse_select(sql) { if info.native_parsed { return info.items.into_iter() - .filter(|item| !item.is_aggregate && !item.is_star && !item.is_measure_ref) + .filter(|item| !item.is_aggregate && !item.is_star && !item.is_measure_ref && !item.contains_window) .filter(|item| !is_literal_constant(&item.expression_sql)) .flat_map(|item| { if item.contains_subquery {