diff --git a/packages/uipath/pyproject.toml b/packages/uipath/pyproject.toml index 035dac2d1..2ca26e087 100644 --- a/packages/uipath/pyproject.toml +++ b/packages/uipath/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "uipath" -version = "2.13.6" +version = "2.13.7" description = "Python SDK and CLI for UiPath Platform, enabling programmatic interaction with automation services, process management, and deployment tools." readme = { file = "README.md", content-type = "text/markdown" } requires-python = ">=3.11" diff --git a/packages/uipath/src/uipath/eval/evaluators/_aggregator_specs.py b/packages/uipath/src/uipath/eval/evaluators/_aggregator_specs.py new file mode 100644 index 000000000..d3d44cc36 --- /dev/null +++ b/packages/uipath/src/uipath/eval/evaluators/_aggregator_specs.py @@ -0,0 +1,62 @@ +"""Aggregator specs embedded in per-datapoint classification evaluator configs. + +Each aggregator is a run-level metric (precision / recall / f-score) attached +to a classification evaluator. Classes are declared once on the parent evaluator +config — every aggregator on the same evaluator operates on the same class +vocabulary, so the field is not repeated per spec. Only the metric-shape fields +(``averaging`` and, for fscore, ``f_value``) live on the spec itself. +""" + +from __future__ import annotations + +from typing import Annotated, Literal, Union + +from pydantic import BaseModel, ConfigDict, Field +from pydantic.alias_generators import to_camel + + +class _AggregatorSpecBase(BaseModel): + """Shared pydantic config for every aggregator variant.""" + + model_config = ConfigDict(alias_generator=to_camel, populate_by_name=True) + + +class PrecisionAggregatorSpec(_AggregatorSpecBase): + """Run-level precision aggregator (multiclass, micro or macro averaged).""" + + type: Literal["precision"] = "precision" + averaging: Literal["macro", "micro"] + + +class RecallAggregatorSpec(_AggregatorSpecBase): + """Run-level recall aggregator (multiclass, micro or macro averaged).""" + + type: Literal["recall"] = "recall" + averaging: Literal["macro", "micro"] + + +class FScoreAggregatorSpec(_AggregatorSpecBase): + """Run-level F-beta aggregator (multiclass, micro or macro averaged).""" + + type: Literal["fscore"] = "fscore" + averaging: Literal["macro", "micro"] + # Upper bound keeps beta² finite — a huge beta overflows to inf and the + # F-score becomes NaN, which is not representable in JSON. + f_value: float = Field(default=1.0, gt=0, le=1000) + + +class ConfusionMatrixAggregatorSpec(_AggregatorSpecBase): + """Run-level raw k×k confusion matrix — no scalar headline, no averaging.""" + + type: Literal["confusion_matrix"] = "confusion_matrix" + + +AggregatorSpec = Annotated[ + Union[ + PrecisionAggregatorSpec, + RecallAggregatorSpec, + FScoreAggregatorSpec, + ConfusionMatrixAggregatorSpec, + ], + Field(discriminator="type"), +] diff --git a/packages/uipath/src/uipath/eval/evaluators/base_dataset_evaluator.py b/packages/uipath/src/uipath/eval/evaluators/base_dataset_evaluator.py new file mode 100644 index 000000000..5f302b076 --- /dev/null +++ b/packages/uipath/src/uipath/eval/evaluators/base_dataset_evaluator.py @@ -0,0 +1,45 @@ +"""Base abstractions for dataset-level evaluators. + +A dataset-level evaluator runs once per evaluation set, after all per-datapoint +evaluators have produced their results. It consumes the per-datapoint +EvaluationResultDto values from one named source evaluator and emits a single +EvaluationResult that summarizes the dataset. + +Concretely distinct from GenericBaseEvaluator: different evaluate() signature, +different lifecycle. Kept as a parallel hierarchy rather than a subclass so the +runtime cannot accidentally dispatch a dataset evaluator through the +per-datapoint loop. +""" + +from __future__ import annotations + +from abc import ABC, abstractmethod + +from ..models.models import EvaluationResult, EvaluationResultDto +from ._aggregator_specs import AggregatorSpec + + +class BaseDatasetEvaluator(ABC): + """Abstract base for dataset-level evaluators. + + Constructed from an :class:`AggregatorSpec`, the source evaluator's name, + and the class vocabulary of the parent per-datapoint evaluator. Classes + live on the evaluator config (not the spec) — every aggregator on the same + evaluator operates on the same vocabulary. + """ + + spec: AggregatorSpec + source_evaluator: str + classes: list[str] + + def __init__( + self, spec: AggregatorSpec, source_evaluator: str, classes: list[str] + ) -> None: + """Store the aggregator spec, source evaluator name, and shared classes.""" + self.spec = spec + self.source_evaluator = source_evaluator + self.classes = classes + + @abstractmethod + def evaluate(self, results: list[EvaluationResultDto]) -> EvaluationResult: + """Reduce per-datapoint results into a single run-level EvaluationResult.""" diff --git a/packages/uipath/src/uipath/eval/evaluators/base_evaluator.py b/packages/uipath/src/uipath/eval/evaluators/base_evaluator.py index ec4e0a7e6..9bb4a4a6e 100644 --- a/packages/uipath/src/uipath/eval/evaluators/base_evaluator.py +++ b/packages/uipath/src/uipath/eval/evaluators/base_evaluator.py @@ -47,6 +47,22 @@ class BaseEvaluatorJustification(BaseModel): expected: str actual: str + @classmethod + def try_from(cls, details: object) -> "BaseEvaluatorJustification | None": + """Coerce a free-form details payload into a justification, or return None. + + Accepts either an existing instance or a dict that ``model_validate`` can + parse. Anything else (str, None, malformed dict) yields ``None``. + """ + if isinstance(details, cls): + return details + if isinstance(details, dict): + try: + return cls.model_validate(details) + except Exception: + return None + return None + # Additional type variables for Config and Justification # Note: C must be BaseEvaluatorConfig[T] to ensure type consistency diff --git a/packages/uipath/src/uipath/eval/evaluators/classification_dataset_evaluators.py b/packages/uipath/src/uipath/eval/evaluators/classification_dataset_evaluators.py new file mode 100644 index 000000000..061982783 --- /dev/null +++ b/packages/uipath/src/uipath/eval/evaluators/classification_dataset_evaluators.py @@ -0,0 +1,261 @@ +"""Dataset-level classification evaluators: Precision, Recall, F-score, Confusion Matrix. + +All variants share the same internal machinery — a k x k confusion matrix built +from each per-datapoint result's BaseEvaluatorJustification (expected, actual) +strings. The scalar variants (precision / recall / fscore) emit per-class +metrics plus micro/macro averages and pick the headline ``score`` per the +spec's ``averaging``; the ``confusion_matrix`` variant emits only the raw grid +with a 0.0 placeholder score. + +The ``details`` payload is the platform wire contract: the Agents reducer +worker (python-dataset-eval-worker) calls this evaluator and ships +``details.model_dump(by_alias=True, exclude_none=True)`` verbatim to the C# +backend, where the frontend's zod schema +(frontend-sw/src/schemas/evaluations/evals.ts) validates it. Changing field +names or shapes here is a cross-repo breaking change. +""" + +from __future__ import annotations + +from dataclasses import dataclass + +from pydantic import BaseModel, ConfigDict, Field +from pydantic.alias_generators import to_camel + +from ..models.models import ( + EvaluationResult, + EvaluationResultDto, + NumericEvaluationResult, +) +from ._aggregator_specs import ( + ConfusionMatrixAggregatorSpec, + FScoreAggregatorSpec, +) +from .base_dataset_evaluator import BaseDatasetEvaluator +from .base_evaluator import BaseEvaluatorJustification + + +class PerClassMetrics(BaseModel): + """Per-class confusion counts plus all three scalar metrics for that class.""" + + model_config = ConfigDict(alias_generator=to_camel, populate_by_name=True) + + tp: int + tn: int + fp: int + fn: int + support: int + precision: float + recall: float + f_score: float + + +class AveragedMetrics(BaseModel): + """Micro- or macro-averaged precision / recall / F-score triple.""" + + model_config = ConfigDict(alias_generator=to_camel, populate_by_name=True) + + precision: float + recall: float + f_score: float + + +class ClassificationDetails(BaseModel): + """Structured details payload emitted by every classification aggregator. + + The scalar metrics (precision / recall / fscore) populate every field; + the ``confusion_matrix`` variant emits only the grid + counts, leaving + ``averaging`` / ``f_value`` / ``per_class`` / ``macro`` / ``micro`` as + None (excluded from the wire dump). + """ + + model_config = ConfigDict(alias_generator=to_camel, populate_by_name=True) + + metric: str + classes: list[str] + confusion_matrix: list[list[int]] = Field( + ..., + description=( + "k x k confusion matrix indexed as " + "``confusion_matrix[predicted_idx][expected_idx]`` " + "(rows are predicted classes, columns are expected). " + "This is the transpose of sklearn's convention " + "(``[true][predicted]``); UI / consumer code must use the " + "orientation documented here." + ), + ) + n_total: int + n_scored: int + n_skipped: int + averaging: str | None = None + f_value: float | None = None + per_class: dict[str, PerClassMetrics] | None = None + macro: AveragedMetrics | None = None + micro: AveragedMetrics | None = None + + +@dataclass(slots=True) +class _ConfusionData: + """Internal: confusion matrix and per-class counts derived from results.""" + + classes: list[str] + matrix: list[list[int]] + n_total: int + n_scored: int + n_skipped: int + + +def _build_confusion( + results: list[EvaluationResultDto], + classes: list[str], +) -> _ConfusionData: + """Build a confusion matrix from per-datapoint results. + + Results without a parseable justification are counted in ``n_skipped`` and + omitted from the matrix. Pairs whose expected or actual label isn't in + ``classes`` are also skipped. Labels are normalized to lowercase for the + lookup index so a classifier returning "Book" vs configured "book" still + matches, but the user-supplied casing is preserved in the returned + ``_ConfusionData.classes`` so downstream output (per_class keys, UI labels) + shows what the user typed. + """ + index_of = {c.lower(): i for i, c in enumerate(classes)} + k = len(classes) + matrix = [[0] * k for _ in range(k)] + + n_total = len(results) + n_scored = 0 + n_skipped = 0 + + for r in results: + j = BaseEvaluatorJustification.try_from(r.details) + if j is None: + n_skipped += 1 + continue + exp = j.expected.lower() + act = j.actual.lower() + if exp not in index_of or act not in index_of: + n_skipped += 1 + continue + matrix[index_of[act]][index_of[exp]] += 1 + n_scored += 1 + + return _ConfusionData( + classes=list(classes), + matrix=matrix, + n_total=n_total, + n_scored=n_scored, + n_skipped=n_skipped, + ) + + +def _f_beta(precision: float, recall: float, beta: float) -> float: + if precision == 0.0 and recall == 0.0: + return 0.0 + b2 = beta * beta + denom = b2 * precision + recall + if denom == 0: + return 0.0 + return (1 + b2) * precision * recall / denom + + +class ClassificationDatasetEvaluator(BaseDatasetEvaluator): + """One implementation for all classification aggregators. + + Scalar variants (precision / recall / fscore) compute the full per-class + P/R/F report and pick the headline by the spec's ``averaging``; the + ``confusion_matrix`` variant returns only the raw grid. + """ + + def evaluate(self, results: list[EvaluationResultDto]) -> EvaluationResult: + """Compute the configured metric report and return the headline as score.""" + confusion = _build_confusion(results, self.classes) + + if isinstance(self.spec, ConfusionMatrixAggregatorSpec): + # No scalar headline — emit the raw grid and let the UI render it. + details = ClassificationDetails( + metric=self.spec.type, + classes=confusion.classes, + confusion_matrix=confusion.matrix, + n_total=confusion.n_total, + n_scored=confusion.n_scored, + n_skipped=confusion.n_skipped, + ) + return NumericEvaluationResult(score=0.0, details=details) + + f_value = ( + self.spec.f_value if isinstance(self.spec, FScoreAggregatorSpec) else 1.0 + ) + k = len(confusion.classes) + + per_class: dict[str, PerClassMetrics] = {} + precisions: list[float] = [] + recalls: list[float] = [] + f_scores: list[float] = [] + total_tp = total_fp = total_fn = 0 + + for c, label in enumerate(confusion.classes): + tp = confusion.matrix[c][c] + row_sum = sum(confusion.matrix[c]) # predicted as `label` + col_sum = sum(confusion.matrix[j][c] for j in range(k)) # true `label` + fp = row_sum - tp + fn = col_sum - tp + tn = confusion.n_scored - tp - fp - fn + + precision = tp / row_sum if row_sum > 0 else 0.0 + recall = tp / col_sum if col_sum > 0 else 0.0 + f_score = _f_beta(precision, recall, f_value) + + per_class[label] = PerClassMetrics( + tp=tp, + tn=tn, + fp=fp, + fn=fn, + support=tp + fn, + precision=precision, + recall=recall, + f_score=f_score, + ) + precisions.append(precision) + recalls.append(recall) + f_scores.append(f_score) + total_tp += tp + total_fp += fp + total_fn += fn + + # AggregatorSpec classes come from the ExactMatch config which requires + # a non-empty list, so k >= 1 always. + macro = AveragedMetrics( + precision=sum(precisions) / k, + recall=sum(recalls) / k, + f_score=sum(f_scores) / k, + ) + micro_p = total_tp / (total_tp + total_fp) if (total_tp + total_fp) > 0 else 0.0 + micro_r = total_tp / (total_tp + total_fn) if (total_tp + total_fn) > 0 else 0.0 + micro = AveragedMetrics( + precision=micro_p, + recall=micro_r, + f_score=_f_beta(micro_p, micro_r, f_value), + ) + + averaged = micro if self.spec.averaging == "micro" else macro + headline = { + "precision": averaged.precision, + "recall": averaged.recall, + "fscore": averaged.f_score, + }[self.spec.type] + + details = ClassificationDetails( + metric=self.spec.type, + averaging=self.spec.averaging, + f_value=f_value, + classes=confusion.classes, + confusion_matrix=confusion.matrix, + per_class=per_class, + macro=macro, + micro=micro, + n_total=confusion.n_total, + n_scored=confusion.n_scored, + n_skipped=confusion.n_skipped, + ) + return NumericEvaluationResult(score=headline, details=details) diff --git a/packages/uipath/src/uipath/eval/evaluators/dataset_evaluator_factory.py b/packages/uipath/src/uipath/eval/evaluators/dataset_evaluator_factory.py new file mode 100644 index 000000000..d30a5748b --- /dev/null +++ b/packages/uipath/src/uipath/eval/evaluators/dataset_evaluator_factory.py @@ -0,0 +1,65 @@ +"""Factory that instantiates dataset-level evaluators from aggregator specs. + +Dataset evaluators are built from a self-contained :class:`AggregatorSpec` +embedded in a per-datapoint classification evaluator's config, plus the source +evaluator's name (supplied by the runtime when walking those configs). All +three aggregator types share a single :class:`ClassificationDatasetEvaluator` +implementation that dispatches on ``spec.type`` internally. +""" + +from __future__ import annotations + +from typing import Sequence + +from ._aggregator_specs import AggregatorSpec +from .classification_dataset_evaluators import ClassificationDatasetEvaluator + + +def build_dataset_evaluator( + spec: AggregatorSpec, + source_evaluator: str, + classes: list[str], +) -> ClassificationDatasetEvaluator: + """Build a dataset evaluator instance from an aggregator spec. + + Args: + spec: A validated :class:`AggregatorSpec` (precision / recall / fscore). + source_evaluator: Name of the per-datapoint evaluator whose results + this aggregator consumes. + classes: The class vocabulary from the parent evaluator's config. Shared + across all aggregators attached to that evaluator — a spec no longer + carries classes of its own. + """ + return ClassificationDatasetEvaluator(spec, source_evaluator, classes) + + +def unique_aggregator_specs(specs: Sequence[AggregatorSpec]) -> list[AggregatorSpec]: + """Drop exact-duplicate specs (same type and parameters), preserving order.""" + seen: set[str] = set() + unique: list[AggregatorSpec] = [] + for spec in specs: + dumped = spec.model_dump_json() + if dumped not in seen: + seen.add(dumped) + unique.append(spec) + return unique + + +def dataset_result_key( + source_evaluator: str, spec: AggregatorSpec, duplicate_type: bool +) -> str: + """Result-map key shared by `uipath eval` and the platform worker. + + ``{source}::{type}``, extended with ``.{averaging}`` (and ``.fb{f_value}`` + for fscore) when the same type appears more than once on one source. + Callers must dedupe via :func:`unique_aggregator_specs` first — after that, + duplicate types always differ in averaging or f_value. + """ + key = f"{source_evaluator}::{spec.type}" + averaging = getattr(spec, "averaging", None) + if not duplicate_type or averaging is None: + return key + f_value = getattr(spec, "f_value", None) + if f_value is not None: + return f"{key}.{averaging}.fb{f_value}" + return f"{key}.{averaging}" diff --git a/packages/uipath/src/uipath/eval/evaluators/exact_match_evaluator.py b/packages/uipath/src/uipath/eval/evaluators/exact_match_evaluator.py index e3873b01f..90e50b17e 100644 --- a/packages/uipath/src/uipath/eval/evaluators/exact_match_evaluator.py +++ b/packages/uipath/src/uipath/eval/evaluators/exact_match_evaluator.py @@ -1,11 +1,14 @@ """Exact match evaluator for workload outputs.""" +from pydantic import Field, model_validator + from ..models import ( EvaluationResult, EvaluatorType, NumericEvaluationResult, WorkloadExecution, ) +from ._aggregator_specs import AggregatorSpec from .base_evaluator import BaseEvaluatorJustification from .output_evaluator import ( OutputEvaluationCriteria, @@ -20,6 +23,59 @@ class ExactMatchEvaluatorConfig(OutputEvaluatorConfig[OutputEvaluationCriteria]) name: str = "ExactMatchEvaluator" case_sensitive: bool = False negated: bool = False + classes: list[str] | None = Field( + default=None, + description=( + "Label vocabulary shared by every aggregator on this evaluator. " + "Labels are matched case-insensitively against the per-datapoint " + "expected/actual outputs." + ), + ) + aggregators: list[AggregatorSpec] | None = Field( + default=None, + description=( + "Dataset-level metrics (precision / recall / F-score / confusion " + "matrix) computed over the per-datapoint match outcomes. Requires " + "``classes``." + ), + ) + + @model_validator(mode="after") + def _validate_aggregators(self) -> "ExactMatchEvaluatorConfig": + """Aggregators need a usable class vocabulary and per-label outcomes.""" + if not self.aggregators: + return self + if not self.classes: + raise ValueError( + f"ExactMatch evaluator '{self.name}' declares aggregators but no " + "``classes`` list. Set ``classes`` to the label vocabulary the " + "aggregators should compute Precision/Recall/F-score over." + ) + if self.line_by_line_evaluator: + raise ValueError( + f"ExactMatch evaluator '{self.name}': aggregators are not " + "supported with line_by_line_evaluator — per-line results carry " + "no expected/actual labels, so every datapoint would be skipped." + ) + if self.case_sensitive: + raise ValueError( + f"ExactMatch evaluator '{self.name}': aggregators are not " + "supported with case_sensitive — the confusion matrix buckets " + "labels case-insensitively, so a datapoint could score 0.0 yet " + "land on the true-positive diagonal." + ) + lowered = [c.lower() for c in self.classes] + if any(not c.strip() or c != c.strip() for c in self.classes) or len( + set(lowered) + ) != len(lowered): + raise ValueError( + f"ExactMatch evaluator '{self.name}': ``classes`` must be " + "non-blank, have no leading/trailing whitespace, and be unique " + "case-insensitively — labels are matched case-insensitively, so " + "duplicates would collapse onto one matrix index, and padded " + "labels would never match anything." + ) + return self class ExactMatchEvaluator( diff --git a/packages/uipath/src/uipath/eval/evaluators_types/ExactMatchEvaluator.json b/packages/uipath/src/uipath/eval/evaluators_types/ExactMatchEvaluator.json index 866b06416..5c302ccf8 100644 --- a/packages/uipath/src/uipath/eval/evaluators_types/ExactMatchEvaluator.json +++ b/packages/uipath/src/uipath/eval/evaluators_types/ExactMatchEvaluator.json @@ -2,6 +2,50 @@ "evaluatorTypeId": "uipath-exact-match", "evaluatorConfigSchema": { "$defs": { + "ConfusionMatrixAggregatorSpec": { + "description": "Run-level raw k\u00d7k confusion matrix \u2014 no scalar headline, no averaging.", + "properties": { + "type": { + "const": "confusion_matrix", + "default": "confusion_matrix", + "title": "Type", + "type": "string" + } + }, + "title": "ConfusionMatrixAggregatorSpec", + "type": "object" + }, + "FScoreAggregatorSpec": { + "description": "Run-level F-beta aggregator (multiclass, micro or macro averaged).", + "properties": { + "type": { + "const": "fscore", + "default": "fscore", + "title": "Type", + "type": "string" + }, + "averaging": { + "enum": [ + "macro", + "micro" + ], + "title": "Averaging", + "type": "string" + }, + "f_value": { + "default": 1.0, + "exclusiveMinimum": 0, + "maximum": 1000, + "title": "F Value", + "type": "number" + } + }, + "required": [ + "averaging" + ], + "title": "FScoreAggregatorSpec", + "type": "object" + }, "OutputEvaluationCriteria": { "description": "Base class for all output evaluation criteria.", "properties": { @@ -23,6 +67,54 @@ ], "title": "OutputEvaluationCriteria", "type": "object" + }, + "PrecisionAggregatorSpec": { + "description": "Run-level precision aggregator (multiclass, micro or macro averaged).", + "properties": { + "type": { + "const": "precision", + "default": "precision", + "title": "Type", + "type": "string" + }, + "averaging": { + "enum": [ + "macro", + "micro" + ], + "title": "Averaging", + "type": "string" + } + }, + "required": [ + "averaging" + ], + "title": "PrecisionAggregatorSpec", + "type": "object" + }, + "RecallAggregatorSpec": { + "description": "Run-level recall aggregator (multiclass, micro or macro averaged).", + "properties": { + "type": { + "const": "recall", + "default": "recall", + "title": "Type", + "type": "string" + }, + "averaging": { + "enum": [ + "macro", + "micro" + ], + "title": "Averaging", + "type": "string" + } + }, + "required": [ + "averaging" + ], + "title": "RecallAggregatorSpec", + "type": "object" } }, "description": "Configuration for the exact match evaluator.", @@ -50,9 +142,31 @@ "default": null }, "target_output_key": { + "anyOf": [ + { + "type": "string" + }, + { + "items": { + "type": "string" + }, + "type": "array" + } + ], "default": "*", - "description": "Key to extract output from agent execution", - "title": "Target Output Key", + "description": "Key or list of keys to extract output from workload execution", + "title": "Target Output Key" + }, + "line_by_line_evaluator": { + "default": false, + "description": "If True, split output by delimiter and evaluate each line separately", + "title": "Line By Line Evaluator", + "type": "boolean" + }, + "line_delimiter": { + "default": "\n", + "description": "Delimiter to split output when line_by_line_evaluator is True", + "title": "Line Delimiter", "type": "string" }, "case_sensitive": { @@ -64,6 +178,60 @@ "default": false, "title": "Negated", "type": "boolean" + }, + "classes": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Label vocabulary shared by every aggregator on this evaluator. Labels are matched case-insensitively against the per-datapoint expected/actual outputs.", + "title": "Classes" + }, + "aggregators": { + "anyOf": [ + { + "items": { + "discriminator": { + "mapping": { + "confusion_matrix": "#/$defs/ConfusionMatrixAggregatorSpec", + "fscore": "#/$defs/FScoreAggregatorSpec", + "precision": "#/$defs/PrecisionAggregatorSpec", + "recall": "#/$defs/RecallAggregatorSpec" + }, + "propertyName": "type" + }, + "oneOf": [ + { + "$ref": "#/$defs/PrecisionAggregatorSpec" + }, + { + "$ref": "#/$defs/RecallAggregatorSpec" + }, + { + "$ref": "#/$defs/FScoreAggregatorSpec" + }, + { + "$ref": "#/$defs/ConfusionMatrixAggregatorSpec" + } + ] + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Dataset-level metrics (precision / recall / F-score / confusion matrix) computed over the per-datapoint match outcomes. Requires ``classes``.", + "title": "Aggregators" } }, "title": "ExactMatchEvaluatorConfig", @@ -91,5 +259,23 @@ "title": "OutputEvaluationCriteria", "type": "object" }, - "justificationSchema": {} + "justificationSchema": { + "description": "Base class for all evaluator justifications.", + "properties": { + "expected": { + "title": "Expected", + "type": "string" + }, + "actual": { + "title": "Actual", + "type": "string" + } + }, + "required": [ + "expected", + "actual" + ], + "title": "BaseEvaluatorJustification", + "type": "object" + } } \ No newline at end of file diff --git a/packages/uipath/src/uipath/eval/runtime/_types.py b/packages/uipath/src/uipath/eval/runtime/_types.py index 2aee5e599..fa84f0d9e 100644 --- a/packages/uipath/src/uipath/eval/runtime/_types.py +++ b/packages/uipath/src/uipath/eval/runtime/_types.py @@ -1,7 +1,7 @@ import logging from opentelemetry.sdk.trace import ReadableSpan -from pydantic import BaseModel, ConfigDict +from pydantic import BaseModel, ConfigDict, Field from pydantic.alias_generators import to_camel from uipath.runtime import UiPathRuntimeResult @@ -78,6 +78,9 @@ class UiPathEvalOutput(BaseModel): evaluation_set_name: str evaluation_set_results: list[UiPathEvalRunResult] + dataset_evaluator_results: dict[str, EvaluationResultDto] = Field( + default_factory=dict + ) @property def score(self) -> float: diff --git a/packages/uipath/src/uipath/eval/runtime/runtime.py b/packages/uipath/src/uipath/eval/runtime/runtime.py index 124eedf47..66468df7b 100644 --- a/packages/uipath/src/uipath/eval/runtime/runtime.py +++ b/packages/uipath/src/uipath/eval/runtime/runtime.py @@ -46,6 +46,12 @@ from .._execution_context import ExecutionSpanCollector from ..evaluators.base_evaluator import GenericBaseEvaluator +from ..evaluators.dataset_evaluator_factory import ( + build_dataset_evaluator, + dataset_result_key, + unique_aggregator_specs, +) +from ..evaluators.exact_match_evaluator import ExactMatchEvaluatorConfig from ..evaluators.output_evaluator import OutputEvaluationCriteria from ..helpers import get_agent_model from ..mocks._cache_manager import CacheManager @@ -202,6 +208,75 @@ def compute_evaluator_scores( return final_score, agg_metrics_per_evaluator +def compute_dataset_evaluator_results( + evaluation_set_results: list[UiPathEvalRunResult], + evaluators: Iterable[GenericBaseEvaluator[Any, Any, Any]], +) -> dict[str, EvaluationResultDto]: + """Run any dataset-level aggregators embedded in per-datapoint evaluator configs. + + Walks ``evaluators`` looking for any whose config carries an ``aggregators`` + list (currently only ExactMatch). For each aggregator spec, builds the + corresponding dataset evaluator via the factory and runs it over the + per-datapoint results that came from that source evaluator. + + Args: + evaluation_set_results: Per-datapoint results from the run. + evaluators: Per-datapoint evaluator instances that ran during this eval + set. Their configs may carry ``aggregators`` lists. + + Returns: + Dict keyed by :func:`dataset_result_key` (same scheme as the platform + worker), with each value's ``details`` dumped to the camelCase wire + shape. Exact-duplicate specs are deduped; aggregators whose source + produced no results still emit a zeroed result. + """ + results_by_evaluator: defaultdict[str, list[EvaluationResultDto]] = defaultdict( + list + ) + for eval_run_result in evaluation_set_results: + for eval_run_result_dto in eval_run_result.evaluation_run_results: + if eval_run_result_dto.is_line_result: + continue + results_by_evaluator[eval_run_result_dto.evaluator_name].append( + eval_run_result_dto.result + ) + + dataset_results: dict[str, EvaluationResultDto] = {} + for evaluator in evaluators: + # Aggregators currently only live on ExactMatch evaluator configs — the + # per-datapoint match outcome (with expected/actual labels in the + # justification) is exactly what the confusion matrix needs. Widen the + # isinstance tuple if a future evaluator type grows an ``aggregators`` + # field. + config = getattr(evaluator, "evaluator_config", None) + if not isinstance(config, ExactMatchEvaluatorConfig): + continue + if not config.aggregators or not config.classes: + continue + source_name = config.name + source_results = results_by_evaluator.get(source_name, []) + specs = unique_aggregator_specs(config.aggregators) + type_counts: dict[str, int] = defaultdict(int) + for spec in specs: + type_counts[spec.type] += 1 + for spec in specs: + dataset_evaluator = build_dataset_evaluator( + spec, source_name, config.classes + ) + key = dataset_result_key(source_name, spec, type_counts[spec.type] > 1) + result = dataset_evaluator.evaluate(source_results) + details: str | dict[str, Any] | None + if isinstance(result.details, BaseModel): + # Same camelCase wire shape the platform worker ships. + details = result.details.model_dump(by_alias=True, exclude_none=True) + else: + details = result.details + dataset_results[key] = EvaluationResultDto( + score=result.score, details=details + ) + return dataset_results + + class UiPathEvalRuntime: """Specialized runtime for evaluation runs, with access to the factory.""" @@ -385,6 +460,14 @@ async def execute(self) -> UiPathRuntimeResult: evaluators, ) + # Run dataset-level aggregators over the per-datapoint results. + results.dataset_evaluator_results = ( + compute_dataset_evaluator_results( + results.evaluation_set_results, + evaluators, + ) + ) + # Configure span with output and metadata await configure_eval_set_run_span( span=span, diff --git a/packages/uipath/tests/evaluators/test_dataset_classification_evaluators.py b/packages/uipath/tests/evaluators/test_dataset_classification_evaluators.py new file mode 100644 index 000000000..3364b228a --- /dev/null +++ b/packages/uipath/tests/evaluators/test_dataset_classification_evaluators.py @@ -0,0 +1,581 @@ +"""Tests for dataset-level classification evaluators (Precision, Recall, FScore). + +Covers the math (2-class, 3-class, micro vs macro, F-beta), edge cases +(empty input, out-of-vocab labels, malformed details), factory dispatch, and +runtime-level routing where compute_dataset_evaluator_results walks +per-datapoint evaluator configs' embedded ``aggregators`` lists. +""" + +import uuid + +import pytest +from pydantic import BaseModel + +from uipath.eval.evaluators._aggregator_specs import ( + ConfusionMatrixAggregatorSpec, + FScoreAggregatorSpec, + PrecisionAggregatorSpec, + RecallAggregatorSpec, +) +from uipath.eval.evaluators.base_evaluator import BaseEvaluatorJustification +from uipath.eval.evaluators.classification_dataset_evaluators import ( + ClassificationDatasetEvaluator, + ClassificationDetails, +) +from uipath.eval.evaluators.dataset_evaluator_factory import build_dataset_evaluator +from uipath.eval.evaluators.exact_match_evaluator import ExactMatchEvaluator +from uipath.eval.models.models import ( + EvaluationResultDto, + NumericEvaluationResult, +) +from uipath.eval.runtime._types import ( + UiPathEvalRunResult, + UiPathEvalRunResultDto, +) +from uipath.eval.runtime.runtime import compute_dataset_evaluator_results + + +def _result(expected: str, actual: str) -> EvaluationResultDto: + """Build an EvaluationResultDto carrying an expected/actual justification.""" + justification = BaseEvaluatorJustification(expected=expected, actual=actual) + return EvaluationResultDto( + score=1.0 if expected.lower() == actual.lower() else 0.0, + details=justification.model_dump(), + ) + + +def _precision( + classes: list[str], averaging: str = "macro" +) -> ClassificationDatasetEvaluator: + spec = PrecisionAggregatorSpec(averaging=averaging) # type: ignore[arg-type] + return ClassificationDatasetEvaluator( + spec, source_evaluator="intent_match", classes=classes + ) + + +def _recall( + classes: list[str], averaging: str = "macro" +) -> ClassificationDatasetEvaluator: + spec = RecallAggregatorSpec(averaging=averaging) # type: ignore[arg-type] + return ClassificationDatasetEvaluator( + spec, source_evaluator="intent_match", classes=classes + ) + + +def _fscore( + classes: list[str], averaging: str = "macro", f_value: float = 1.0 +) -> ClassificationDatasetEvaluator: + spec = FScoreAggregatorSpec( + averaging=averaging, # type: ignore[arg-type] + f_value=f_value, + ) + return ClassificationDatasetEvaluator( + spec, source_evaluator="intent_match", classes=classes + ) + + +def _details(result: object) -> ClassificationDetails: + """Type-narrowing helper for asserting on details.""" + assert isinstance(result, NumericEvaluationResult) + assert isinstance(result.details, ClassificationDetails) + return result.details + + +def _exact_match_evaluator( + name: str, + classes: list[str], + aggregators: list[BaseModel], +) -> ExactMatchEvaluator: + """Build a per-datapoint ExactMatch evaluator with attached aggregators. + + Aggregators + classes live on the evaluator config — every aggregator on + the same ExactMatch config shares the ``classes`` vocabulary declared here. + """ + return ExactMatchEvaluator.model_validate( + { + "id": str(uuid.uuid4()), + "evaluatorConfig": { + "name": name, + "classes": classes, + "aggregators": [spec.model_dump(by_alias=True) for spec in aggregators], + }, + } + ) + + +class TestPrecisionEvaluator: + def test_empty_input_returns_zeroed_result(self) -> None: + result = _precision(["cat", "dog"]).evaluate([]) + assert isinstance(result, NumericEvaluationResult) + assert result.score == 0.0 + d = _details(result) + assert d.n_total == 0 and d.n_scored == 0 + assert d.confusion_matrix == [[0, 0], [0, 0]] + assert d.per_class["cat"].tp == 0 + assert d.per_class["cat"].tn == 0 + + def test_confusion_matrix_is_predicted_by_expected(self) -> None: + # Pin the documented orientation: confusion_matrix[predicted][expected]. + # Differs from sklearn's [true][predicted] convention. + results = [ + _result("cat", "cat"), # expected=cat, predicted=cat -> [cat][cat] + _result("cat", "dog"), # expected=cat, predicted=dog -> [dog][cat] + _result("dog", "dog"), # expected=dog, predicted=dog -> [dog][dog] + _result("dog", "dog"), + ] + d = _details(_precision(["cat", "dog"]).evaluate(results)) + # classes -> index: cat=0, dog=1 + # [predicted=cat][expected=cat] = 1 + assert d.confusion_matrix[0][0] == 1 + # [predicted=dog][expected=cat] = 1 (the FP for dog / FN for cat) + assert d.confusion_matrix[1][0] == 1 + # [predicted=dog][expected=dog] = 2 + assert d.confusion_matrix[1][1] == 2 + # [predicted=cat][expected=dog] = 0 + assert d.confusion_matrix[0][1] == 0 + + def test_precision_two_class_macro(self) -> None: + results = [ + _result("yes", "yes"), + _result("yes", "yes"), + _result("yes", "no"), + _result("no", "yes"), + ] + result = _precision(["yes", "no"], averaging="macro").evaluate(results) + d = _details(result) + # precision_yes = 2 / (2 + 1) = 2/3 + # precision_no = 0 / (0 + 1) = 0 + # macro = (2/3 + 0) / 2 = 1/3 + assert d.per_class["yes"].precision == pytest.approx(2 / 3) + assert d.per_class["no"].precision == pytest.approx(0.0) + assert d.macro.precision == pytest.approx((2 / 3 + 0.0) / 2) + assert result.score == pytest.approx(d.macro.precision) + + def test_two_class_micro_equals_accuracy(self) -> None: + results = [ + _result("yes", "yes"), + _result("yes", "yes"), + _result("yes", "no"), + _result("no", "yes"), + ] + result = _precision(["yes", "no"], averaging="micro").evaluate(results) + d = _details(result) + assert d.micro.precision == pytest.approx(0.5) + assert result.score == pytest.approx(0.5) + + def test_three_class_macro(self) -> None: + pairs = [ + ("cat", "cat"), + ("cat", "cat"), + ("cat", "dog"), + ("dog", "dog"), + ("dog", "dog"), + ("dog", "bird"), + ("bird", "bird"), + ("bird", "bird"), + ("bird", "cat"), + ] + result = _precision(["cat", "dog", "bird"], averaging="macro").evaluate( + [_result(e, a) for e, a in pairs] + ) + d = _details(result) + for label in ("cat", "dog", "bird"): + m = d.per_class[label] + assert m.tp == 2 and m.fp == 1 and m.fn == 1 and m.tn == 5 + assert m.precision == pytest.approx(2 / 3) + assert d.macro.precision == pytest.approx(2 / 3) + assert result.score == pytest.approx(2 / 3) + + +class TestRecallEvaluator: + def test_recall_two_class_macro(self) -> None: + results = [ + _result("yes", "yes"), + _result("yes", "yes"), + _result("yes", "no"), + _result("no", "yes"), + ] + result = _recall(["yes", "no"], averaging="macro").evaluate(results) + d = _details(result) + assert d.per_class["yes"].recall == pytest.approx(2 / 3) + assert d.per_class["no"].recall == pytest.approx(0.0) + assert result.score == pytest.approx(1 / 3) + + def test_recall_differs_from_precision(self) -> None: + results = [ + _result("yes", "yes"), + _result("yes", "yes"), + _result("no", "yes"), + _result("no", "yes"), + _result("no", "no"), + ] + p = _details(_precision(["yes", "no"], averaging="macro").evaluate(results)) + r = _details(_recall(["yes", "no"], averaging="macro").evaluate(results)) + assert p.per_class["yes"].precision == pytest.approx(0.5) + assert p.per_class["no"].precision == pytest.approx(1.0) + assert r.per_class["yes"].recall == pytest.approx(1.0) + assert r.per_class["no"].recall == pytest.approx(1 / 3) + + +class TestFScoreEvaluator: + def test_f1_equals_harmonic_mean_of_p_and_r(self) -> None: + results = [ + _result("yes", "yes"), + _result("yes", "yes"), + _result("yes", "no"), + _result("no", "yes"), + ] + f = _details( + _fscore(["yes", "no"], averaging="macro", f_value=1.0).evaluate(results) + ) + assert f.per_class["yes"].f_score == pytest.approx(2 / 3) + assert f.per_class["no"].f_score == pytest.approx(0.0) + assert f.macro.f_score == pytest.approx((2 / 3 + 0.0) / 2) + + def test_f_beta_emphasizes_recall_when_beta_above_one(self) -> None: + results = [ + _result("yes", "yes"), + _result("yes", "yes"), + _result("no", "yes"), + _result("no", "yes"), + _result("no", "no"), + ] + f1 = _details( + _fscore(["yes", "no"], averaging="macro", f_value=1.0).evaluate(results) + ) + f2 = _details( + _fscore(["yes", "no"], averaging="macro", f_value=2.0).evaluate(results) + ) + assert f2.per_class["yes"].f_score > f1.per_class["yes"].f_score + + def test_three_class_micro_pools_across_classes(self) -> None: + pairs = [ + ("cat", "cat"), + ("cat", "cat"), + ("cat", "dog"), + ("dog", "dog"), + ("dog", "dog"), + ("dog", "bird"), + ("bird", "bird"), + ("bird", "bird"), + ("bird", "cat"), + ] + d = _details( + _fscore(["cat", "dog", "bird"], averaging="micro", f_value=1.0).evaluate( + [_result(e, a) for e, a in pairs] + ) + ) + assert d.micro.f_score == pytest.approx(6 / 9) + + +class TestSkippingAndEdgeCases: + def test_out_of_vocab_labels_are_skipped(self) -> None: + results = [ + _result("cat", "cat"), + _result("cat", "platypus"), + _result("zebra", "dog"), + ] + d = _details(_precision(["cat", "dog"]).evaluate(results)) + assert d.n_total == 3 and d.n_scored == 1 and d.n_skipped == 2 + + def test_results_without_justification_are_skipped(self) -> None: + results = [ + _result("cat", "cat"), + EvaluationResultDto(score=1.0, details="just a string"), + EvaluationResultDto(score=0.0, details={"unrelated": "shape"}), + ] + d = _details(_precision(["cat", "dog"]).evaluate(results)) + assert d.n_total == 3 and d.n_scored == 1 and d.n_skipped == 2 + + def test_case_insensitive(self) -> None: + results = [_result("Cat", "CAT"), _result("DOG", "dog")] + d = _details(_precision(["cat", "dog"]).evaluate(results)) + assert d.per_class["cat"].tp == 1 + assert d.per_class["dog"].tp == 1 + + +class TestFactory: + """The factory builds from an AggregatorSpec instance + source name.""" + + def test_builds_precision_from_spec(self) -> None: + spec = PrecisionAggregatorSpec(averaging="macro") + evaluator = build_dataset_evaluator(spec, "intent_match", classes=["yes", "no"]) + assert isinstance(evaluator, ClassificationDatasetEvaluator) + assert evaluator.spec.type == "precision" + assert evaluator.source_evaluator == "intent_match" + + def test_builds_recall_from_spec(self) -> None: + spec = RecallAggregatorSpec(averaging="micro") + evaluator = build_dataset_evaluator(spec, "intent_match", classes=["yes", "no"]) + assert isinstance(evaluator, ClassificationDatasetEvaluator) + assert evaluator.spec.type == "recall" + + def test_builds_fscore_from_spec(self) -> None: + spec = FScoreAggregatorSpec(averaging="macro", f_value=2.0) + evaluator = build_dataset_evaluator(spec, "intent_match", classes=["yes", "no"]) + assert isinstance(evaluator, ClassificationDatasetEvaluator) + assert isinstance(evaluator.spec, FScoreAggregatorSpec) + assert evaluator.spec.f_value == 2.0 + + +class TestAggregatorSpecJsonRoundTrip: + """Pin the wire shape sent to the C# side.""" + + def test_precision_spec_wire_shape(self) -> None: + """Specs carry only metric-shape fields; ``classes`` lives on the + parent evaluator config. + """ + spec = PrecisionAggregatorSpec.model_validate( + { + "type": "precision", + "averaging": "macro", + } + ) + dumped = spec.model_dump(by_alias=True) + assert dumped == { + "type": "precision", + "averaging": "macro", + } + + def test_fscore_uses_camelcase_fvalue_on_wire(self) -> None: + spec = FScoreAggregatorSpec.model_validate( + { + "type": "fscore", + "averaging": "macro", + "fValue": 1.5, + } + ) + assert spec.f_value == 1.5 + dumped = spec.model_dump(by_alias=True) + assert dumped["fValue"] == 1.5 + assert "f_value" not in dumped + + def test_exact_match_evaluator_round_trips_aggregators(self) -> None: + """Per-datapoint evaluator config carries aggregators[]; survives dump+load.""" + ev = _exact_match_evaluator( + "intent_classifier", + classes=["book", "cancel", "reschedule"], + aggregators=[ + PrecisionAggregatorSpec(averaging="macro"), + FScoreAggregatorSpec(averaging="macro", f_value=1.0), + ], + ) + assert ev.evaluator_config.aggregators is not None + assert len(ev.evaluator_config.aggregators) == 2 + assert ev.evaluator_config.aggregators[0].type == "precision" + assert ev.evaluator_config.aggregators[1].type == "fscore" + + +class TestComputeDatasetEvaluatorResults: + """End-to-end: runtime walks evaluator configs' aggregators[].""" + + def test_walks_aggregators_on_classification_evaluator(self) -> None: + evaluator = _exact_match_evaluator( + "intent_match", + classes=["yes", "no"], + aggregators=[ + PrecisionAggregatorSpec(averaging="macro"), + RecallAggregatorSpec(averaging="macro"), + ], + ) + + eval_results = [ + UiPathEvalRunResult( + evaluation_name="dp1", + evaluation_run_results=[ + UiPathEvalRunResultDto( + evaluator_name="intent_match", + evaluator_id=str(uuid.uuid4()), + result=_result("yes", "yes"), + ), + UiPathEvalRunResultDto( + evaluator_name="some_other_evaluator", + evaluator_id=str(uuid.uuid4()), + result=EvaluationResultDto(score=0.5), + ), + ], + ), + UiPathEvalRunResult( + evaluation_name="dp2", + evaluation_run_results=[ + UiPathEvalRunResultDto( + evaluator_name="intent_match", + evaluator_id=str(uuid.uuid4()), + result=_result("yes", "no"), + ), + ], + ), + ] + + out = compute_dataset_evaluator_results(eval_results, [evaluator]) + # Two aggregators on intent_match → two keys, prefixed by source name. + assert set(out) == {"intent_match::precision", "intent_match::recall"} + precision_dto = out["intent_match::precision"] + assert isinstance(precision_dto, EvaluationResultDto) + assert isinstance(precision_dto.details, dict) + # The unrelated 0.5 score from some_other_evaluator must NOT be in the matrix. + assert precision_dto.details["nScored"] == 2 + + def test_evaluator_without_aggregators_is_skipped(self) -> None: + evaluator = _exact_match_evaluator( + "intent_match", classes=["yes", "no"], aggregators=[] + ) + eval_results = [ + UiPathEvalRunResult( + evaluation_name="dp1", + evaluation_run_results=[ + UiPathEvalRunResultDto( + evaluator_name="intent_match", + evaluator_id=str(uuid.uuid4()), + result=_result("yes", "yes"), + ), + ], + ), + ] + out = compute_dataset_evaluator_results(eval_results, [evaluator]) + assert out == {} + + def test_line_by_line_subresults_are_excluded(self) -> None: + evaluator = _exact_match_evaluator( + "intent_match", + classes=["yes", "no"], + aggregators=[ + PrecisionAggregatorSpec(averaging="macro"), + ], + ) + eval_results = [ + UiPathEvalRunResult( + evaluation_name="dp1", + evaluation_run_results=[ + UiPathEvalRunResultDto( + evaluator_name="intent_match", + evaluator_id=str(uuid.uuid4()), + result=_result("yes", "yes"), + is_line_result=True, + ), + UiPathEvalRunResultDto( + evaluator_name="intent_match", + evaluator_id=str(uuid.uuid4()), + result=_result("no", "no"), + ), + ], + ), + ] + out = compute_dataset_evaluator_results(eval_results, [evaluator]) + assert isinstance(out["intent_match::precision"].details, dict) + assert out["intent_match::precision"].details["nScored"] == 1 + + def test_source_with_no_results_produces_zeroed_report(self) -> None: + evaluator = _exact_match_evaluator( + "intent_match", + classes=["yes", "no"], + aggregators=[ + PrecisionAggregatorSpec(averaging="macro"), + ], + ) + eval_results = [ + UiPathEvalRunResult( + evaluation_name="dp1", + evaluation_run_results=[ + UiPathEvalRunResultDto( + evaluator_name="some_other_evaluator", + evaluator_id=str(uuid.uuid4()), + result=EvaluationResultDto(score=1.0), + ), + ], + ), + ] + out = compute_dataset_evaluator_results(eval_results, [evaluator]) + dto = out["intent_match::precision"] + assert dto.score == 0.0 + assert isinstance(dto.details, dict) + assert dto.details["nScored"] == 0 + + def test_duplicate_aggregator_type_disambiguates_by_averaging(self) -> None: + """Two aggregators of the same type get distinct keys (no overwrite).""" + evaluator = _exact_match_evaluator( + "intent_match", + classes=["yes", "no"], + aggregators=[ + PrecisionAggregatorSpec(averaging="macro"), + PrecisionAggregatorSpec(averaging="micro"), + ], + ) + eval_results = [ + UiPathEvalRunResult( + evaluation_name="dp1", + evaluation_run_results=[ + UiPathEvalRunResultDto( + evaluator_name="intent_match", + evaluator_id=str(uuid.uuid4()), + result=_result("yes", "yes"), + ), + ], + ), + ] + out = compute_dataset_evaluator_results(eval_results, [evaluator]) + # Same type appears twice → averaging suffix disambiguates so neither + # is silently overwritten. + assert set(out) == { + "intent_match::precision.macro", + "intent_match::precision.micro", + } + + def test_exact_duplicate_specs_are_deduped(self) -> None: + """Identical specs collapse to one result; duplicate confusion_matrix + (no averaging field) must not crash key disambiguation.""" + evaluator = _exact_match_evaluator( + "intent_match", + classes=["yes", "no"], + aggregators=[ + PrecisionAggregatorSpec(averaging="macro"), + PrecisionAggregatorSpec(averaging="macro"), + ConfusionMatrixAggregatorSpec(), + ConfusionMatrixAggregatorSpec(), + ], + ) + eval_results = [ + UiPathEvalRunResult( + evaluation_name="dp1", + evaluation_run_results=[ + UiPathEvalRunResultDto( + evaluator_name="intent_match", + evaluator_id=str(uuid.uuid4()), + result=_result("yes", "yes"), + ), + ], + ), + ] + out = compute_dataset_evaluator_results(eval_results, [evaluator]) + assert set(out) == { + "intent_match::precision", + "intent_match::confusion_matrix", + } + + def test_details_are_dumped_to_camelcase_wire_shape(self) -> None: + """The local path ships the same JSON shape as the platform worker: + camelCase keys, absent (not null) optional fields.""" + evaluator = _exact_match_evaluator( + "intent_match", + classes=["yes", "no"], + aggregators=[ConfusionMatrixAggregatorSpec()], + ) + eval_results = [ + UiPathEvalRunResult( + evaluation_name="dp1", + evaluation_run_results=[ + UiPathEvalRunResultDto( + evaluator_name="intent_match", + evaluator_id=str(uuid.uuid4()), + result=_result("yes", "yes"), + ), + ], + ), + ] + out = compute_dataset_evaluator_results(eval_results, [evaluator]) + details = out["intent_match::confusion_matrix"].details + assert isinstance(details, dict) + assert details["confusionMatrix"] == [[1, 0], [0, 0]] + assert details["nScored"] == 1 + # confusion_matrix variant: scalar fields are absent, not null. + assert "perClass" not in details and "macro" not in details diff --git a/packages/uipath/tests/evaluators/test_exact_match_aggregators.py b/packages/uipath/tests/evaluators/test_exact_match_aggregators.py new file mode 100644 index 000000000..3bf1b1e8d --- /dev/null +++ b/packages/uipath/tests/evaluators/test_exact_match_aggregators.py @@ -0,0 +1,183 @@ +"""ExactMatch aggregator config surface. + +The platform's dataset-evaluator pass (Agents repo) reads ``classes`` and +``aggregators`` off the ExactMatch evaluator config. These tests pin the SDK +side of that wire contract: the discriminated spec union, the config fields, +and the config validator. The aggregation math lives in +``classification_dataset_evaluators.py`` (tested separately) and is consumed +by both `uipath eval` and the platform's python-dataset-eval-worker. +""" + +import uuid + +import pytest +from pydantic import TypeAdapter, ValidationError + +from uipath.eval.evaluators._aggregator_specs import ( + AggregatorSpec, + ConfusionMatrixAggregatorSpec, + FScoreAggregatorSpec, + PrecisionAggregatorSpec, + RecallAggregatorSpec, +) +from uipath.eval.evaluators.exact_match_evaluator import ExactMatchEvaluator + + +def _evaluator(config: dict) -> ExactMatchEvaluator: + return ExactMatchEvaluator.model_validate( + {"evaluatorConfig": config, "id": str(uuid.uuid4())} + ) + + +class TestAggregatorSpecUnion: + """Wire shape sent to the platform — {type, averaging, [fValue]}.""" + + def test_discriminates_on_type(self) -> None: + adapter = TypeAdapter(AggregatorSpec) + assert isinstance( + adapter.validate_python({"type": "precision", "averaging": "macro"}), + PrecisionAggregatorSpec, + ) + assert isinstance( + adapter.validate_python({"type": "recall", "averaging": "micro"}), + RecallAggregatorSpec, + ) + fscore = adapter.validate_python( + {"type": "fscore", "averaging": "macro", "fValue": 2.0} + ) + assert isinstance(fscore, FScoreAggregatorSpec) + assert fscore.f_value == 2.0 + assert isinstance( + adapter.validate_python({"type": "confusion_matrix"}), + ConfusionMatrixAggregatorSpec, + ) + + def test_specs_do_not_carry_classes(self) -> None: + # Classes live once on the evaluator config, shared by all aggregators. + dumped = PrecisionAggregatorSpec(averaging="macro").model_dump(by_alias=True) + assert dumped == {"type": "precision", "averaging": "macro"} + + def test_fscore_uses_camelcase_fvalue_on_wire(self) -> None: + dumped = FScoreAggregatorSpec(averaging="macro", f_value=1.5).model_dump( + by_alias=True + ) + assert dumped["fValue"] == 1.5 + assert "f_value" not in dumped + + def test_fscore_fvalue_is_bounded(self) -> None: + # A huge beta overflows beta² to inf → NaN score → unrepresentable JSON. + adapter = TypeAdapter(AggregatorSpec) + for bad in (0, -1, 1e200, float("inf"), float("nan")): + with pytest.raises(ValidationError): + adapter.validate_python( + {"type": "fscore", "averaging": "macro", "fValue": bad} + ) + + +class TestExactMatchAggregatorConfig: + def test_accepts_aggregators_with_classes(self) -> None: + ev = _evaluator( + { + "name": "IntentClassifier", + "classes": ["book", "cancel", "reschedule"], + "aggregators": [ + {"type": "precision", "averaging": "macro"}, + {"type": "recall", "averaging": "macro"}, + {"type": "fscore", "averaging": "macro", "fValue": 1.0}, + ], + } + ) + config = ev.evaluator_config + assert config.classes == ["book", "cancel", "reschedule"] + assert config.aggregators is not None + assert [s.type for s in config.aggregators] == ["precision", "recall", "fscore"] + + def test_rejects_aggregators_without_classes(self) -> None: + # The SDK wraps pydantic's ValidationError in UiPathEvaluationError at + # evaluator construction; match the message rather than the type. + with pytest.raises(Exception, match="classes"): + _evaluator( + { + "name": "IntentClassifier", + "aggregators": [{"type": "precision", "averaging": "macro"}], + } + ) + + def test_rejects_case_duplicate_classes(self) -> None: + # Labels match case-insensitively, so "Yes"/"yes" would collapse onto + # one matrix index and silently skew every metric. + with pytest.raises(Exception, match="unique"): + _evaluator( + { + "name": "IntentClassifier", + "classes": ["Yes", "yes"], + "aggregators": [{"type": "precision", "averaging": "macro"}], + } + ) + + def test_rejects_blank_class_labels(self) -> None: + with pytest.raises(Exception, match="non-blank"): + _evaluator( + { + "name": "IntentClassifier", + "classes": ["yes", " "], + "aggregators": [{"type": "precision", "averaging": "macro"}], + } + ) + + def test_rejects_aggregators_with_case_sensitive(self) -> None: + # Per-datapoint scoring would be case-sensitive while the matrix + # buckets case-insensitively — a 0.0-scored datapoint could land on + # the true-positive diagonal. + with pytest.raises(Exception, match="case_sensitive"): + _evaluator( + { + "name": "IntentClassifier", + "classes": ["yes", "no"], + "caseSensitive": True, + "aggregators": [{"type": "precision", "averaging": "macro"}], + } + ) + + def test_rejects_padded_class_labels(self) -> None: + # Padded labels pass a blank check but never match at lookup time — + # every datapoint would silently land in nSkipped. + with pytest.raises(Exception, match="whitespace"): + _evaluator( + { + "name": "IntentClassifier", + "classes": ["yes", "no "], + "aggregators": [{"type": "precision", "averaging": "macro"}], + } + ) + + def test_rejects_aggregators_with_line_by_line(self) -> None: + # Per-line results carry no expected/actual labels — every datapoint + # would land in n_skipped and all metrics would silently read 0. + with pytest.raises(Exception, match="line_by_line"): + _evaluator( + { + "name": "IntentClassifier", + "classes": ["yes", "no"], + "lineByLineEvaluator": True, + "aggregators": [{"type": "precision", "averaging": "macro"}], + } + ) + + def test_plain_config_needs_neither_field(self) -> None: + ev = _evaluator({"name": "PlainMatch"}) + assert ev.evaluator_config.classes is None + assert ev.evaluator_config.aggregators is None + + def test_round_trips_through_dump_and_load(self) -> None: + config = { + "name": "IntentClassifier", + "classes": ["yes", "no"], + "aggregators": [{"type": "fscore", "averaging": "micro", "fValue": 2.0}], + } + ev = _evaluator(config) + dumped = ev.evaluator_config.model_dump(by_alias=True, exclude_none=True) + assert dumped["classes"] == ["yes", "no"] + assert dumped["aggregators"] == [ + {"type": "fscore", "averaging": "micro", "fValue": 2.0} + ] diff --git a/packages/uipath/uv.lock b/packages/uipath/uv.lock index 19e974d12..b26e1984a 100644 --- a/packages/uipath/uv.lock +++ b/packages/uipath/uv.lock @@ -2598,7 +2598,7 @@ wheels = [ [[package]] name = "uipath" -version = "2.13.6" +version = "2.13.7" source = { editable = "." } dependencies = [ { name = "applicationinsights" },