add evaluator_name and datapoint_id identifiers to EvaluationResult + initial version for results aggregation

Author: andrei-rusu

src/uipath/_cli/_evals/_models/_output.py

@@ -24,6 +24,8 @@ class EvaluationResultDto(BaseModel):
     score: float
     details: Optional[str | BaseModel] = None
     evaluation_time: Optional[float] = None
+    evaluator_name: Optional[str] = None
+    datapoint_id: Optional[str] = None
 
     @model_serializer(mode="wrap")
     def serialize_model(self, serializer, info):
@@ -49,6 +51,8 @@ def from_evaluation_result(
             score=score,
             details=evaluation_result.details,
             evaluation_time=evaluation_result.evaluation_time,
+            evaluator_name=evaluation_result.evaluator_name,
+            datapoint_id=evaluation_result.datapoint_id,
         )
 
 

src/uipath/eval/_helpers/coded_evaluators_helpers.py

@@ -1,12 +1,20 @@
 import ast
+import hashlib
 import json
+from collections import defaultdict
 from collections.abc import Mapping, Sequence
 from datetime import datetime
 from typing import Any
 
 from opentelemetry.sdk.trace import ReadableSpan
 
-from ..models import ToolCall, ToolOutput
+from ..models import (
+    AgentExecution,
+    EvaluationResult,
+    NumericEvaluationResult,
+    ToolCall,
+    ToolOutput,
+)
 
 COMPARATOR_MAPPINGS = {
     ">": "gt",
@@ -21,6 +29,48 @@
 COMMUNITY_agents_SUFFIX = "-community-agents"
 
 
+def generate_datapoint_id(agent_execution: AgentExecution) -> str:
+    """Generate a collision-safe but readable datapoint ID from agent_input.
+
+    Creates a short, readable ID that includes meaningful content from the input
+    plus a hash suffix for collision safety.
+
+    Args:
+        agent_execution: The agent execution containing agent_input
+
+    Returns:
+        String datapoint ID in format: "readable_part_HASH"
+    """
+    if not agent_execution.agent_input:
+        # Handle empty input case
+        raw_input = "empty_input"
+    else:
+        # Convert agent_input to JSON string for hashing
+        raw_input = json.dumps(
+            agent_execution.agent_input, sort_keys=True, separators=(",", ":")
+        )
+
+    # Create readable part from input (first 30 chars, alphanumeric only)
+    readable_part = ""
+    if isinstance(agent_execution.agent_input, dict):
+        # Try to extract meaningful text from common fields
+        for key in ["query", "question", "input", "prompt", "text", "message"]:
+            if key in agent_execution.agent_input and agent_execution.agent_input[key]:
+                text = str(agent_execution.agent_input[key])
+                readable_part = "".join(c for c in text if c.isalnum() or c in " _-")
+                readable_part = readable_part.replace(" ", "_").lower()[:30]
+                break
+
+    # If no readable part found, use "input" prefix
+    if not readable_part:
+        readable_part = "input"
+
+    # Generate 8-character hash for collision safety
+    hash_part = hashlib.md5(raw_input.encode("utf-8")).hexdigest()[:8]
+
+    return f"{readable_part}_{hash_part}"
+
+
 def extract_tool_calls_names(spans: Sequence[ReadableSpan]) -> list[str]:
     """Extract the tool call names from execution spans IN ORDER.
 
@@ -456,3 +506,89 @@ def trace_to_str(agent_trace: Sequence[ReadableSpan]) -> str:
             platform_history.append("")
 
     return "\n".join(platform_history)
+
+
+def calculate_final_score(
+    evaluation_results: list[EvaluationResult],
+    evaluator_weights: dict[str, float] | None = None,
+    default_weight: float = 1.0,
+) -> tuple[float, dict[str, float]]:
+    """Aggregate evaluation results with deduplication and weighted scoring.
+
+    Only NumericEvaluationResult can be aggregated, other types of results are ignored.
+
+    This function performs the following steps:
+    1. Deduplicates results by datapoint_id and evaluator_name (averages duplicates)
+    2. Calculates average score per evaluator across all datapoints
+    3. Computes final weighted score across evaluators
+
+    Args:
+        evaluation_results: List of EvaluationResult objects with datapoint_id and evaluator_name
+        evaluator_weights: Optional dict mapping evaluator names to weights
+
+    Returns:
+        Tuple of (final_score, agg_metrics_per_evaluator)
+        - final_score: Weighted average across evaluators
+        - agg_metrics_per_evaluator: Dict mapping evaluator names to their average scores
+    """
+    if not evaluation_results:
+        return 0.0, {}
+
+    if evaluator_weights is None:
+        evaluator_weights = {}
+
+    # Step 1: Group by datapoint_id and evaluator_name for deduplication
+    grouped_by_datapoint_evaluator = defaultdict(
+        lambda: defaultdict(list[NumericEvaluationResult])
+    )
+
+    for result in evaluation_results:
+        # Only NumericEvaluationResult can be aggregated
+        if isinstance(result, NumericEvaluationResult):
+            datapoint_id = result.datapoint_id or "unknown_datapoint"
+            evaluator_name = result.evaluator_name or "unknown_evaluator"
+            grouped_by_datapoint_evaluator[datapoint_id][evaluator_name].append(result)
+
+    # Step 2: Deduplicate by averaging same evaluator results for same datapoint
+    dedup_results: list[NumericEvaluationResult] = []
+    for datapoint_id, evaluators_dict in grouped_by_datapoint_evaluator.items():
+        for evaluator_name, results_list in evaluators_dict.items():
+            if results_list:
+                # Average the scores for this evaluator on this datapoint
+                avg_score = sum(r.score for r in results_list) / len(results_list)
+                # Create a representative result (use first result as template)
+                first_result = results_list[0]
+                dedup_result = type(first_result)(
+                    score=avg_score,
+                    datapoint_id=datapoint_id,
+                    evaluator_name=evaluator_name,
+                    details=first_result.details,
+                    evaluation_time=first_result.evaluation_time,
+                )
+                dedup_results.append(dedup_result)
+
+    # Step 3: Group by evaluator and calculate average score per evaluator
+    grouped_by_evaluator = defaultdict(list[NumericEvaluationResult])
+    for result in dedup_results:
+        grouped_by_evaluator[result.evaluator_name].append(result)
+
+    agg_metrics_per_evaluator = {}
+    for evaluator_name, results_list in grouped_by_evaluator.items():
+        avg_score = sum(r.score for r in results_list) / len(results_list)
+        agg_metrics_per_evaluator[evaluator_name] = avg_score
+
+    # Step 4: Calculate final weighted score
+    if not agg_metrics_per_evaluator:
+        return 0.0, {}
+
+    total_weighted_score = 0.0
+    total_weight = 0.0
+
+    for evaluator_name, avg_score in agg_metrics_per_evaluator.items():
+        weight = evaluator_weights.get(evaluator_name, default_weight)
+        total_weighted_score += avg_score * weight
+        total_weight += weight
+
+    final_score = total_weighted_score / total_weight if total_weight > 0 else 0.0
+
+    return final_score, agg_metrics_per_evaluator

src/uipath/eval/coded_evaluators/contains_evaluator.py

@@ -1,5 +1,6 @@
 """Contains evaluator for agent outputs."""
 
+from .._helpers.coded_evaluators_helpers import generate_datapoint_id
 from ..models import AgentExecution, EvaluationResult, NumericEvaluationResult
 from .base_evaluator import BaseEvaluationCriteria, BaseEvaluator, BaseEvaluatorConfig
 
@@ -57,7 +58,11 @@ async def evaluate(
         if self.evaluator_config.negated:
             is_contains = not is_contains
 
-        return NumericEvaluationResult(score=float(is_contains))
+        return NumericEvaluationResult(
+            score=float(is_contains),
+            evaluator_name=self.evaluator_config.name,
+            datapoint_id=generate_datapoint_id(agent_execution),
+        )
 
     def _get_actual_output(self, agent_execution: AgentExecution) -> str:
         """Get the actual output from the agent execution."""

src/uipath/eval/coded_evaluators/exact_match_evaluator.py

@@ -1,5 +1,6 @@
 """Exact match evaluator for agent outputs."""
 
+from .._helpers.coded_evaluators_helpers import generate_datapoint_id
 from ..models import AgentExecution, EvaluationResult, NumericEvaluationResult
 from .output_evaluator import (
     OutputEvaluationCriteria,
@@ -52,4 +53,8 @@ async def evaluate(
         if self.evaluator_config.negated:
             is_exact_match = not is_exact_match
 
-        return NumericEvaluationResult(score=float(is_exact_match))
+        return NumericEvaluationResult(
+            score=float(is_exact_match),
+            evaluator_name=self.evaluator_config.name,
+            datapoint_id=generate_datapoint_id(agent_execution),
+        )

src/uipath/eval/coded_evaluators/json_similarity_evaluator.py

@@ -5,6 +5,7 @@
 
 from pydantic import Field
 
+from .._helpers.coded_evaluators_helpers import generate_datapoint_id
 from ..models import AgentExecution, EvaluationResult, NumericEvaluationResult
 from .output_evaluator import (
     OutputEvaluationCriteria,
@@ -55,6 +56,8 @@ async def evaluate(
         return NumericEvaluationResult(
             score=score,
             details=validated_justification,
+            evaluator_name=self.evaluator_config.name,
+            datapoint_id=generate_datapoint_id(agent_execution),
         )
 
     def _compare_json(self, expected: Any, actual: Any) -> tuple[float, str]:

src/uipath/eval/coded_evaluators/llm_as_judge_evaluator.py

@@ -7,7 +7,10 @@
 
 from pydantic import BaseModel, Field, model_validator
 
-from .._helpers.coded_evaluators_helpers import COMMUNITY_agents_SUFFIX
+from .._helpers.coded_evaluators_helpers import (
+    COMMUNITY_agents_SUFFIX,
+    generate_datapoint_id,
+)
 from ..models import (
     AgentExecution,
     EvaluationResult,
@@ -99,8 +102,10 @@ async def evaluate(
         )
 
         return NumericEvaluationResult(
-            score=round(llm_response.score / 100.0, 2),
+            score=max(0.0, min(1.0, round(llm_response.score / 100.0, 2))),
             details=validated_justification,
+            evaluator_name=self.evaluator_config.name,
+            datapoint_id=generate_datapoint_id(agent_execution),
         )
 
     def _create_evaluation_prompt(

src/uipath/eval/coded_evaluators/tool_call_args_evaluator.py

@@ -2,6 +2,7 @@
 
 from .._helpers.coded_evaluators_helpers import (
     extract_tool_calls,
+    generate_datapoint_id,
     tool_calls_args_score,
 )
 from ..models import AgentExecution, EvaluationResult, NumericEvaluationResult, ToolCall
@@ -74,4 +75,6 @@ async def evaluate(
         return NumericEvaluationResult(
             score=score,
             details=validated_justification,
+            evaluator_name=self.evaluator_config.name,
+            datapoint_id=generate_datapoint_id(agent_execution),
         )

src/uipath/eval/coded_evaluators/tool_call_count_evaluator.py

@@ -4,6 +4,7 @@
 
 from .._helpers.coded_evaluators_helpers import (
     extract_tool_calls_names,
+    generate_datapoint_id,
     tool_calls_count_score,
 )
 from ..models import AgentExecution, EvaluationResult, NumericEvaluationResult
@@ -77,4 +78,6 @@ async def evaluate(
         return NumericEvaluationResult(
             score=score,
             details=validated_justification,
+            evaluator_name=self.evaluator_config.name,
+            datapoint_id=generate_datapoint_id(agent_execution),
         )

src/uipath/eval/coded_evaluators/tool_call_order_evaluator.py

@@ -2,6 +2,7 @@
 
 from .._helpers.coded_evaluators_helpers import (
     extract_tool_calls_names,
+    generate_datapoint_id,
     tool_calls_order_score,
 )
 from ..models import AgentExecution, EvaluationResult, NumericEvaluationResult
@@ -74,4 +75,6 @@ async def evaluate(
         return NumericEvaluationResult(
             score=score,
             details=validated_justification,
+            evaluator_name=self.evaluator_config.name,
+            datapoint_id=generate_datapoint_id(agent_execution),
         )

src/uipath/eval/coded_evaluators/tool_call_output_evaluator.py

@@ -2,6 +2,7 @@
 
 from .._helpers.coded_evaluators_helpers import (
     extract_tool_calls_outputs,
+    generate_datapoint_id,
     tool_calls_output_score,
 )
 from ..models import (
@@ -77,4 +78,6 @@ async def evaluate(
         return NumericEvaluationResult(
             score=score,
             details=validated_justification,
+            evaluator_name=self.evaluator_config.name,
+            datapoint_id=generate_datapoint_id(agent_execution),
         )