Merge pull request #483 from UiPath/feat/evals

feat: add eval cli command

Author: radu-mocanu

pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "uipath"
-version = "2.1.8"
+version = "2.1.9"
 description = "Python SDK and CLI for UiPath Platform, enabling programmatic interaction with automation services, process management, and deployment tools."
 readme = { file = "README.md", content-type = "text/markdown" }
 requires-python = ">=3.10"
@@ -103,7 +103,6 @@ line-ending = "auto"
 plugins = ["pydantic.mypy"]
 exclude = ["samples/.*"]
 
-
 follow_imports = "silent"
 warn_redundant_casts = true
 warn_unused_ignores = true

src/uipath/_cli/__init__.py

@@ -5,6 +5,7 @@
 
 from .cli_auth import auth as auth  # type: ignore
 from .cli_deploy import deploy as deploy  # type: ignore
+from .cli_eval import eval as eval  # type: ignore
 from .cli_init import init as init  # type: ignore
 from .cli_invoke import invoke as invoke  # type: ignore
 from .cli_new import new as new  # type: ignore
@@ -67,3 +68,4 @@ def cli(lv: bool, v: bool) -> None:
 cli.add_command(invoke)
 cli.add_command(push)
 cli.add_command(pull)
+cli.add_command(eval)

src/uipath/_cli/_evals/_evaluators/__init__.py

@@ -0,0 +1,20 @@
+"""Evaluators package for the evaluation system.
+
+This package contains all evaluator types and the factory for creating them.
+"""
+
+from ._agent_scorer_evaluator import AgentScorerEvaluator
+from ._deterministic_evaluator import DeterministicEvaluator
+from ._evaluator_base import EvaluatorBase
+from ._evaluator_factory import EvaluatorFactory
+from ._llm_as_judge_evaluator import LlmAsAJudgeEvaluator
+from ._trajectory_evaluator import TrajectoryEvaluator
+
+__all__ = [
+    "EvaluatorBase",
+    "EvaluatorFactory",
+    "DeterministicEvaluator",
+    "LlmAsAJudgeEvaluator",
+    "AgentScorerEvaluator",
+    "TrajectoryEvaluator",
+]

src/uipath/_cli/_evals/_evaluators/_agent_scorer_evaluator.py

@@ -0,0 +1,48 @@
+from typing import Any, Dict
+
+from .._models import EvaluationResult
+from ._evaluator_base import EvaluatorBase
+
+
+class AgentScorerEvaluator(EvaluatorBase):
+    """Evaluator that uses an agent to score outputs."""
+
+    def __init__(
+        self,
+        agent_config: Dict[str, Any],
+        scoring_criteria: Dict[str, Any],
+        target_output_key: str = "*",
+    ):
+        """Initialize the agent scorer evaluator.
+
+        Args:
+            agent_config: Configuration for the scoring agent
+            scoring_criteria: Criteria used for scoring
+            target_output_key: Key in output to evaluate ("*" for entire output)
+        """
+        super().__init__()
+        self.agent_config = agent_config or {}
+        self.scoring_criteria = scoring_criteria or {}
+        self.target_output_key = target_output_key
+
+    async def evaluate(
+        self,
+        evaluation_id: str,
+        evaluation_name: str,
+        input_data: Dict[str, Any],
+        expected_output: Dict[str, Any],
+        actual_output: Dict[str, Any],
+    ) -> EvaluationResult:
+        """Evaluate using an agent scorer.
+
+        Args:
+            evaluation_id: The ID of the evaluation being processed
+            evaluation_name: The name of the evaluation
+            input_data: The input data for the evaluation
+            expected_output: The expected output
+            actual_output: The actual output from the agent
+
+        Returns:
+            EvaluationResult containing the score and details
+        """
+        raise NotImplementedError()

src/uipath/_cli/_evals/_evaluators/_deterministic_evaluator.py

@@ -0,0 +1,41 @@
+from typing import Any, Dict
+
+from .._models import EvaluationResult
+from ._evaluator_base import EvaluatorBase
+
+
+class DeterministicEvaluator(EvaluatorBase):
+    """Evaluator for deterministic/rule-based evaluations."""
+
+    def __init__(self, rule_config: Dict[str, Any], target_output_key: str = "*"):
+        """Initialize the deterministic evaluator.
+
+        Args:
+            rule_config: Configuration for the rule (expected_value, regex_pattern, etc.)
+            target_output_key: Key in output to evaluate ("*" for entire output)
+        """
+        super().__init__()
+        self.rule_config = rule_config or {}
+        self.target_output_key = target_output_key
+
+    async def evaluate(
+        self,
+        evaluation_id: str,
+        evaluation_name: str,
+        input_data: Dict[str, Any],
+        expected_output: Dict[str, Any],
+        actual_output: Dict[str, Any],
+    ) -> EvaluationResult:
+        """Evaluate using deterministic rules.
+
+        Args:
+            evaluation_id: The ID of the evaluation being processed
+            evaluation_name: The name of the evaluation
+            input_data: The input data for the evaluation
+            expected_output: The expected output
+            actual_output: The actual output from the agent
+
+        Returns:
+            EvaluationResult containing the score and details
+        """
+        raise NotImplementedError()

src/uipath/_cli/_evals/_evaluators/_evaluator_base.py

@@ -0,0 +1,124 @@
+import functools
+import time
+from abc import ABC, abstractmethod
+from dataclasses import dataclass
+from typing import Any, Dict
+
+from uipath._cli._evals._models import (
+    EvaluationResult,
+    EvaluatorCategory,
+    EvaluatorType,
+)
+
+
+def measure_execution_time(func):
+    """Decorator to measure execution time and update EvaluationResult.evaluation_time."""
+
+    @functools.wraps(func)
+    async def wrapper(*args, **kwargs) -> EvaluationResult:
+        start_time = time.time()
+        result = await func(*args, **kwargs)
+        end_time = time.time()
+        execution_time = end_time - start_time
+
+        result.evaluation_time = execution_time
+        return result
+
+    return wrapper
+
+
+@dataclass
+class EvaluatorBaseParams:
+    """Parameters for initializing the base evaluator."""
+
+    evaluator_id: str
+    category: EvaluatorCategory
+    evaluator_type: EvaluatorType
+    name: str
+    description: str
+    created_at: str
+    updated_at: str
+    target_output_key: str
+
+
+class EvaluatorBase(ABC):
+    """Abstract base class for all evaluators."""
+
+    def __init__(self):
+        # initialization done via 'from_params' function
+        self.id: str
+        self.name: str
+        self.description: str
+        self.created_at: str
+        self.updated_at: str
+        self.category: EvaluatorCategory
+        self.type: EvaluatorType
+        self.target_output_key: str
+        pass
+
+    @classmethod
+    def from_params(cls, params: EvaluatorBaseParams, **kwargs):
+        """Initialize the base evaluator from parameters.
+
+        Args:
+            params: EvaluatorBaseParams containing base configuration
+            **kwargs: Additional specific parameters for concrete evaluators
+
+        Returns:
+            Initialized evaluator instance
+        """
+        instance = cls(**kwargs)
+        instance.id = params.evaluator_id
+        instance.category = params.category
+        instance.type = params.evaluator_type
+        instance.name = params.name
+        instance.description = params.description
+        instance.created_at = params.created_at
+        instance.updated_at = params.updated_at
+        instance.target_output_key = params.target_output_key
+        return instance
+
+    @measure_execution_time
+    @abstractmethod
+    async def evaluate(
+        self,
+        evaluation_id: str,
+        evaluation_name: str,
+        input_data: Dict[str, Any],
+        expected_output: Dict[str, Any],
+        actual_output: Dict[str, Any],
+    ) -> EvaluationResult:
+        """Evaluate the given data and return a result.
+
+        Args:
+            evaluation_id: The ID of the evaluation being processed
+            evaluation_name: The name of the evaluation
+            input_data: The input data for the evaluation
+            expected_output: The expected output
+            actual_output: The actual output from the agent
+
+        Returns:
+            EvaluationResult containing the score and details
+        """
+        pass
+
+    def to_dict(self) -> Dict[str, Any]:
+        """Convert the evaluator instance to a dictionary representation.
+
+        Returns:
+            Dict[str, Any]: Dictionary containing all evaluator properties
+        """
+        return {
+            "id": self.id,
+            "name": self.name,
+            "description": self.description,
+            "created_at": self.created_at,
+            "updated_at": self.updated_at,
+            "category": self.category.name if self.category else None,
+            "type": self.type.name if self.type else None,
+            "target_output_key": self.target_output_key,
+        }
+
+    def __repr__(self) -> str:
+        """String representation of the evaluator."""
+        return f"{self.__class__.__name__}(id='{self.id}', name='{self.name}', category={self.category.name})"

src/uipath/_cli/_evals/_evaluators/_evaluator_factory.py

@@ -0,0 +1,103 @@
+from typing import Any, Dict
+
+from .._models import EvaluatorCategory, EvaluatorType
+from ._agent_scorer_evaluator import AgentScorerEvaluator
+from ._deterministic_evaluator import DeterministicEvaluator
+from ._evaluator_base import EvaluatorBase, EvaluatorBaseParams
+from ._llm_as_judge_evaluator import LlmAsAJudgeEvaluator
+from ._trajectory_evaluator import TrajectoryEvaluator
+
+
+class EvaluatorFactory:
+    """Factory class for creating evaluator instances based on configuration."""
+
+    @staticmethod
+    def create_evaluator(data: Dict[str, Any]) -> EvaluatorBase:
+        """Create an evaluator instance from configuration data.
+
+        Args:
+            data: Dictionary containing evaluator configuration from JSON file
+
+        Returns:
+            Appropriate evaluator instance based on category
+
+        Raises:
+            ValueError: If category is unknown or required fields are missing
+        """
+        # Extract common fields
+        evaluator_id = data.get("id")
+        if not evaluator_id:
+            raise ValueError("Evaluator configuration must include 'id' field")
+
+        category = EvaluatorCategory.from_int(data.get("category"))
+        evaluator_type = EvaluatorType.from_int(data.get("type", EvaluatorType.Unknown))
+        name = data.get("name", "")
+        description = data.get("description", "")
+        created_at = data.get("createdAt", "")
+        updated_at = data.get("updatedAt", "")
+        target_output_key = data.get("targetOutputKey", "")
+
+        # Create base parameters
+        base_params = EvaluatorBaseParams(
+            evaluator_id=evaluator_id,
+            category=category,
+            evaluator_type=evaluator_type,
+            name=name,
+            description=description,
+            created_at=created_at,
+            updated_at=updated_at,
+            target_output_key=target_output_key,
+        )
+
+        # Create evaluator based on category
+        if category == EvaluatorCategory.Deterministic:
+            return EvaluatorFactory._create_deterministic_evaluator(base_params, data)
+        elif category == EvaluatorCategory.LlmAsAJudge:
+            return EvaluatorFactory._create_llm_as_judge_evaluator(base_params, data)
+        elif category == EvaluatorCategory.AgentScorer:
+            return EvaluatorFactory._create_agent_scorer_evaluator(base_params, data)
+        elif category == EvaluatorCategory.Trajectory:
+            return EvaluatorFactory._create_trajectory_evaluator(base_params, data)
+        else:
+            raise ValueError(f"Unknown evaluator category: {category}")
+
+    @staticmethod
+    def _create_deterministic_evaluator(
+        base_params: EvaluatorBaseParams, data: Dict[str, Any]
+    ) -> DeterministicEvaluator:
+        """Create a deterministic evaluator."""
+        raise NotImplementedError()
+
+    @staticmethod
+    def _create_llm_as_judge_evaluator(
+        base_params: EvaluatorBaseParams, data: Dict[str, Any]
+    ) -> LlmAsAJudgeEvaluator:
+        """Create an LLM-as-a-judge evaluator."""
+        prompt = data.get("prompt", "")
+        if not prompt:
+            raise ValueError("LLM evaluator must include 'prompt' field")
+
+        model = data.get("model", "")
+        if not model:
+            raise ValueError("LLM evaluator must include 'model' field")
+
+        return LlmAsAJudgeEvaluator.from_params(
+            base_params,
+            prompt=prompt,
+            model=model,
+            target_output_key=data.get("targetOutputKey", "*"),
+        )
+
+    @staticmethod
+    def _create_agent_scorer_evaluator(
+        base_params: EvaluatorBaseParams, data: Dict[str, Any]
+    ) -> AgentScorerEvaluator:
+        """Create an agent scorer evaluator."""
+        raise NotImplementedError()
+
+    @staticmethod
+    def _create_trajectory_evaluator(
+        base_params: EvaluatorBaseParams, data: Dict[str, Any]
+    ) -> TrajectoryEvaluator:
+        """Create a trajectory evaluator."""
+        raise NotImplementedError()