Merge pull request #578 from UiPath/feat/run-evals

feat: run evaluations

Author: radu-mocanu

pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "uipath"
-version = "2.1.52"
+version = "2.1.53"
 description = "Python SDK and CLI for UiPath Platform, enabling programmatic interaction with automation services, process management, and deployment tools."
 readme = { file = "README.md", content-type = "text/markdown" }
 requires-python = ">=3.10"

…_evals/_evaluators/_evaluator_factory.py …uipath/_cli/_evals/_evaluator_factory.pysrc/uipath/_cli/_evals/_evaluators/_evaluator_factory.py renamed to src/uipath/_cli/_evals/_evaluator_factory.py src/uipath/_cli/_evals/_evaluators/_evaluator_factory.py renamed to src/uipath/_cli/_evals/_evaluator_factory.py

@@ -1,18 +1,21 @@
 from typing import Any, Dict
 
-from .._models import EvaluatorCategory, EvaluatorType
-from ._evaluator_base import EvaluatorBase, EvaluatorBaseParams
-from ._exact_match_evaluator import ExactMatchEvaluator
-from ._json_similarity_evaluator import JsonSimilarityEvaluator
-from ._llm_as_judge_evaluator import LlmAsAJudgeEvaluator
-from ._trajectory_evaluator import TrajectoryEvaluator
+from uipath._cli._evals._models._evaluator_base_params import EvaluatorBaseParams
+from uipath.eval.evaluators import (
+    BaseEvaluator,
+    ExactMatchEvaluator,
+    JsonSimilarityEvaluator,
+    LlmAsAJudgeEvaluator,
+    TrajectoryEvaluator,
+)
+from uipath.eval.models.models import EvaluatorCategory, EvaluatorType
 
 
 class EvaluatorFactory:
     """Factory class for creating evaluator instances based on configuration."""
 
-    @staticmethod
-    def create_evaluator(data: Dict[str, Any]) -> EvaluatorBase:
+    @classmethod
+    def create_evaluator(cls, data: Dict[str, Any]) -> BaseEvaluator[Any]:
         """Create an evaluator instance from configuration data.
 
         Args:
@@ -25,21 +28,23 @@ def create_evaluator(data: Dict[str, Any]) -> EvaluatorBase:
             ValueError: If category is unknown or required fields are missing
         """
         # Extract common fields
-        evaluator_id = data.get("id")
-        if not evaluator_id:
+        name = data.get("name", "")
+        if not name:
+            raise ValueError("Evaluator configuration must include 'name' field")
+        id = data.get("id", "")
+        if not id:
             raise ValueError("Evaluator configuration must include 'id' field")
 
         category = EvaluatorCategory.from_int(data.get("category"))
         evaluator_type = EvaluatorType.from_int(data.get("type", EvaluatorType.Unknown))
-        name = data.get("name", "")
         description = data.get("description", "")
         created_at = data.get("createdAt", "")
         updated_at = data.get("updatedAt", "")
         target_output_key = data.get("targetOutputKey", "")
 
         # Create base parameters
         base_params = EvaluatorBaseParams(
-            evaluator_id=evaluator_id,
+            id=id,
             category=category,
             evaluator_type=evaluator_type,
             name=name,
@@ -49,7 +54,6 @@ def create_evaluator(data: Dict[str, Any]) -> EvaluatorBase:
             target_output_key=target_output_key,
         )
 
-        # Create evaluator based on category
         match category:
             case EvaluatorCategory.Deterministic:
                 if evaluator_type == evaluator_type.Equals:
@@ -80,19 +84,17 @@ def _create_exact_match_evaluator(
         base_params: EvaluatorBaseParams, data: Dict[str, Any]
     ) -> ExactMatchEvaluator:
         """Create a deterministic evaluator."""
-        return ExactMatchEvaluator.from_params(
-            base_params,
-            target_output_key=data.get("targetOutputKey", "*"),
+        return ExactMatchEvaluator(
+            **base_params.model_dump(),
         )
 
     @staticmethod
     def _create_json_similarity_evaluator(
         base_params: EvaluatorBaseParams, data: Dict[str, Any]
     ) -> JsonSimilarityEvaluator:
         """Create a deterministic evaluator."""
-        return JsonSimilarityEvaluator.from_params(
-            base_params,
-            target_output_key=data.get("targetOutputKey", "*"),
+        return JsonSimilarityEvaluator(
+            **base_params.model_dump(),
         )
 
     @staticmethod
@@ -112,16 +114,15 @@ def _create_llm_as_judge_evaluator(
                 "'same-as-agent' model option is not supported by coded agents evaluations. Please select a specific model for the evaluator."
             )
 
-        return LlmAsAJudgeEvaluator.from_params(
-            base_params,
+        return LlmAsAJudgeEvaluator(
+            **base_params.model_dump(),
             prompt=prompt,
             model=model,
-            target_output_key=data.get("targetOutputKey", "*"),
         )
 
     @staticmethod
     def _create_trajectory_evaluator(
         base_params: EvaluatorBaseParams, data: Dict[str, Any]
-    ) -> TrajectoryEvaluator:
+    ) -> TrajectoryEvaluator[Any]:
         """Create a trajectory evaluator."""
         raise NotImplementedError()