feat: run evaluations

Author: radu-mocanu

pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "uipath"
-version = "2.1.46"
+version = "2.1.47"
 description = "Python SDK and CLI for UiPath Platform, enabling programmatic interaction with automation services, process management, and deployment tools."
 readme = { file = "README.md", content-type = "text/markdown" }
 requires-python = ">=3.10"

…_evals/_evaluators/_evaluator_factory.py …uipath/_cli/_evals/_evaluator_factory.pysrc/uipath/_cli/_evals/_evaluators/_evaluator_factory.py renamed to src/uipath/_cli/_evals/_evaluator_factory.py src/uipath/_cli/_evals/_evaluators/_evaluator_factory.py renamed to src/uipath/_cli/_evals/_evaluator_factory.py

@@ -1,18 +1,25 @@
+import os
+from pathlib import Path
 from typing import Any, Dict
 
-from .._models import EvaluatorCategory, EvaluatorType
-from ._evaluator_base import EvaluatorBase, EvaluatorBaseParams
-from ._exact_match_evaluator import ExactMatchEvaluator
-from ._json_similarity_evaluator import JsonSimilarityEvaluator
-from ._llm_as_judge_evaluator import LlmAsAJudgeEvaluator
-from ._trajectory_evaluator import TrajectoryEvaluator
+from uipath._cli._evals._models._evaluator_base_params import EvaluatorBaseParams
+from uipath.eval.evaluators import (
+    BaseEvaluator,
+    ExactMatchEvaluator,
+    JsonSimilarityEvaluator,
+    LlmAsAJudgeEvaluator,
+    TrajectoryEvaluator,
+)
+from uipath.eval.models.models import EvaluatorCategory, EvaluatorType
 
 
 class EvaluatorFactory:
     """Factory class for creating evaluator instances based on configuration."""
 
-    @staticmethod
-    def create_evaluator(data: Dict[str, Any]) -> EvaluatorBase:
+    coded_evals_dir_path: Path = os.path.join("evals", "evaluators", "coded")
+
+    @classmethod
+    def create_evaluator(cls, data: Dict[str, Any]) -> BaseEvaluator:
         """Create an evaluator instance from configuration data.
 
         Args:
@@ -25,21 +32,23 @@ def create_evaluator(data: Dict[str, Any]) -> EvaluatorBase:
             ValueError: If category is unknown or required fields are missing
         """
         # Extract common fields
-        evaluator_id = data.get("id")
-        if not evaluator_id:
+        name = data.get("name", "")
+        if not name:
+            raise ValueError("Evaluator configuration must include 'name' field")
+        id = data.get("id", "")
+        if not id:
             raise ValueError("Evaluator configuration must include 'id' field")
 
         category = EvaluatorCategory.from_int(data.get("category"))
         evaluator_type = EvaluatorType.from_int(data.get("type", EvaluatorType.Unknown))
-        name = data.get("name", "")
         description = data.get("description", "")
         created_at = data.get("createdAt", "")
         updated_at = data.get("updatedAt", "")
         target_output_key = data.get("targetOutputKey", "")
 
         # Create base parameters
         base_params = EvaluatorBaseParams(
-            evaluator_id=evaluator_id,
+            id=id,
             category=category,
             evaluator_type=evaluator_type,
             name=name,
@@ -49,7 +58,6 @@ def create_evaluator(data: Dict[str, Any]) -> EvaluatorBase:
             target_output_key=target_output_key,
         )
 
-        # Create evaluator based on category
         match category:
             case EvaluatorCategory.Deterministic:
                 if evaluator_type == evaluator_type.Equals:
@@ -81,8 +89,7 @@ def _create_exact_match_evaluator(
     ) -> ExactMatchEvaluator:
         """Create a deterministic evaluator."""
         return ExactMatchEvaluator.from_params(
-            base_params,
-            target_output_key=data.get("targetOutputKey", "*"),
+            **base_params.model_dump(),
         )
 
     @staticmethod
@@ -91,8 +98,7 @@ def _create_json_similarity_evaluator(
     ) -> JsonSimilarityEvaluator:
         """Create a deterministic evaluator."""
         return JsonSimilarityEvaluator.from_params(
-            base_params,
-            target_output_key=data.get("targetOutputKey", "*"),
+            **base_params.model_dump(),
         )
 
     @staticmethod
@@ -113,10 +119,9 @@ def _create_llm_as_judge_evaluator(
             )
 
         return LlmAsAJudgeEvaluator.from_params(
-            base_params,
+            **base_params.model_dump(),
             prompt=prompt,
             model=model,
-            target_output_key=data.get("targetOutputKey", "*"),
         )
 
     @staticmethod