feat: add AgentEvaluator

Author: radu-mocanu

pyproject.toml

@@ -10,7 +10,7 @@ dependencies = [
   "opentelemetry-sdk>=1.31.1",
   "pydantic>=2.11.1",
   "python-dotenv>=1.0.1",
-  "tenacity>=9.0.0",
+  "tenacity>=8.0.0",
   "tomli>=2.2.1",
   "pathlib>=1.0.1",
   "rich>=13.0.0",
@@ -101,7 +101,7 @@ line-ending = "auto"
 
 [tool.mypy]
 plugins = ["pydantic.mypy"]
-exclude = ["samples/.*"]
+exclude = ["samples/.*", "tests/.*"]
 
 follow_imports = "silent"
 warn_redundant_casts = true

…uipath/_cli/_evals/evaluation_service.py …ipath/_cli/_evals/_evaluation_service.pysrc/uipath/_cli/_evals/evaluation_service.py renamed to src/uipath/_cli/_evals/_evaluation_service.py src/uipath/_cli/_evals/evaluation_service.py renamed to src/uipath/_cli/_evals/_evaluation_service.py

@@ -1,18 +1,22 @@
 from typing import Any, Dict
 
-from .._models import EvaluatorCategory, EvaluatorType
-from ._evaluator_base import EvaluatorBase, EvaluatorBaseParams
-from ._exact_match_evaluator import ExactMatchEvaluator
-from ._json_similarity_evaluator import JsonSimilarityEvaluator
-from ._llm_as_judge_evaluator import LlmAsAJudgeEvaluator
-from ._trajectory_evaluator import TrajectoryEvaluator
+from uipath.eval.evaluators import (
+    BaseEvaluator,
+    ExactMatchEvaluator,
+    JsonSimilarityEvaluator,
+    LlmAsAJudgeEvaluator,
+    TrajectoryEvaluator,
+)
+from uipath.eval.models import EvaluatorCategory, EvaluatorType
+
+from ._models import EvaluatorBaseParams
 
 
 class EvaluatorFactory:
     """Factory class for creating evaluator instances based on configuration."""
 
     @staticmethod
-    def create_evaluator(data: Dict[str, Any]) -> EvaluatorBase:
+    def create_evaluator(data: Dict[str, Any]) -> BaseEvaluator:
         """Create an evaluator instance from configuration data.
 
         Args:
@@ -81,8 +85,7 @@ def _create_exact_match_evaluator(
     ) -> ExactMatchEvaluator:
         """Create a deterministic evaluator."""
         return ExactMatchEvaluator.from_params(
-            base_params,
-            target_output_key=data.get("targetOutputKey", "*"),
+            **base_params.model_dump(),
         )
 
     @staticmethod
@@ -91,8 +94,7 @@ def _create_json_similarity_evaluator(
     ) -> JsonSimilarityEvaluator:
         """Create a deterministic evaluator."""
         return JsonSimilarityEvaluator.from_params(
-            base_params,
-            target_output_key=data.get("targetOutputKey", "*"),
+            **base_params.model_dump(),
         )
 
     @staticmethod
@@ -113,10 +115,9 @@ def _create_llm_as_judge_evaluator(
             )
 
         return LlmAsAJudgeEvaluator.from_params(
-            base_params,
+            **base_params.model_dump(),
             prompt=prompt,
             model=model,
-            target_output_key=data.get("targetOutputKey", "*"),
         )
 
     @staticmethod

…_evals/_evaluators/_evaluator_factory.py …uipath/_cli/_evals/_evaluator_factory.pysrc/uipath/_cli/_evals/_evaluators/_evaluator_factory.py renamed to src/uipath/_cli/_evals/_evaluator_factory.py src/uipath/_cli/_evals/_evaluators/_evaluator_factory.py renamed to src/uipath/_cli/_evals/_evaluator_factory.py

@@ -1,18 +1,12 @@
 from uipath._cli._evals._models._evaluation_set import EvaluationItem, EvaluationSet
 from uipath._cli._evals._models._evaluators import (
-    EvaluationResult,
     EvaluationSetResult,
-    EvaluatorCategory,
-    EvaluatorType,
-    LLMResponse,
+    EvaluatorBaseParams,
 )
 
 __all__ = [
-    "LLMResponse",
-    "EvaluatorCategory",
-    "EvaluatorType",
-    "EvaluationResult",
     "EvaluationSetResult",
     "EvaluationItem",
     "EvaluationSet",
+    "EvaluatorBaseParams",
 ]

src/uipath/_cli/_evals/_evaluators/__init__.py

@@ -36,7 +36,7 @@ class EvaluationSet(BaseModel):
     createdAt: str
     updatedAt: str
 
-    def extract_selected_evals(self, eval_ids) -> None:
+    def extract_selected_evals(self, eval_ids: list[str]) -> None:
         selected_evals: list[EvaluationItem] = []
         for evaluation in self.evaluations:
             if evaluation.id in eval_ids: