fix: move some functions to utility and dedup the legacy and coded

Author: AAgnihotry

packages/uipath/samples/line_by_line_test/README.md

@@ -45,17 +45,23 @@ uv run uipath eval main evaluations/eval-sets/default.json --workers 1
 
 ## Evaluation Results
 
-The sample includes three test cases with three evaluators:
+The sample includes three test cases with five evaluators:
+
+### ExactMatch Evaluators
 - **LineByLineExactMatch** - New evaluator with line-by-line support
 - **RegularExactMatch** - New evaluator without line-by-line (for comparison)
 - **LegacyLineByLineExactMatch** - Legacy evaluator with line-by-line support
 
+### Contains Evaluators
+- **LineByLineContains** - New evaluator with line-by-line support (checks if each line contains the search text)
+- **RegularContains** - New evaluator without line-by-line (checks if the entire output contains the search text)
+
 Test cases:
 1. **All lines match exactly** - All evaluators score 1.0
-2. **One line doesn't match** - Line-by-line evaluators: 0.67, Regular: 0.0 (shows partial credit!)
+2. **One line doesn't match** - Line-by-line ExactMatch: 0.67, Regular ExactMatch: 0.0 (shows partial credit!)
 3. **Single item** - All evaluators score 1.0
 
-Expected output:
+Expected output (showing ExactMatch evaluators):
 ```
 ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓
 ┃  Evaluation                   ┃  LineByLineExactMatch  ┃  RegularExactMatch  ┃  LegacyLineByLineExactMatch  ┃
@@ -68,6 +74,8 @@ Expected output:
 └───────────────────────────────┴────────────────────────┴─────────────────────┴───────────────────────────────┘
 ```
 
+Contains evaluators will all score 1.0 since all test outputs contain "Item:".
+
 ## Configuration
 
 ### Evaluator Configuration
@@ -104,9 +112,59 @@ Legacy evaluators also support line-by-line evaluation in `evaluations/evaluator
 }
 ```
 
-Key options for both evaluator types:
+#### Contains Evaluators
+
+The Contains evaluator checks if the output contains a specific search text. In line-by-line mode, it checks each line independently:
+
+**Line-by-line Contains** (`evaluations/evaluators/line-by-line-contains.json`):
+```json
+{
+  "version": "1.0",
+  "evaluatorTypeId": "uipath-contains",
+  "evaluatorConfig": {
+    "name": "LineByLineContains",
+    "target_output_key": "result",
+    "line_by_line_evaluator": true,
+    "line_delimiter": "\n",
+    "case_sensitive": false,
+    "negated": false
+  }
+}
+```
+
+**Regular Contains** (`evaluations/evaluators/regular-contains.json`):
+```json
+{
+  "version": "1.0",
+  "evaluatorTypeId": "uipath-contains",
+  "evaluatorConfig": {
+    "name": "RegularContains",
+    "target_output_key": "result",
+    "line_by_line_evaluator": false,
+    "case_sensitive": false,
+    "negated": false
+  }
+}
+```
+
+In evaluation criteria, specify the search text:
+```json
+{
+  "LineByLineContains": {
+    "searchText": "Item:"
+  }
+}
+```
+
+**Behavior difference**:
+- **Line-by-line**: Checks if each line contains "Item:", gives partial credit (e.g., 2/3 if one line is missing it)
+- **Regular**: Checks if the entire output contains "Item:" at least once, returns 1.0 or 0.0
+
+Key options for all evaluator types:
 - `lineByLineEvaluator`/`lineByLineEvaluation`: Enable line-by-line evaluation (default: `false`)
 - `lineDelimiter`: Delimiter to split lines (default: `"\n"`)
+- `case_sensitive`: Case-sensitive comparison (default: `false` for Contains, `true` for ExactMatch)
+- `negated`: Invert the result (default: `false`, only for Contains)
 
 ### Custom Delimiters
 
@@ -130,11 +188,13 @@ line_by_line_test/
 ├── pyproject.toml                               # Dependencies (uses TestPyPI)
 └── evaluations/
     ├── evaluators/
-    │   ├── line-by-line-exact-match.json           # New line-by-line evaluator
-    │   ├── regular-exact-match.json                 # New regular evaluator (for comparison)
-    │   └── legacy-line-by-line-exact-match.json    # Legacy line-by-line evaluator
+    │   ├── line-by-line-exact-match.json           # New line-by-line ExactMatch evaluator
+    │   ├── regular-exact-match.json                 # New regular ExactMatch evaluator
+    │   ├── legacy-line-by-line-exact-match.json    # Legacy line-by-line ExactMatch evaluator
+    │   ├── line-by-line-contains.json              # New line-by-line Contains evaluator
+    │   └── regular-contains.json                    # New regular Contains evaluator
     └── eval-sets/
-        └── default.json                             # Test cases
+        └── default.json                             # Test cases with all 5 evaluators
 ```
 
 ## Learn More

packages/uipath/samples/line_by_line_test/evaluations/eval-sets/default.json

@@ -5,14 +5,20 @@
   "evaluatorRefs": [
     "LineByLineExactMatch",
     "RegularExactMatch",
-    "LegacyLineByLineExactMatch"
+    "LegacyLineByLineExactMatch",
+    "LineByLineContains",
+    "RegularContains"
   ],
   "evaluations": [
     {
       "id": "test-all-lines-match",
       "name": "Test all lines match exactly",
       "inputs": {
-        "items": ["apple", "banana", "cherry"]
+        "items": [
+          "apple",
+          "banana",
+          "cherry"
+        ]
       },
       "evaluationCriterias": {
         "LineByLineExactMatch": {
@@ -30,14 +36,24 @@
             "result": "Item: apple\nItem: banana\nItem: cherry"
           },
           "expectedAgentBehavior": ""
+        },
+        "LineByLineContains": {
+          "searchText": "apple"
+        },
+        "RegularContains": {
+          "searchText": "apple"
         }
       }
     },
     {
       "id": "test-partial-line-mismatch",
       "name": "Test when one line doesn't match",
       "inputs": {
-        "items": ["apple", "banana", "cherry"]
+        "items": [
+          "apple",
+          "banana",
+          "cherry"
+        ]
       },
       "evaluationCriterias": {
         "LineByLineExactMatch": {
@@ -55,14 +71,22 @@
             "result": "Item: apple\nItem: WRONG\nItem: cherry"
           },
           "expectedAgentBehavior": ""
+        },
+        "LineByLineContains": {
+          "searchText": "Item:"
+        },
+        "RegularContains": {
+          "searchText": "Item:"
         }
       }
     },
     {
       "id": "test-single-item",
       "name": "Test with single item",
       "inputs": {
-        "items": ["orange"]
+        "items": [
+          "orange"
+        ]
       },
       "evaluationCriterias": {
         "LineByLineExactMatch": {
@@ -80,8 +104,14 @@
             "result": "Item: orange"
           },
           "expectedAgentBehavior": ""
+        },
+        "LineByLineContains": {
+          "searchText": "Item:"
+        },
+        "RegularContains": {
+          "searchText": "Item:"
         }
       }
     }
   ]
-}
+}

packages/uipath/samples/line_by_line_test/evaluations/evaluators/line-by-line-contains.json

@@ -0,0 +1,14 @@
+{
+  "version": "1.0",
+  "evaluatorTypeId": "uipath-contains",
+  "id": "LineByLineContains",
+  "name": "LineByLineContains",
+  "evaluatorConfig": {
+    "name": "LineByLineContains",
+    "target_output_key": "result",
+    "line_by_line_evaluator": true,
+    "line_delimiter": "\n",
+    "case_sensitive": false,
+    "negated": false
+  }
+}

packages/uipath/samples/line_by_line_test/evaluations/evaluators/regular-contains.json

@@ -0,0 +1,13 @@
+{
+  "version": "1.0",
+  "evaluatorTypeId": "uipath-contains",
+  "id": "RegularContains",
+  "name": "RegularContains",
+  "evaluatorConfig": {
+    "name": "RegularContains",
+    "target_output_key": "result",
+    "line_by_line_evaluator": false,
+    "case_sensitive": false,
+    "negated": false
+  }
+}

packages/uipath/src/uipath/eval/evaluators/__init__.py

@@ -34,6 +34,7 @@
     LLMJudgeTrajectorySimulationEvaluator,
 )
 from .multiclass_classification_evaluator import MulticlassClassificationEvaluator
+from .output_evaluator import AggregationMethod
 from .tool_call_args_evaluator import ToolCallArgsEvaluator
 from .tool_call_count_evaluator import ToolCallCountEvaluator
 from .tool_call_order_evaluator import ToolCallOrderEvaluator
@@ -84,4 +85,5 @@
     "BaseEvaluatorConfig",
     "BaseEvaluatorJustification",
     "LLMJudgeJustification",
+    "AggregationMethod",
 ]