lint

Signed-off-by: Anxhela Coba <acoba@redhat.com>

fix path

Signed-off-by: Anxhela Coba <acoba@redhat.com>

fix lint

Signed-off-by: Anxhela Coba <acoba@redhat.com>

fix e2e

Signed-off-by: Anxhela Coba <acoba@redhat.com>

lint

Signed-off-by: Anxhela Coba <acoba@redhat.com>

fix e2e

Signed-off-by: Anxhela Coba <acoba@redhat.com>

fix e2e

Signed-off-by: Anxhela Coba <acoba@redhat.com>

fix e2e

Signed-off-by: Anxhela Coba <acoba@redhat.com>

test

Signed-off-by: Anxhela Coba <acoba@redhat.com>

test

Signed-off-by: Anxhela Coba <acoba@redhat.com>

test

Signed-off-by: Anxhela Coba <acoba@redhat.com>

add doc_id to referenced docs

Signed-off-by: Anxhela Coba <acoba@redhat.com>

update unit tests

Signed-off-by: Anxhela Coba <acoba@redhat.com>

Author: Anxhela21

docker-compose.yaml

@@ -24,6 +24,7 @@ services:
     environment:
       - BRAVE_SEARCH_API_KEY=${BRAVE_SEARCH_API_KEY:-}
       - TAVILY_SEARCH_API_KEY=${TAVILY_SEARCH_API_KEY:-}
+      - EXTERNAL_PROVIDERS_DIR=${EXTERNAL_PROVIDERS_DIR:-/opt/app-root/external_providers}
       # OpenAI
       - OPENAI_API_KEY=${OPENAI_API_KEY}
       - E2E_OPENAI_MODEL=${E2E_OPENAI_MODEL:-gpt-4o-mini}

src/models/common/turn_summary.py

@@ -31,6 +31,7 @@ class ReferencedDocument(BaseModel):
     Attributes:
         doc_url: Url to the referenced doc.
         doc_title: Title of the referenced doc.
+        document_id: Document ID for preserving identity during deduplication.
     """
 
     doc_url: Optional[AnyUrl] = Field(
@@ -46,6 +47,11 @@ class ReferencedDocument(BaseModel):
         description="Index name identifying the knowledge source from configuration",
     )
 
+    document_id: Optional[str] = Field(
+        default=None,
+        description="Document ID for preserving identity during deduplication",
+    )
+
 
 class RAGContext(BaseModel):
     """Result of building RAG context from all enabled pre-query RAG sources.

src/utils/reranker.py

@@ -31,7 +31,7 @@ async def _get_cross_encoder(model_name: str) -> Any:
         Loaded CrossEncoder model instance, or None if loading fails.
     """
     # Check if reranking is enabled before attempting to load the model
-    if not configuration.reranker.enabled:
+    if not configuration.reranker.enabled:  # pylint: disable=no-member
         logger.debug("Reranker is disabled, not loading cross-encoder model")
         return None
 
@@ -41,9 +41,9 @@ async def _get_cross_encoder(model_name: str) -> Any:
         if model_name in _cross_encoder_models:
             return _cross_encoder_models[model_name]
         try:
-            from sentence_transformers import (
+            from sentence_transformers import (  # pylint: disable=import-outside-toplevel
                 CrossEncoder,
-            )  # pylint: disable=import-outside-toplevel
+            )
 
             model = await asyncio.to_thread(CrossEncoder, model_name)
             _cross_encoder_models[model_name] = model
@@ -162,12 +162,7 @@ async def rerank_chunks_with_cross_encoder(
 
         # Return RAGChunk list with combined scores
         return [
-            RAGChunk(
-                content=chunk.content,
-                source=chunk.source,
-                score=float(score),
-                attributes=chunk.attributes,
-            )
+            chunk.model_copy(update={"score": float(score)})
             for score, chunk in top_indexed
         ]
 
@@ -201,16 +196,10 @@ def apply_byok_rerank_boost(
         score = chunk.score if chunk.score is not None else float("-inf")
         if chunk.source != constants.OKP_RAG_ID:
             score = score * boost
-        boosted.append(
-            RAGChunk(
-                content=chunk.content,
-                source=chunk.source,
-                score=score,
-                attributes=chunk.attributes,
-            )
-        )
+        boosted.append(chunk.model_copy(update={"score": score}))
     boosted.sort(
         key=lambda c: c.score if c.score is not None else float("-inf"),
         reverse=True,
     )
-    return boosted
+
+    return boosted

src/utils/responses.py

@@ -856,6 +856,7 @@ def parse_referenced_documents(  # pylint: disable=too-many-locals
                     or attributes.get("link")
                 )
                 doc_title = attributes.get("title")
+                doc_id = attributes.get("document_id") or attributes.get("doc_id")
 
                 if doc_title or doc_url:
                     # Treat empty string as None for URL to satisfy Optional[AnyUrl]
@@ -866,6 +867,7 @@ def parse_referenced_documents(  # pylint: disable=too-many-locals
                                 doc_url=final_url,
                                 doc_title=doc_title,
                                 source=resolved_source,
+                                document_id=doc_id,
                             )
                         )
                         seen_docs.add((final_url, doc_title))

src/utils/vector_search.py

@@ -27,7 +27,6 @@
 logger = get_logger(__name__)
 
 
-
 def _filter_documents_for_chunks(
     all_documents: list[ReferencedDocument],
     final_chunks: list[RAGChunk],
@@ -50,6 +49,8 @@ def _filter_documents_for_chunks(
             attrs.get("reference_url") or attrs.get("doc_url") or attrs.get("docs_url")
         )
         doc_id = attrs.get("document_id") or attrs.get("doc_id")
+        # Use same precedence as _process_byok_rag_chunks_for_documents:
+        # reference_url first, then doc_id
         dedup_key = doc_url or doc_id or chunk.source or ""
         if dedup_key:
             final_chunk_identifiers.add(dedup_key)
@@ -58,9 +59,11 @@ def _filter_documents_for_chunks(
     filtered_documents = []
     seen = set()
     for doc in all_documents:
-        # Build same dedup key for document
+        # Build same dedup key for document using same logic as extraction
         doc_url_str = str(doc.doc_url) if doc.doc_url else None
-        dedup_key = doc_url_str or doc.source or ""
+        # Use the same dedup key logic as _process_byok_rag_chunks_for_documents
+        # which uses reference_url or doc_id as the key
+        dedup_key = doc_url_str or doc.document_id or doc.source or ""
 
         if dedup_key in final_chunk_identifiers and dedup_key not in seen:
             seen.add(dedup_key)
@@ -313,8 +316,16 @@ def _process_byok_rag_chunks_for_documents(
             or metadata.get("docs_url")
         )
 
+        # If no standard document identifiers are available, create a fallback
+        # using the source (vector store ID) to ensure referenced documents
+        # are still created for e2e tests where metadata may be minimal
         if not doc_id and not reference_url:
-            continue
+            # Use source as fallback document identifier
+            fallback_doc_id = result.get("source", "unknown")
+            if fallback_doc_id and fallback_doc_id != "unknown":
+                doc_id = fallback_doc_id
+            else:
+                continue
 
         # Use doc_id or reference_url as deduplication key
         dedup_key = reference_url or doc_id
@@ -334,6 +345,7 @@ def _process_byok_rag_chunks_for_documents(
                     doc_title=title,
                     doc_url=parsed_url,
                     source=result.get("source"),  # Vector store ID
+                    document_id=doc_id,
                 )
             )
 
@@ -387,6 +399,7 @@ def _process_solr_chunks_for_documents(
                     doc_title=title,
                     doc_url=parsed_url,
                     source=constants.OKP_RAG_ID,
+                    document_id=doc_id,
                 )
             )
 

tests/unit/app/endpoints/test_query.py

@@ -212,7 +212,7 @@ async def test_query_merges_inline_and_tool_rag_chunks_and_documents(
         )
 
         inline_chunk = RAGChunk(content="inline chunk content", source="byok")
-        inline_doc = ReferencedDocument(doc_title="Inline Doc")
+        inline_doc = ReferencedDocument(doc_title="Inline Doc", document_id="inline_doc_1")
         inline_rag = RAGContext(
             context_text="",
             rag_chunks=[inline_chunk],
@@ -237,7 +237,7 @@ async def test_query_merges_inline_and_tool_rag_chunks_and_documents(
         )
 
         tool_chunk = RAGChunk(content="tool chunk content", source="vs-1")
-        tool_doc = ReferencedDocument(doc_title="Tool Doc")
+        tool_doc = ReferencedDocument(doc_title="Tool Doc", document_id="tool_doc_1")
         mock_turn_summary = TurnSummary()
         mock_turn_summary.rag_chunks = [tool_chunk]
         mock_turn_summary.referenced_documents = [tool_doc]

tests/unit/cache/test_postgres_cache.py

@@ -637,7 +637,7 @@ def test_insert_and_get_with_referenced_documents(
 
     # Create a CacheEntry with referenced documents
     docs = [
-        ReferencedDocument(doc_title="Test Doc", doc_url=AnyUrl("http://example.com/"))
+        ReferencedDocument(doc_title="Test Doc", doc_url=AnyUrl("http://example.com/"), document_id="test_doc_postgres_1")
     ]
     entry_with_docs = CacheEntry(
         query="user message",
@@ -664,7 +664,7 @@ def test_insert_and_get_with_referenced_documents(
     inserted_json_str = sql_params[-3]
 
     assert json.loads(inserted_json_str) == [
-        {"doc_url": "http://example.com/", "doc_title": "Test Doc", "source": None}
+        {"doc_url": "http://example.com/", "doc_title": "Test Doc", "source": None, "document_id": "test_doc_postgres_1"}
     ]
 
     # Simulate the database returning that data
@@ -675,7 +675,7 @@ def test_insert_and_get_with_referenced_documents(
         "bar",
         "start_time",
         "end_time",
-        [{"doc_url": "http://example.com/", "doc_title": "Test Doc"}],
+        [{"doc_url": "http://example.com/", "doc_title": "Test Doc", "document_id": "test_doc_postgres_1"}],
         None,  # tool_calls
         None,  # tool_results
     )

tests/unit/cache/test_sqlite_cache.py

@@ -467,7 +467,7 @@ def test_insert_and_get_with_referenced_documents(tmpdir: Path) -> None:
 
     # Create a CacheEntry with referenced documents
     docs = [
-        ReferencedDocument(doc_title="Test Doc", doc_url=AnyUrl("http://example.com"))
+        ReferencedDocument(doc_title="Test Doc", doc_url=AnyUrl("http://example.com"), document_id="test_doc_cache_1")
     ]
     entry_with_docs = CacheEntry(
         query="user message",
@@ -571,7 +571,7 @@ def test_insert_and_get_with_all_fields(tmpdir: Path) -> None:
 
     # Create all fields
     docs = [
-        ReferencedDocument(doc_title="Test Doc", doc_url=AnyUrl("http://example.com"))
+        ReferencedDocument(doc_title="Test Doc", doc_url=AnyUrl("http://example.com"), document_id="test_doc_cache_1")
     ]
     tool_calls = [
         ToolCallSummary(

tests/unit/models/responses/test_successful_responses.py

@@ -321,7 +321,7 @@ def test_constructor_full(self) -> None:
             )
         ]
         referenced_docs = [
-            ReferencedDocument(doc_url=AnyUrl("https://example.com"), doc_title="Doc")
+            ReferencedDocument(doc_url=AnyUrl("https://example.com"), doc_title="Doc", document_id="test_doc_1")
         ]
 
         response = QueryResponse(  # type: ignore[call-arg]

tests/unit/utils/test_responses.py

@@ -2126,10 +2126,11 @@ def test_parse_referenced_documents_file_search_call(
         mock_result1.attributes = {
             "link": "https://example.com/doc1",
             "title": "Document 1",
+            "document_id": "doc_1",
         }
 
         mock_result2 = {
-            "attributes": {"url": "https://example.com/doc2", "title": "Document 2"},
+            "attributes": {"url": "https://example.com/doc2", "title": "Document 2", "doc_id": "doc_2"},
         }
 
         mock_output_item = mocker.Mock()
@@ -2143,8 +2144,10 @@ def test_parse_referenced_documents_file_search_call(
         assert len(result) == 2
         assert result[0].doc_title == "Document 1"
         assert result[0].doc_url == AnyUrl("https://example.com/doc1")
+        assert result[0].document_id == "doc_1"
         assert result[1].doc_title == "Document 2"
         assert result[1].doc_url == AnyUrl("https://example.com/doc2")
+        assert result[1].document_id == "doc_2"
 
     def test_parse_referenced_documents_message_annotations(
         self, mocker: MockerFixture
@@ -2206,6 +2209,7 @@ def test_parse_referenced_documents_deduplication(
         mock_result.attributes = {
             "link": "https://example.com/doc1",
             "title": "Document 1",
+            "document_id": "doc_1",
         }
 
         mock_output_item = mocker.Mock()
@@ -2992,6 +2996,7 @@ def test_single_store_source_populated(self, mocker: MockerFixture) -> None:
         mock_result.attributes = {
             "url": "https://docs.example.com/page",
             "title": "Example Page",
+            "document_id": "doc_page_1",
         }
 
         mock_output = mocker.Mock()
@@ -3014,7 +3019,7 @@ def test_single_store_source_populated(self, mocker: MockerFixture) -> None:
     def test_no_mapping_source_is_none(self, mocker: MockerFixture) -> None:
         """Test that source is None when no mapping provided."""
         mock_result = mocker.Mock()
-        mock_result.attributes = {"title": "Doc"}
+        mock_result.attributes = {"title": "Doc", "document_id": "doc_no_mapping"}
 
         mock_output = mocker.Mock()
         mock_output.type = "file_search_call"
@@ -3031,7 +3036,7 @@ def test_no_mapping_source_is_none(self, mocker: MockerFixture) -> None:
     def test_multiple_stores_source_is_none(self, mocker: MockerFixture) -> None:
         """Test that source is None with multiple stores (ambiguous)."""
         mock_result = mocker.Mock()
-        mock_result.attributes = {"title": "Doc"}
+        mock_result.attributes = {"title": "Doc", "document_id": "doc_multi_stores"}
 
         mock_output = mocker.Mock()
         mock_output.type = "file_search_call"