{ "schema_version": 1, "kind": "comparison", "generated_at": "2026-10-19T15:26:52.160690+00:00", "state": "complete", "input_name": "docs", "evaluation_level": "reviewed_comparison", "policy": "fixed_k", "baseline": "bm25", "phase": "dev", "resumed": false, "heldout_use": {}, "query_call_upper_bound": 12, "participants": [ { "name": "bm25", "state": "complete", "error": null, "report": { "schema_version": 1, "kind": "inspection", "generated_at": "2026-10-19T15:26:51.998364+00:00", "evaluation_level": "reviewed_evaluation", "preparation": { "implementation": "portable-files-v1", "length_unit": "characters", "chunk_size": 100, "chunk_overlap": 10, "text_field": null, "include_patterns": [], "exclude_patterns": [], "limits": { "max_file_bytes": 20971520, "max_total_bytes": 104857600, "max_files": 1000, "max_records": 10000, "max_extracted_characters": 20971520 } }, "manifest": { "run_id": "ea7295c1a91f", "schema_version": 2, "hash_algorithm": "canonical-json-v1", "evaluation_level": "reviewed_evaluation", "created_at": "2026-10-19T15:26:51+00:00", "product_version": "0.2.1", "python_version": "3.12.14", "platform": "Linux-6.18.33.2-microsoft-standard-WSL2-x86_64-with-glibc2.43", "dataset_hash": "6607e06b253048c606257174df51e82d8070a0a8f30cb605c181c8201303e4d4", "corpus_hash": "b4bdb92293aa66d4636c27dd25ca77a791dbbeea111d33eb4873d314acfa0386", "questions_hash": "c058b113c76f2d061279f04523dfe7348131314011b7de2045eb3c9726603a7e", "references_hash": "cd3ccaa312b66201e19ad814a9d61015802a174050757bd54fa1ed7a9584e338", "representation_hash": "a04060688ab87c64a6a3964da1e83b86e20db941c809b6ed19f6b91df32bf6a8", "configuration_hash": "3b821feab0321341e4d73997b51f705614716d960cb8159696584daaafa9c846", "input_preparation": { "implementation": "portable-files-v1", "length_unit": "characters", "chunk_size": 100, "chunk_overlap": 10, "text_field": null, "include_patterns": [], "exclude_patterns": [], "limits": { "max_file_bytes": 20971520, "max_total_bytes": 104857600, "max_files": 1000, "max_records": 10000, "max_extracted_characters": 20971520 } }, "config": { "name": "bm25", "strategy": "bm25", "representation": "body", "k": 1, "model": null, "provider": null, "candidate_k": 1, "lexical_weight": 0.5, "rrf_constant": 60, "child_splitter": "none", "child_size": 500, "child_overlap": 50, "query_variants": {}, "reranker_model": null }, "index_time_ms": 77.14363600098295, "warnings": [] }, "evaluation": { "schema_version": 1, "method": "original-span-union-v1", "references_hash": "cd3ccaa312b66201e19ad814a9d61015802a174050757bd54fa1ed7a9584e338", "questions": [ { "question_id": "Q1", "split": "dev", "answerable": true, "tags": [], "required_evidence_ids": [ "POWER", "RESET" ], "found_evidence_ids": [ "RESET" ], "missing_evidence_ids": [ "POWER" ], "sufficient_evidence": false, "evidence_coverage": 0.5, "first_complete_evidence_rank": 1, "reciprocal_rank": 1.0, "precision_at_k": 1.0 }, { "question_id": "Q2", "split": "dev", "answerable": true, "tags": [], "required_evidence_ids": [ "EXCLUSION", "WARRANTY" ], "found_evidence_ids": [ "EXCLUSION", "WARRANTY" ], "missing_evidence_ids": [], "sufficient_evidence": true, "evidence_coverage": 1.0, "first_complete_evidence_rank": 1, "reciprocal_rank": 1.0, "precision_at_k": 1.0 }, { "question_id": "Q3", "split": "dev", "answerable": true, "tags": [], "required_evidence_ids": [ "CHARGE" ], "found_evidence_ids": [ "CHARGE" ], "missing_evidence_ids": [], "sufficient_evidence": true, "evidence_coverage": 1.0, "first_complete_evidence_rank": 1, "reciprocal_rank": 1.0, "precision_at_k": 1.0 }, { "question_id": "Q4", "split": "dev", "answerable": true, "tags": [], "required_evidence_ids": [ "DELIVERY" ], "found_evidence_ids": [ "DELIVERY" ], "missing_evidence_ids": [], "sufficient_evidence": true, "evidence_coverage": 1.0, "first_complete_evidence_rank": 1, "reciprocal_rank": 1.0, "precision_at_k": 1.0 } ], "summaries": { "dev": { "question_count": 4, "answerable_questions": 4, "sufficient_count": 3, "sufficiency_rate": 0.75, "mean_evidence_coverage": 0.875, "mean_reciprocal_rank": 1.0, "mean_precision_at_k": 1.0 }, "heldout": { "question_count": 0, "answerable_questions": 0, "sufficient_count": 0, "sufficiency_rate": null, "mean_evidence_coverage": null, "mean_reciprocal_rank": null, "mean_precision_at_k": null } }, "definitions": { "reciprocal_rank": "First rank prefix completing at least one required evidence item", "precision_at_k": "Returned hits overlapping required spans divided by configured k", "sufficiency": "All spans of every required item covered by the returned union", "unanswerable": "Excluded from retrieval quality rates; abstention is not evaluated" }, "scope": "exact_returned_spans" }, "input": { "name": "docs", "corpus_hash": "b4bdb92293aa66d4636c27dd25ca77a791dbbeea111d33eb4873d314acfa0386", "document_count": 1, "file_count": 1, "passage_count": 10, "source_characters": 713, "indexed_characters": 763, "skipped": [], "skipped_count": 0, "failed_count": 0, "warning_count": 0, "sources": [ { "source_unit_id": "doc-aaa8b734a101d7f34afb", "source_path": "manual.md", "location": {}, "location_label": "Source text", "heading_path": [], "loader": "utf8-text-v1", "file_sha256": "60d754e2f2aa039fe08127ae15f0775df6ea8309a1531920cbd9d94e46d405b9", "text_hash": "e24765cdef868e7e3b83177ad778e6eb80c5913fe7501e39f37302bc8092ca7b", "characters": 713 } ] }, "questions": [ { "question_id": "Q1", "question": "How long should I hold the reset button, and what power is required?", "hits": [ { "source_unit_id": "passage-5bdc4916bcb2462d0250", "rank": 1, "score": 2.2885582447052, "evidence_ids": [], "preview": "## Reset pairing\n\nHold the reset button for eight seconds to clear pairing. The status light ", "metadata": { "document_id": "doc-aaa8b734a101d7f34afb" }, "text": "## Reset pairing\n\nHold the reset button for eight seconds to clear pairing. The status light ", "heading_path": [ "Orion device manual", "Reset pairing" ], "source_path": "manual.md", "location": {}, "location_label": "Source text", "loader": "utf8-text-v1", "start": 23, "end": 116, "origin_source_id": "doc-aaa8b734a101d7f34afb", "line_start": 3, "line_end": 5 } ], "query_time_ms": 2.7301770023768768, "context_characters": 93, "context_hash": "12b666efab1d77f87c5ab8ee8cd7ad5367a91f6555a3eaf4282073f7546f9eb7", "observations": [], "metrics": { "question_id": "Q1", "split": "dev", "answerable": true, "tags": [], "required_evidence_ids": [ "POWER", "RESET" ], "found_evidence_ids": [ "RESET" ], "missing_evidence_ids": [ "POWER" ], "sufficient_evidence": false, "evidence_coverage": 0.5, "first_complete_evidence_rank": 1, "reciprocal_rank": 1.0, "precision_at_k": 1.0 }, "context_measurement": { "adapter": "unicode-codepoints-v1", "unit": "characters", "exact": true, "count": 160, "budget": null, "scope": "rendered_retrieval_context", "skipped": [], "policy": "ranked-whole-passages-v1" }, "stages": [ { "name": "retrieval", "elapsed_ms": 1.016311001876602, "hits": [ { "source_unit_id": "passage-5bdc4916bcb2462d0250", "rank": 1, "score": 2.2885582447052, "evidence_ids": [], "preview": "## Reset pairing\n\nHold the reset button for eight seconds to clear pairing. The status light ", "metadata": { "document_id": "doc-aaa8b734a101d7f34afb" } } ], "details": { "query": "How long should I hold the reset button, and what power is required?", "limit": 1, "index_cache": "not_applicable", "model_cache": "unknown", "runtime_versions": { "bm25s": "0.2.14", "numpy": "2.5.4" } } }, { "name": "context_selection", "elapsed_ms": 0.050705999456113204, "hits": [ { "source_unit_id": "passage-5bdc4916bcb2462d0250", "rank": 1, "score": 2.2885582447052, "evidence_ids": [], "preview": "## Reset pairing\n\nHold the reset button for eight seconds to clear pairing. The status light ", "metadata": { "document_id": "doc-aaa8b734a101d7f34afb" } } ], "details": { "adapter": "unicode-codepoints-v1", "unit": "characters", "exact": true, "count": 160, "budget": null, "scope": "rendered_retrieval_context", "skipped": [], "policy": "ranked-whole-passages-v1" } } ] }, { "question_id": "Q2", "question": "How long is the warranty and what damage is excluded?", "hits": [ { "source_unit_id": "passage-4be5e239b1ab47982a8e", "rank": 1, "score": 3.4753642082214355, "evidence_ids": [], "preview": "## Warranty\n\nThe warranty lasts two years. Keep your receipt. Accidental water damage is excluded. ", "metadata": { "document_id": "doc-aaa8b734a101d7f34afb" }, "text": "## Warranty\n\nThe warranty lasts two years. Keep your receipt. Accidental water damage is excluded. ", "heading_path": [ "Orion device manual", "Warranty" ], "source_path": "manual.md", "location": {}, "location_label": "Source text", "loader": "utf8-text-v1", "start": 227, "end": 326, "origin_source_id": "doc-aaa8b734a101d7f34afb", "line_start": 7, "line_end": 9 } ], "query_time_ms": 0.2231840007880237, "context_characters": 99, "context_hash": "3a614c027a26a3e6ad6e99d73dd050e7c11d51167df98f22c88788acb46fdf2b", "observations": [], "metrics": { "question_id": "Q2", "split": "dev", "answerable": true, "tags": [], "required_evidence_ids": [ "EXCLUSION", "WARRANTY" ], "found_evidence_ids": [ "EXCLUSION", "WARRANTY" ], "missing_evidence_ids": [], "sufficient_evidence": true, "evidence_coverage": 1.0, "first_complete_evidence_rank": 1, "reciprocal_rank": 1.0, "precision_at_k": 1.0 }, "context_measurement": { "adapter": "unicode-codepoints-v1", "unit": "characters", "exact": true, "count": 161, "budget": null, "scope": "rendered_retrieval_context", "skipped": [], "policy": "ranked-whole-passages-v1" }, "stages": [ { "name": "retrieval", "elapsed_ms": 0.09969899838324636, "hits": [ { "source_unit_id": "passage-4be5e239b1ab47982a8e", "rank": 1, "score": 3.4753642082214355, "evidence_ids": [], "preview": "## Warranty\n\nThe warranty lasts two years. Keep your receipt. Accidental water damage is excluded. ", "metadata": { "document_id": "doc-aaa8b734a101d7f34afb" } } ], "details": { "query": "How long is the warranty and what damage is excluded?", "limit": 1, "index_cache": "not_applicable", "model_cache": "unknown", "runtime_versions": { "bm25s": "0.2.14", "numpy": "2.5.4" } } }, { "name": "context_selection", "elapsed_ms": 0.028574999305419624, "hits": [ { "source_unit_id": "passage-4be5e239b1ab47982a8e", "rank": 1, "score": 3.4753642082214355, "evidence_ids": [], "preview": "## Warranty\n\nThe warranty lasts two years. Keep your receipt. Accidental water damage is excluded. ", "metadata": { "document_id": "doc-aaa8b734a101d7f34afb" } } ], "details": { "adapter": "unicode-codepoints-v1", "unit": "characters", "exact": true, "count": 161, "budget": null, "scope": "rendered_retrieval_context", "skipped": [], "policy": "ranked-whole-passages-v1" } } ] }, { "question_id": "Q3", "question": "How often should I charge an unused device?", "hits": [ { "source_unit_id": "passage-32a3368b3b75076e202c", "rank": 1, "score": 1.5058517456054688, "evidence_ids": [], "preview": "## Battery care\n\nStore the device in a cool, dry place. Charge it every three months when unused. ", "metadata": { "document_id": "doc-aaa8b734a101d7f34afb" }, "text": "## Battery care\n\nStore the device in a cool, dry place. Charge it every three months when unused. ", "heading_path": [ "Orion device manual", "Battery care" ], "source_path": "manual.md", "location": {}, "location_label": "Source text", "loader": "utf8-text-v1", "start": 379, "end": 477, "origin_source_id": "doc-aaa8b734a101d7f34afb", "line_start": 11, "line_end": 13 } ], "query_time_ms": 0.1333030013483949, "context_characters": 98, "context_hash": "7c804446513508f27f1c0f5042fe24b9915c0973f768acb27e1807629f8bd769", "observations": [], "metrics": { "question_id": "Q3", "split": "dev", "answerable": true, "tags": [], "required_evidence_ids": [ "CHARGE" ], "found_evidence_ids": [ "CHARGE" ], "missing_evidence_ids": [], "sufficient_evidence": true, "evidence_coverage": 1.0, "first_complete_evidence_rank": 1, "reciprocal_rank": 1.0, "precision_at_k": 1.0 }, "context_measurement": { "adapter": "unicode-codepoints-v1", "unit": "characters", "exact": true, "count": 164, "budget": null, "scope": "rendered_retrieval_context", "skipped": [], "policy": "ranked-whole-passages-v1" }, "stages": [ { "name": "retrieval", "elapsed_ms": 0.06612499782931991, "hits": [ { "source_unit_id": "passage-32a3368b3b75076e202c", "rank": 1, "score": 1.5058517456054688, "evidence_ids": [], "preview": "## Battery care\n\nStore the device in a cool, dry place. Charge it every three months when unused. ", "metadata": { "document_id": "doc-aaa8b734a101d7f34afb" } } ], "details": { "query": "How often should I charge an unused device?", "limit": 1, "index_cache": "not_applicable", "model_cache": "unknown", "runtime_versions": { "bm25s": "0.2.14", "numpy": "2.5.4" } } }, { "name": "context_selection", "elapsed_ms": 0.026300000172341242, "hits": [ { "source_unit_id": "passage-32a3368b3b75076e202c", "rank": 1, "score": 1.5058517456054688, "evidence_ids": [], "preview": "## Battery care\n\nStore the device in a cool, dry place. Charge it every three months when unused. ", "metadata": { "document_id": "doc-aaa8b734a101d7f34afb" } } ], "details": { "adapter": "unicode-codepoints-v1", "unit": "characters", "exact": true, "count": 164, "budget": null, "scope": "rendered_retrieval_context", "skipped": [], "policy": "ranked-whole-passages-v1" } } ] }, { "question_id": "Q4", "question": "Where is tracked delivery available?", "hits": [ { "source_unit_id": "passage-1518103967d65de40d95", "rank": 1, "score": 3.3280820846557617, "evidence_ids": [], "preview": "## Delivery\n\nStandard delivery takes five working days. Tracked delivery is available in Morocco. ", "metadata": { "document_id": "doc-aaa8b734a101d7f34afb" }, "text": "## Delivery\n\nStandard delivery takes five working days. Tracked delivery is available in Morocco. ", "heading_path": [ "Orion device manual", "Delivery" ], "source_path": "manual.md", "location": {}, "location_label": "Source text", "loader": "utf8-text-v1", "start": 564, "end": 662, "origin_source_id": "doc-aaa8b734a101d7f34afb", "line_start": 15, "line_end": 17 } ], "query_time_ms": 0.11743299910449423, "context_characters": 98, "context_hash": "222f297b0605ceef1a9d28663d84c2da3a0a9445cfe2bf19edca6758653f61cd", "observations": [], "metrics": { "question_id": "Q4", "split": "dev", "answerable": true, "tags": [], "required_evidence_ids": [ "DELIVERY" ], "found_evidence_ids": [ "DELIVERY" ], "missing_evidence_ids": [], "sufficient_evidence": true, "evidence_coverage": 1.0, "first_complete_evidence_rank": 1, "reciprocal_rank": 1.0, "precision_at_k": 1.0 }, "context_measurement": { "adapter": "unicode-codepoints-v1", "unit": "characters", "exact": true, "count": 160, "budget": null, "scope": "rendered_retrieval_context", "skipped": [], "policy": "ranked-whole-passages-v1" }, "stages": [ { "name": "retrieval", "elapsed_ms": 0.05724900256609544, "hits": [ { "source_unit_id": "passage-1518103967d65de40d95", "rank": 1, "score": 3.3280820846557617, "evidence_ids": [], "preview": "## Delivery\n\nStandard delivery takes five working days. Tracked delivery is available in Morocco. ", "metadata": { "document_id": "doc-aaa8b734a101d7f34afb" } } ], "details": { "query": "Where is tracked delivery available?", "limit": 1, "index_cache": "not_applicable", "model_cache": "unknown", "runtime_versions": { "bm25s": "0.2.14", "numpy": "2.5.4" } } }, { "name": "context_selection", "elapsed_ms": 0.02544800008763559, "hits": [ { "source_unit_id": "passage-1518103967d65de40d95", "rank": 1, "score": 3.3280820846557617, "evidence_ids": [], "preview": "## Delivery\n\nStandard delivery takes five working days. Tracked delivery is available in Morocco. ", "metadata": { "document_id": "doc-aaa8b734a101d7f34afb" } } ], "details": { "adapter": "unicode-codepoints-v1", "unit": "characters", "exact": true, "count": 160, "budget": null, "scope": "rendered_retrieval_context", "skipped": [], "policy": "ranked-whole-passages-v1" } } ] } ], "observations": [ "Reviewed evidence is scored against the union of exact original-text spans." ], "limitations": [ "Answer correctness, citations, abstention and generation request tokens are not evaluated.", "Retriever scores describe this ranking; they are not probabilities of correctness.", "Chunk sizes count characters. Rendered context counts characters unless an explicit local tokenizer is identified; model request overhead is excluded.", "Character and text-line offsets refer to the extracted page/record/section, not byte offsets in the original file. Extraction does not verify layout fidelity." ] }, "resources": { "query_ms": { "mean_known": 0.8010242509044474, "known_count": 4, "missing_count": 0 }, "runtime_versions": { "bm25s": "0.2.14", "numpy": "2.5.4" }, "context_units": { "mean_known": 161.25, "known_count": 4, "missing_count": 0 }, "context_characters": { "mean_known": 97.0, "known_count": 4, "missing_count": 0 }, "context_unit": "characters", "index_ms": 77.14363600098295, "vector_cache": "not_applicable", "model_cache": "unknown" }, "usage": { "source": "user_supplied_usage", "scope": "supplied_totals_not_inferred_from_repeats", "estimate": true, "cost": null, "currency": null, "pricing_identity": null, "input_tokens": null, "output_tokens": null }, "trials": [ { "trial": 0, "reused": false, "artifact_directory": "jobs/bm25/trial-0/attempt-20261019T152651-07023b19cdc5", "resources": { "query_ms": { "mean_known": 0.8010242509044474, "known_count": 4, "missing_count": 0 }, "runtime_versions": { "bm25s": "0.2.14", "numpy": "2.5.4" }, "context_units": { "mean_known": 161.25, "known_count": 4, "missing_count": 0 }, "context_characters": { "mean_known": 97.0, "known_count": 4, "missing_count": 0 }, "context_unit": "characters", "index_ms": 77.14363600098295, "vector_cache": "not_applicable", "model_cache": "unknown" } } ], "repeat_rankings_changed": false, "evidence_scope": "exact_returned_spans", "summaries": { "dev": { "all": { "question_count": 4, "answerable_questions": 4, "sufficient_count": 3, "sufficiency_rate": 0.75, "mean_evidence_coverage": 0.875, "mean_reciprocal_rank": 1.0, "mean_precision_at_k": 1.0 } }, "heldout": { "all": { "question_count": 0, "answerable_questions": 0, "sufficient_count": 0, "sufficiency_rate": null, "mean_evidence_coverage": null, "mean_reciprocal_rank": null, "mean_precision_at_k": null } } }, "split_resources": { "dev": { "query_ms": { "mean_known": 0.8010242509044474, "known_count": 4, "missing_count": 0 }, "runtime_versions": { "bm25s": "0.2.14", "numpy": "2.5.4" }, "context_units": { "mean_known": 161.25, "known_count": 4, "missing_count": 0 }, "context_characters": { "mean_known": 97.0, "known_count": 4, "missing_count": 0 }, "context_unit": "characters", "index_ms": 77.14363600098295, "vector_cache": "not_applicable", "model_cache": "unknown" }, "heldout": { "query_ms": { "mean_known": null, "known_count": 0, "missing_count": 0 }, "runtime_versions": {}, "context_units": { "mean_known": null, "known_count": 0, "missing_count": 0 }, "context_characters": { "mean_known": null, "known_count": 0, "missing_count": 0 }, "context_unit": null, "index_ms": 77.14363600098295, "vector_cache": "unavailable", "model_cache": "unknown" } }, "criteria": { "dev": { "status": "not_requested", "checks": [] }, "heldout": { "status": "not_requested", "checks": [] } } }, { "name": "parent-child", "state": "complete", "error": null, "report": { "schema_version": 1, "kind": "inspection", "generated_at": "2026-10-19T15:26:52.025167+00:00", "evaluation_level": "reviewed_evaluation", "preparation": { "implementation": "portable-files-v1", "length_unit": "characters", "chunk_size": 100, "chunk_overlap": 10, "text_field": null, "include_patterns": [], "exclude_patterns": [], "limits": { "max_file_bytes": 20971520, "max_total_bytes": 104857600, "max_files": 1000, "max_records": 10000, "max_extracted_characters": 20971520 } }, "manifest": { "run_id": "4259a0b0af80", "schema_version": 2, "hash_algorithm": "canonical-json-v1", "evaluation_level": "reviewed_evaluation", "created_at": "2026-10-19T15:26:52+00:00", "product_version": "0.2.1", "python_version": "3.12.14", "platform": "Linux-6.18.33.2-microsoft-standard-WSL2-x86_64-with-glibc2.43", "dataset_hash": "6607e06b253048c606257174df51e82d8070a0a8f30cb605c181c8201303e4d4", "corpus_hash": "b4bdb92293aa66d4636c27dd25ca77a791dbbeea111d33eb4873d314acfa0386", "questions_hash": "c058b113c76f2d061279f04523dfe7348131314011b7de2045eb3c9726603a7e", "references_hash": "cd3ccaa312b66201e19ad814a9d61015802a174050757bd54fa1ed7a9584e338", "representation_hash": "d1fdc1583894a3b45f31923224e56e6e80d8ec593082741ca80a38101ac72b3a", "configuration_hash": "a4f37f3dab3e00746ef7506da67f91f1df63a22897b7774976d78ac0700ab147", "input_preparation": { "implementation": "portable-files-v1", "length_unit": "characters", "chunk_size": 100, "chunk_overlap": 10, "text_field": null, "include_patterns": [], "exclude_patterns": [], "limits": { "max_file_bytes": 20971520, "max_total_bytes": 104857600, "max_files": 1000, "max_records": 10000, "max_extracted_characters": 20971520 } }, "config": { "name": "parent-child", "strategy": "bm25", "representation": "body", "k": 1, "model": null, "provider": null, "candidate_k": 20, "lexical_weight": 0.5, "rrf_constant": 60, "child_splitter": "recursive", "child_size": 100, "child_overlap": 10, "query_variants": {}, "reranker_model": null }, "index_time_ms": 0.9598130018275697, "warnings": [ "Child retrieval returns full parent source units. Evidence metrics describe parent context; they do not establish that the matching child contains every fact. Child candidate limits can return fewer than k unique parents." ] }, "evaluation": { "schema_version": 1, "method": "original-span-union-v1", "references_hash": "cd3ccaa312b66201e19ad814a9d61015802a174050757bd54fa1ed7a9584e338", "questions": [ { "question_id": "Q1", "split": "dev", "answerable": true, "tags": [], "required_evidence_ids": [ "POWER", "RESET" ], "found_evidence_ids": [ "POWER", "RESET" ], "missing_evidence_ids": [], "sufficient_evidence": true, "evidence_coverage": 1.0, "first_complete_evidence_rank": 1, "reciprocal_rank": 1.0, "precision_at_k": 1.0 }, { "question_id": "Q2", "split": "dev", "answerable": true, "tags": [], "required_evidence_ids": [ "EXCLUSION", "WARRANTY" ], "found_evidence_ids": [ "EXCLUSION", "WARRANTY" ], "missing_evidence_ids": [], "sufficient_evidence": true, "evidence_coverage": 1.0, "first_complete_evidence_rank": 1, "reciprocal_rank": 1.0, "precision_at_k": 1.0 }, { "question_id": "Q3", "split": "dev", "answerable": true, "tags": [], "required_evidence_ids": [ "CHARGE" ], "found_evidence_ids": [ "CHARGE" ], "missing_evidence_ids": [], "sufficient_evidence": true, "evidence_coverage": 1.0, "first_complete_evidence_rank": 1, "reciprocal_rank": 1.0, "precision_at_k": 1.0 }, { "question_id": "Q4", "split": "dev", "answerable": true, "tags": [], "required_evidence_ids": [ "DELIVERY" ], "found_evidence_ids": [ "DELIVERY" ], "missing_evidence_ids": [], "sufficient_evidence": true, "evidence_coverage": 1.0, "first_complete_evidence_rank": 1, "reciprocal_rank": 1.0, "precision_at_k": 1.0 } ], "summaries": { "dev": { "question_count": 4, "answerable_questions": 4, "sufficient_count": 4, "sufficiency_rate": 1.0, "mean_evidence_coverage": 1.0, "mean_reciprocal_rank": 1.0, "mean_precision_at_k": 1.0 }, "heldout": { "question_count": 0, "answerable_questions": 0, "sufficient_count": 0, "sufficiency_rate": null, "mean_evidence_coverage": null, "mean_reciprocal_rank": null, "mean_precision_at_k": null } }, "definitions": { "reciprocal_rank": "First rank prefix completing at least one required evidence item", "precision_at_k": "Returned hits overlapping required spans divided by configured k", "sufficiency": "All spans of every required item covered by the returned union", "unanswerable": "Excluded from retrieval quality rates; abstention is not evaluated" }, "scope": "exact_returned_spans" }, "input": { "name": "docs", "corpus_hash": "b4bdb92293aa66d4636c27dd25ca77a791dbbeea111d33eb4873d314acfa0386", "document_count": 1, "file_count": 1, "passage_count": 10, "source_characters": 713, "indexed_characters": 763, "skipped": [], "skipped_count": 0, "failed_count": 0, "warning_count": 0, "sources": [ { "source_unit_id": "doc-aaa8b734a101d7f34afb", "source_path": "manual.md", "location": {}, "location_label": "Source text", "heading_path": [], "loader": "utf8-text-v1", "file_sha256": "60d754e2f2aa039fe08127ae15f0775df6ea8309a1531920cbd9d94e46d405b9", "text_hash": "e24765cdef868e7e3b83177ad778e6eb80c5913fe7501e39f37302bc8092ca7b", "characters": 713 } ] }, "questions": [ { "question_id": "Q1", "question": "How long should I hold the reset button, and what power is required?", "hits": [ { "source_unit_id": "doc-aaa8b734a101d7f34afb", "rank": 1, "score": 2.315727949142456, "evidence_ids": [], "preview": "# Orion device manual\n\n## Reset pairing\n\nHold the reset button for eight seconds to clear pairing. The status light flashes twice. Disconnect the companion app before starting. Keep USB-C power connected throughout the reset.\n\n## Warranty\n\n", "metadata": { "document_id": "doc-aaa8b734a101d7f34afb", "matched_child_id": "child-0", "matched_child_start": 0, "matched_child_end": 99, "returned_context": "full_parent" }, "text": "# Orion device manual\n\n## Reset pairing\n\nHold the reset button for eight seconds to clear pairing. The status light flashes twice. Disconnect the companion app before starting. Keep USB-C power connected throughout the reset.\n\n## Warranty\n\nThe warranty lasts two years. Keep your receipt. Accidental water damage is excluded. Contact the support team before returning a device.\n\n## Battery care\n\nStore the device in a cool, dry place. Charge it every three months when unused. Use only the supplied USB-C cable. Recycle batteries at an approved collection point.\n\n## Delivery\n\nStandard delivery takes five working days. Tracked delivery is available in Morocco. Keep the tracking number until the parcel arrives.\n", "heading_path": [], "source_path": "manual.md", "location": {}, "location_label": "Source text", "loader": "utf8-text-v1", "start": 0, "end": 713, "origin_source_id": "doc-aaa8b734a101d7f34afb", "line_start": 1, "line_end": 17 } ], "query_time_ms": 0.22048900063964538, "context_characters": 713, "context_hash": "3b5a4e8ae04775940ba9a77c3d0decd0902b8c858a35f8fe631b46c96803b3c6", "observations": [], "metrics": { "question_id": "Q1", "split": "dev", "answerable": true, "tags": [], "required_evidence_ids": [ "POWER", "RESET" ], "found_evidence_ids": [ "POWER", "RESET" ], "missing_evidence_ids": [], "sufficient_evidence": true, "evidence_coverage": 1.0, "first_complete_evidence_rank": 1, "reciprocal_rank": 1.0, "precision_at_k": 1.0 }, "context_measurement": { "adapter": "unicode-codepoints-v1", "unit": "characters", "exact": true, "count": 741, "budget": null, "scope": "rendered_retrieval_context", "skipped": [], "policy": "ranked-whole-passages-v1" }, "stages": [ { "name": "children", "elapsed_ms": 0.11351600187481381, "hits": [ { "source_unit_id": "child-0", "rank": 1, "score": 2.315727949142456, "evidence_ids": [], "preview": "# Orion device manual\n\n## Reset pairing\n\nHold the reset button for eight seconds to clear pairing. ", "metadata": { "document_id": "doc-aaa8b734a101d7f34afb" } }, { "source_unit_id": "child-2", "rank": 2, "score": 1.4516243934631348, "evidence_ids": [], "preview": "eep USB-C power connected throughout the reset.\n\n## Warranty\n\nThe warranty lasts two years. Keep ", "metadata": { "document_id": "doc-aaa8b734a101d7f34afb" } }, { "source_unit_id": "child-7", "rank": 3, "score": 0.6855702996253967, "evidence_ids": [], "preview": "ing days. Tracked delivery is available in Morocco. Keep the tracking number until the parcel ", "metadata": { "document_id": "doc-aaa8b734a101d7f34afb" } }, { "source_unit_id": "child-3", "rank": 4, "score": 0.6573668718338013, "evidence_ids": [], "preview": "ars. Keep your receipt. Accidental water damage is excluded. Contact the support team before ", "metadata": { "document_id": "doc-aaa8b734a101d7f34afb" } }, { "source_unit_id": "child-1", "rank": 5, "score": 0.16223493218421936, "evidence_ids": [], "preview": " pairing. The status light flashes twice. Disconnect the companion app before starting. Keep USB-C ", "metadata": { "document_id": "doc-aaa8b734a101d7f34afb" } }, { "source_unit_id": "child-5", "rank": 6, "score": 0.10600543022155762, "evidence_ids": [], "preview": "Charge it every three months when unused. Use only the supplied USB-C cable. Recycle batteries at ", "metadata": { "document_id": "doc-aaa8b734a101d7f34afb" } }, { "source_unit_id": "child-4", "rank": 7, "score": 0.10283465683460236, "evidence_ids": [], "preview": "am before returning a device.\n\n## Battery care\n\nStore the device in a cool, dry place. Charge it ", "metadata": { "document_id": "doc-aaa8b734a101d7f34afb" } }, { "source_unit_id": "child-6", "rank": 8, "score": 0.0, "evidence_ids": [], "preview": "teries at an approved collection point.\n\n## Delivery\n\nStandard delivery takes five working days. ", "metadata": { "document_id": "doc-aaa8b734a101d7f34afb" } }, { "source_unit_id": "child-8", "rank": 9, "score": 0.0, "evidence_ids": [], "preview": "he parcel arrives.\n", "metadata": { "document_id": "doc-aaa8b734a101d7f34afb" } } ], "details": { "query": "How long should I hold the reset button, and what power is required?", "limit": 20, "locations": [ { "child_id": "child-0", "parent_id": "doc-aaa8b734a101d7f34afb", "start": 0, "end": 99 }, { "child_id": "child-2", "parent_id": "doc-aaa8b734a101d7f34afb", "start": 178, "end": 275 }, { "child_id": "child-7", "parent_id": "doc-aaa8b734a101d7f34afb", "start": 610, "end": 704 }, { "child_id": "child-3", "parent_id": "doc-aaa8b734a101d7f34afb", "start": 265, "end": 358 }, { "child_id": "child-1", "parent_id": "doc-aaa8b734a101d7f34afb", "start": 89, "end": 188 }, { "child_id": "child-5", "parent_id": "doc-aaa8b734a101d7f34afb", "start": 435, "end": 533 }, { "child_id": "child-4", "parent_id": "doc-aaa8b734a101d7f34afb", "start": 348, "end": 445 }, { "child_id": "child-6", "parent_id": "doc-aaa8b734a101d7f34afb", "start": 523, "end": 620 }, { "child_id": "child-8", "parent_id": "doc-aaa8b734a101d7f34afb", "start": 694, "end": 713 } ] } }, { "name": "parent_expansion", "elapsed_ms": 0.008194998372346163, "hits": [ { "source_unit_id": "doc-aaa8b734a101d7f34afb", "rank": 1, "score": 2.315727949142456, "evidence_ids": [], "preview": "# Orion device manual\n\n## Reset pairing\n\nHold the reset button for eight seconds to clear pairing. The status light flashes twice. Disconnect the companion app before starting. Keep USB-C power connected throughout the reset.\n\n## Warranty\n\n", "metadata": { "document_id": "doc-aaa8b734a101d7f34afb", "matched_child_id": "child-0", "matched_child_start": 0, "matched_child_end": 99, "returned_context": "full_parent" } } ], "details": { "query": "How long should I hold the reset button, and what power is required?" } }, { "name": "retrieval", "elapsed_ms": 0.14025600103195757, "hits": [ { "source_unit_id": "doc-aaa8b734a101d7f34afb", "rank": 1, "score": 2.315727949142456, "evidence_ids": [], "preview": "# Orion device manual\n\n## Reset pairing\n\nHold the reset button for eight seconds to clear pairing. The status light flashes twice. Disconnect the companion app before starting. Keep USB-C power connected throughout the reset.\n\n## Warranty\n\n", "metadata": { "document_id": "doc-aaa8b734a101d7f34afb", "matched_child_id": "child-0", "matched_child_start": 0, "matched_child_end": 99, "returned_context": "full_parent" } } ], "details": { "query": "How long should I hold the reset button, and what power is required?", "limit": 1, "index_cache": "not_applicable", "model_cache": "unknown", "runtime_versions": { "bm25s": "0.2.14", "numpy": "2.5.4" } } }, { "name": "context_selection", "elapsed_ms": 0.03715999991982244, "hits": [ { "source_unit_id": "doc-aaa8b734a101d7f34afb", "rank": 1, "score": 2.315727949142456, "evidence_ids": [], "preview": "# Orion device manual\n\n## Reset pairing\n\nHold the reset button for eight seconds to clear pairing. The status light flashes twice. Disconnect the companion app before starting. Keep USB-C power connected throughout the reset.\n\n## Warranty\n\n", "metadata": { "document_id": "doc-aaa8b734a101d7f34afb", "matched_child_id": "child-0", "matched_child_start": 0, "matched_child_end": 99, "returned_context": "full_parent" } } ], "details": { "adapter": "unicode-codepoints-v1", "unit": "characters", "exact": true, "count": 741, "budget": null, "scope": "rendered_retrieval_context", "skipped": [], "policy": "ranked-whole-passages-v1" } } ] }, { "question_id": "Q2", "question": "How long is the warranty and what damage is excluded?", "hits": [ { "source_unit_id": "doc-aaa8b734a101d7f34afb", "rank": 1, "score": 2.69175124168396, "evidence_ids": [], "preview": "# Orion device manual\n\n## Reset pairing\n\nHold the reset button for eight seconds to clear pairing. The status light flashes twice. Disconnect the companion app before starting. Keep USB-C power connected throughout the reset.\n\n## Warranty\n\n", "metadata": { "document_id": "doc-aaa8b734a101d7f34afb", "matched_child_id": "child-3", "matched_child_start": 265, "matched_child_end": 358, "returned_context": "full_parent" }, "text": "# Orion device manual\n\n## Reset pairing\n\nHold the reset button for eight seconds to clear pairing. The status light flashes twice. Disconnect the companion app before starting. Keep USB-C power connected throughout the reset.\n\n## Warranty\n\nThe warranty lasts two years. Keep your receipt. Accidental water damage is excluded. Contact the support team before returning a device.\n\n## Battery care\n\nStore the device in a cool, dry place. Charge it every three months when unused. Use only the supplied USB-C cable. Recycle batteries at an approved collection point.\n\n## Delivery\n\nStandard delivery takes five working days. Tracked delivery is available in Morocco. Keep the tracking number until the parcel arrives.\n", "heading_path": [], "source_path": "manual.md", "location": {}, "location_label": "Source text", "loader": "utf8-text-v1", "start": 0, "end": 713, "origin_source_id": "doc-aaa8b734a101d7f34afb", "line_start": 1, "line_end": 17 } ], "query_time_ms": 0.15538499792455696, "context_characters": 713, "context_hash": "3b5a4e8ae04775940ba9a77c3d0decd0902b8c858a35f8fe631b46c96803b3c6", "observations": [], "metrics": { "question_id": "Q2", "split": "dev", "answerable": true, "tags": [], "required_evidence_ids": [ "EXCLUSION", "WARRANTY" ], "found_evidence_ids": [ "EXCLUSION", "WARRANTY" ], "missing_evidence_ids": [], "sufficient_evidence": true, "evidence_coverage": 1.0, "first_complete_evidence_rank": 1, "reciprocal_rank": 1.0, "precision_at_k": 1.0 }, "context_measurement": { "adapter": "unicode-codepoints-v1", "unit": "characters", "exact": true, "count": 741, "budget": null, "scope": "rendered_retrieval_context", "skipped": [], "policy": "ranked-whole-passages-v1" }, "stages": [ { "name": "children", "elapsed_ms": 0.08000199886737391, "hits": [ { "source_unit_id": "child-3", "rank": 1, "score": 2.69175124168396, "evidence_ids": [], "preview": "ars. Keep your receipt. Accidental water damage is excluded. Contact the support team before ", "metadata": { "document_id": "doc-aaa8b734a101d7f34afb" } }, { "source_unit_id": "child-2", "rank": 2, "score": 1.2320934534072876, "evidence_ids": [], "preview": "eep USB-C power connected throughout the reset.\n\n## Warranty\n\nThe warranty lasts two years. Keep ", "metadata": { "document_id": "doc-aaa8b734a101d7f34afb" } }, { "source_unit_id": "child-7", "rank": 3, "score": 1.2126452922821045, "evidence_ids": [], "preview": "ing days. Tracked delivery is available in Morocco. Keep the tracking number until the parcel ", "metadata": { "document_id": "doc-aaa8b734a101d7f34afb" } }, { "source_unit_id": "child-1", "rank": 4, "score": 0.16223493218421936, "evidence_ids": [], "preview": " pairing. The status light flashes twice. Disconnect the companion app before starting. Keep USB-C ", "metadata": { "document_id": "doc-aaa8b734a101d7f34afb" } }, { "source_unit_id": "child-0", "rank": 5, "score": 0.10937795042991638, "evidence_ids": [], "preview": "# Orion device manual\n\n## Reset pairing\n\nHold the reset button for eight seconds to clear pairing. ", "metadata": { "document_id": "doc-aaa8b734a101d7f34afb" } }, { "source_unit_id": "child-5", "rank": 6, "score": 0.10600543022155762, "evidence_ids": [], "preview": "Charge it every three months when unused. Use only the supplied USB-C cable. Recycle batteries at ", "metadata": { "document_id": "doc-aaa8b734a101d7f34afb" } }, { "source_unit_id": "child-4", "rank": 7, "score": 0.10283465683460236, "evidence_ids": [], "preview": "am before returning a device.\n\n## Battery care\n\nStore the device in a cool, dry place. Charge it ", "metadata": { "document_id": "doc-aaa8b734a101d7f34afb" } }, { "source_unit_id": "child-6", "rank": 8, "score": 0.0, "evidence_ids": [], "preview": "teries at an approved collection point.\n\n## Delivery\n\nStandard delivery takes five working days. ", "metadata": { "document_id": "doc-aaa8b734a101d7f34afb" } }, { "source_unit_id": "child-8", "rank": 9, "score": 0.0, "evidence_ids": [], "preview": "he parcel arrives.\n", "metadata": { "document_id": "doc-aaa8b734a101d7f34afb" } } ], "details": { "query": "How long is the warranty and what damage is excluded?", "limit": 20, "locations": [ { "child_id": "child-3", "parent_id": "doc-aaa8b734a101d7f34afb", "start": 265, "end": 358 }, { "child_id": "child-2", "parent_id": "doc-aaa8b734a101d7f34afb", "start": 178, "end": 275 }, { "child_id": "child-7", "parent_id": "doc-aaa8b734a101d7f34afb", "start": 610, "end": 704 }, { "child_id": "child-1", "parent_id": "doc-aaa8b734a101d7f34afb", "start": 89, "end": 188 }, { "child_id": "child-0", "parent_id": "doc-aaa8b734a101d7f34afb", "start": 0, "end": 99 }, { "child_id": "child-5", "parent_id": "doc-aaa8b734a101d7f34afb", "start": 435, "end": 533 }, { "child_id": "child-4", "parent_id": "doc-aaa8b734a101d7f34afb", "start": 348, "end": 445 }, { "child_id": "child-6", "parent_id": "doc-aaa8b734a101d7f34afb", "start": 523, "end": 620 }, { "child_id": "child-8", "parent_id": "doc-aaa8b734a101d7f34afb", "start": 694, "end": 713 } ] } }, { "name": "parent_expansion", "elapsed_ms": 0.005971996870357543, "hits": [ { "source_unit_id": "doc-aaa8b734a101d7f34afb", "rank": 1, "score": 2.69175124168396, "evidence_ids": [], "preview": "# Orion device manual\n\n## Reset pairing\n\nHold the reset button for eight seconds to clear pairing. The status light flashes twice. Disconnect the companion app before starting. Keep USB-C power connected throughout the reset.\n\n## Warranty\n\n", "metadata": { "document_id": "doc-aaa8b734a101d7f34afb", "matched_child_id": "child-3", "matched_child_start": 265, "matched_child_end": 358, "returned_context": "full_parent" } } ], "details": { "query": "How long is the warranty and what damage is excluded?" } }, { "name": "retrieval", "elapsed_ms": 0.0980160002654884, "hits": [ { "source_unit_id": "doc-aaa8b734a101d7f34afb", "rank": 1, "score": 2.69175124168396, "evidence_ids": [], "preview": "# Orion device manual\n\n## Reset pairing\n\nHold the reset button for eight seconds to clear pairing. The status light flashes twice. Disconnect the companion app before starting. Keep USB-C power connected throughout the reset.\n\n## Warranty\n\n", "metadata": { "document_id": "doc-aaa8b734a101d7f34afb", "matched_child_id": "child-3", "matched_child_start": 265, "matched_child_end": 358, "returned_context": "full_parent" } } ], "details": { "query": "How long is the warranty and what damage is excluded?", "limit": 1, "index_cache": "not_applicable", "model_cache": "unknown", "runtime_versions": { "bm25s": "0.2.14", "numpy": "2.5.4" } } }, { "name": "context_selection", "elapsed_ms": 0.02166199919884093, "hits": [ { "source_unit_id": "doc-aaa8b734a101d7f34afb", "rank": 1, "score": 2.69175124168396, "evidence_ids": [], "preview": "# Orion device manual\n\n## Reset pairing\n\nHold the reset button for eight seconds to clear pairing. The status light flashes twice. Disconnect the companion app before starting. Keep USB-C power connected throughout the reset.\n\n## Warranty\n\n", "metadata": { "document_id": "doc-aaa8b734a101d7f34afb", "matched_child_id": "child-3", "matched_child_start": 265, "matched_child_end": 358, "returned_context": "full_parent" } } ], "details": { "adapter": "unicode-codepoints-v1", "unit": "characters", "exact": true, "count": 741, "budget": null, "scope": "rendered_retrieval_context", "skipped": [], "policy": "ranked-whole-passages-v1" } } ] }, { "question_id": "Q3", "question": "How often should I charge an unused device?", "hits": [ { "source_unit_id": "doc-aaa8b734a101d7f34afb", "rank": 1, "score": 1.2256488800048828, "evidence_ids": [], "preview": "# Orion device manual\n\n## Reset pairing\n\nHold the reset button for eight seconds to clear pairing. The status light flashes twice. Disconnect the companion app before starting. Keep USB-C power connected throughout the reset.\n\n## Warranty\n\n", "metadata": { "document_id": "doc-aaa8b734a101d7f34afb", "matched_child_id": "child-4", "matched_child_start": 348, "matched_child_end": 445, "returned_context": "full_parent" }, "text": "# Orion device manual\n\n## Reset pairing\n\nHold the reset button for eight seconds to clear pairing. The status light flashes twice. Disconnect the companion app before starting. Keep USB-C power connected throughout the reset.\n\n## Warranty\n\nThe warranty lasts two years. Keep your receipt. Accidental water damage is excluded. Contact the support team before returning a device.\n\n## Battery care\n\nStore the device in a cool, dry place. Charge it every three months when unused. Use only the supplied USB-C cable. Recycle batteries at an approved collection point.\n\n## Delivery\n\nStandard delivery takes five working days. Tracked delivery is available in Morocco. Keep the tracking number until the parcel arrives.\n", "heading_path": [], "source_path": "manual.md", "location": {}, "location_label": "Source text", "loader": "utf8-text-v1", "start": 0, "end": 713, "origin_source_id": "doc-aaa8b734a101d7f34afb", "line_start": 1, "line_end": 17 } ], "query_time_ms": 0.13644899809150957, "context_characters": 713, "context_hash": "3b5a4e8ae04775940ba9a77c3d0decd0902b8c858a35f8fe631b46c96803b3c6", "observations": [], "metrics": { "question_id": "Q3", "split": "dev", "answerable": true, "tags": [], "required_evidence_ids": [ "CHARGE" ], "found_evidence_ids": [ "CHARGE" ], "missing_evidence_ids": [], "sufficient_evidence": true, "evidence_coverage": 1.0, "first_complete_evidence_rank": 1, "reciprocal_rank": 1.0, "precision_at_k": 1.0 }, "context_measurement": { "adapter": "unicode-codepoints-v1", "unit": "characters", "exact": true, "count": 741, "budget": null, "scope": "rendered_retrieval_context", "skipped": [], "policy": "ranked-whole-passages-v1" }, "stages": [ { "name": "children", "elapsed_ms": 0.07028399704722688, "hits": [ { "source_unit_id": "child-4", "rank": 1, "score": 1.2256488800048828, "evidence_ids": [], "preview": "am before returning a device.\n\n## Battery care\n\nStore the device in a cool, dry place. Charge it ", "metadata": { "document_id": "doc-aaa8b734a101d7f34afb" } }, { "source_unit_id": "child-5", "rank": 2, "score": 1.209876298904419, "evidence_ids": [], "preview": "Charge it every three months when unused. Use only the supplied USB-C cable. Recycle batteries at ", "metadata": { "document_id": "doc-aaa8b734a101d7f34afb" } }, { "source_unit_id": "child-6", "rank": 3, "score": 0.7703071236610413, "evidence_ids": [], "preview": "teries at an approved collection point.\n\n## Delivery\n\nStandard delivery takes five working days. ", "metadata": { "document_id": "doc-aaa8b734a101d7f34afb" } }, { "source_unit_id": "child-0", "rank": 4, "score": 0.5270749926567078, "evidence_ids": [], "preview": "# Orion device manual\n\n## Reset pairing\n\nHold the reset button for eight seconds to clear pairing. ", "metadata": { "document_id": "doc-aaa8b734a101d7f34afb" } }, { "source_unit_id": "child-1", "rank": 5, "score": 0.0, "evidence_ids": [], "preview": " pairing. The status light flashes twice. Disconnect the companion app before starting. Keep USB-C ", "metadata": { "document_id": "doc-aaa8b734a101d7f34afb" } }, { "source_unit_id": "child-2", "rank": 6, "score": 0.0, "evidence_ids": [], "preview": "eep USB-C power connected throughout the reset.\n\n## Warranty\n\nThe warranty lasts two years. Keep ", "metadata": { "document_id": "doc-aaa8b734a101d7f34afb" } }, { "source_unit_id": "child-3", "rank": 7, "score": 0.0, "evidence_ids": [], "preview": "ars. Keep your receipt. Accidental water damage is excluded. Contact the support team before ", "metadata": { "document_id": "doc-aaa8b734a101d7f34afb" } }, { "source_unit_id": "child-7", "rank": 8, "score": 0.0, "evidence_ids": [], "preview": "ing days. Tracked delivery is available in Morocco. Keep the tracking number until the parcel ", "metadata": { "document_id": "doc-aaa8b734a101d7f34afb" } }, { "source_unit_id": "child-8", "rank": 9, "score": 0.0, "evidence_ids": [], "preview": "he parcel arrives.\n", "metadata": { "document_id": "doc-aaa8b734a101d7f34afb" } } ], "details": { "query": "How often should I charge an unused device?", "limit": 20, "locations": [ { "child_id": "child-4", "parent_id": "doc-aaa8b734a101d7f34afb", "start": 348, "end": 445 }, { "child_id": "child-5", "parent_id": "doc-aaa8b734a101d7f34afb", "start": 435, "end": 533 }, { "child_id": "child-6", "parent_id": "doc-aaa8b734a101d7f34afb", "start": 523, "end": 620 }, { "child_id": "child-0", "parent_id": "doc-aaa8b734a101d7f34afb", "start": 0, "end": 99 }, { "child_id": "child-1", "parent_id": "doc-aaa8b734a101d7f34afb", "start": 89, "end": 188 }, { "child_id": "child-2", "parent_id": "doc-aaa8b734a101d7f34afb", "start": 178, "end": 275 }, { "child_id": "child-3", "parent_id": "doc-aaa8b734a101d7f34afb", "start": 265, "end": 358 }, { "child_id": "child-7", "parent_id": "doc-aaa8b734a101d7f34afb", "start": 610, "end": 704 }, { "child_id": "child-8", "parent_id": "doc-aaa8b734a101d7f34afb", "start": 694, "end": 713 } ] } }, { "name": "parent_expansion", "elapsed_ms": 0.004427998646860942, "hits": [ { "source_unit_id": "doc-aaa8b734a101d7f34afb", "rank": 1, "score": 1.2256488800048828, "evidence_ids": [], "preview": "# Orion device manual\n\n## Reset pairing\n\nHold the reset button for eight seconds to clear pairing. The status light flashes twice. Disconnect the companion app before starting. Keep USB-C power connected throughout the reset.\n\n## Warranty\n\n", "metadata": { "document_id": "doc-aaa8b734a101d7f34afb", "matched_child_id": "child-4", "matched_child_start": 348, "matched_child_end": 445, "returned_context": "full_parent" } } ], "details": { "query": "How often should I charge an unused device?" } }, { "name": "retrieval", "elapsed_ms": 0.08442000034847297, "hits": [ { "source_unit_id": "doc-aaa8b734a101d7f34afb", "rank": 1, "score": 1.2256488800048828, "evidence_ids": [], "preview": "# Orion device manual\n\n## Reset pairing\n\nHold the reset button for eight seconds to clear pairing. The status light flashes twice. Disconnect the companion app before starting. Keep USB-C power connected throughout the reset.\n\n## Warranty\n\n", "metadata": { "document_id": "doc-aaa8b734a101d7f34afb", "matched_child_id": "child-4", "matched_child_start": 348, "matched_child_end": 445, "returned_context": "full_parent" } } ], "details": { "query": "How often should I charge an unused device?", "limit": 1, "index_cache": "not_applicable", "model_cache": "unknown", "runtime_versions": { "bm25s": "0.2.14", "numpy": "2.5.4" } } }, { "name": "context_selection", "elapsed_ms": 0.019537001207936555, "hits": [ { "source_unit_id": "doc-aaa8b734a101d7f34afb", "rank": 1, "score": 1.2256488800048828, "evidence_ids": [], "preview": "# Orion device manual\n\n## Reset pairing\n\nHold the reset button for eight seconds to clear pairing. The status light flashes twice. Disconnect the companion app before starting. Keep USB-C power connected throughout the reset.\n\n## Warranty\n\n", "metadata": { "document_id": "doc-aaa8b734a101d7f34afb", "matched_child_id": "child-4", "matched_child_start": 348, "matched_child_end": 445, "returned_context": "full_parent" } } ], "details": { "adapter": "unicode-codepoints-v1", "unit": "characters", "exact": true, "count": 741, "budget": null, "scope": "rendered_retrieval_context", "skipped": [], "policy": "ranked-whole-passages-v1" } } ] }, { "question_id": "Q4", "question": "Where is tracked delivery available?", "hits": [ { "source_unit_id": "doc-aaa8b734a101d7f34afb", "rank": 1, "score": 2.4967360496520996, "evidence_ids": [], "preview": "# Orion device manual\n\n## Reset pairing\n\nHold the reset button for eight seconds to clear pairing. The status light flashes twice. Disconnect the companion app before starting. Keep USB-C power connected throughout the reset.\n\n## Warranty\n\n", "metadata": { "document_id": "doc-aaa8b734a101d7f34afb", "matched_child_id": "child-7", "matched_child_start": 610, "matched_child_end": 704, "returned_context": "full_parent" }, "text": "# Orion device manual\n\n## Reset pairing\n\nHold the reset button for eight seconds to clear pairing. The status light flashes twice. Disconnect the companion app before starting. Keep USB-C power connected throughout the reset.\n\n## Warranty\n\nThe warranty lasts two years. Keep your receipt. Accidental water damage is excluded. Contact the support team before returning a device.\n\n## Battery care\n\nStore the device in a cool, dry place. Charge it every three months when unused. Use only the supplied USB-C cable. Recycle batteries at an approved collection point.\n\n## Delivery\n\nStandard delivery takes five working days. Tracked delivery is available in Morocco. Keep the tracking number until the parcel arrives.\n", "heading_path": [], "source_path": "manual.md", "location": {}, "location_label": "Source text", "loader": "utf8-text-v1", "start": 0, "end": 713, "origin_source_id": "doc-aaa8b734a101d7f34afb", "line_start": 1, "line_end": 17 } ], "query_time_ms": 0.150306001160061, "context_characters": 713, "context_hash": "3b5a4e8ae04775940ba9a77c3d0decd0902b8c858a35f8fe631b46c96803b3c6", "observations": [], "metrics": { "question_id": "Q4", "split": "dev", "answerable": true, "tags": [], "required_evidence_ids": [ "DELIVERY" ], "found_evidence_ids": [ "DELIVERY" ], "missing_evidence_ids": [], "sufficient_evidence": true, "evidence_coverage": 1.0, "first_complete_evidence_rank": 1, "reciprocal_rank": 1.0, "precision_at_k": 1.0 }, "context_measurement": { "adapter": "unicode-codepoints-v1", "unit": "characters", "exact": true, "count": 741, "budget": null, "scope": "rendered_retrieval_context", "skipped": [], "policy": "ranked-whole-passages-v1" }, "stages": [ { "name": "children", "elapsed_ms": 0.07966200064402074, "hits": [ { "source_unit_id": "child-7", "rank": 1, "score": 2.4967360496520996, "evidence_ids": [], "preview": "ing days. Tracked delivery is available in Morocco. Keep the tracking number until the parcel ", "metadata": { "document_id": "doc-aaa8b734a101d7f34afb" } }, { "source_unit_id": "child-6", "rank": 2, "score": 0.8006759881973267, "evidence_ids": [], "preview": "teries at an approved collection point.\n\n## Delivery\n\nStandard delivery takes five working days. ", "metadata": { "document_id": "doc-aaa8b734a101d7f34afb" } }, { "source_unit_id": "child-3", "rank": 3, "score": 0.5443947315216064, "evidence_ids": [], "preview": "ars. Keep your receipt. Accidental water damage is excluded. Contact the support team before ", "metadata": { "document_id": "doc-aaa8b734a101d7f34afb" } }, { "source_unit_id": "child-0", "rank": 4, "score": 0.0, "evidence_ids": [], "preview": "# Orion device manual\n\n## Reset pairing\n\nHold the reset button for eight seconds to clear pairing. ", "metadata": { "document_id": "doc-aaa8b734a101d7f34afb" } }, { "source_unit_id": "child-1", "rank": 5, "score": 0.0, "evidence_ids": [], "preview": " pairing. The status light flashes twice. Disconnect the companion app before starting. Keep USB-C ", "metadata": { "document_id": "doc-aaa8b734a101d7f34afb" } }, { "source_unit_id": "child-2", "rank": 6, "score": 0.0, "evidence_ids": [], "preview": "eep USB-C power connected throughout the reset.\n\n## Warranty\n\nThe warranty lasts two years. Keep ", "metadata": { "document_id": "doc-aaa8b734a101d7f34afb" } }, { "source_unit_id": "child-4", "rank": 7, "score": 0.0, "evidence_ids": [], "preview": "am before returning a device.\n\n## Battery care\n\nStore the device in a cool, dry place. Charge it ", "metadata": { "document_id": "doc-aaa8b734a101d7f34afb" } }, { "source_unit_id": "child-5", "rank": 8, "score": 0.0, "evidence_ids": [], "preview": "Charge it every three months when unused. Use only the supplied USB-C cable. Recycle batteries at ", "metadata": { "document_id": "doc-aaa8b734a101d7f34afb" } }, { "source_unit_id": "child-8", "rank": 9, "score": 0.0, "evidence_ids": [], "preview": "he parcel arrives.\n", "metadata": { "document_id": "doc-aaa8b734a101d7f34afb" } } ], "details": { "query": "Where is tracked delivery available?", "limit": 20, "locations": [ { "child_id": "child-7", "parent_id": "doc-aaa8b734a101d7f34afb", "start": 610, "end": 704 }, { "child_id": "child-6", "parent_id": "doc-aaa8b734a101d7f34afb", "start": 523, "end": 620 }, { "child_id": "child-3", "parent_id": "doc-aaa8b734a101d7f34afb", "start": 265, "end": 358 }, { "child_id": "child-0", "parent_id": "doc-aaa8b734a101d7f34afb", "start": 0, "end": 99 }, { "child_id": "child-1", "parent_id": "doc-aaa8b734a101d7f34afb", "start": 89, "end": 188 }, { "child_id": "child-2", "parent_id": "doc-aaa8b734a101d7f34afb", "start": 178, "end": 275 }, { "child_id": "child-4", "parent_id": "doc-aaa8b734a101d7f34afb", "start": 348, "end": 445 }, { "child_id": "child-5", "parent_id": "doc-aaa8b734a101d7f34afb", "start": 435, "end": 533 }, { "child_id": "child-8", "parent_id": "doc-aaa8b734a101d7f34afb", "start": 694, "end": 713 } ] } }, { "name": "parent_expansion", "elapsed_ms": 0.005029000021750107, "hits": [ { "source_unit_id": "doc-aaa8b734a101d7f34afb", "rank": 1, "score": 2.4967360496520996, "evidence_ids": [], "preview": "# Orion device manual\n\n## Reset pairing\n\nHold the reset button for eight seconds to clear pairing. The status light flashes twice. Disconnect the companion app before starting. Keep USB-C power connected throughout the reset.\n\n## Warranty\n\n", "metadata": { "document_id": "doc-aaa8b734a101d7f34afb", "matched_child_id": "child-7", "matched_child_start": 610, "matched_child_end": 704, "returned_context": "full_parent" } } ], "details": { "query": "Where is tracked delivery available?" } }, { "name": "retrieval", "elapsed_ms": 0.09733499973663129, "hits": [ { "source_unit_id": "doc-aaa8b734a101d7f34afb", "rank": 1, "score": 2.4967360496520996, "evidence_ids": [], "preview": "# Orion device manual\n\n## Reset pairing\n\nHold the reset button for eight seconds to clear pairing. The status light flashes twice. Disconnect the companion app before starting. Keep USB-C power connected throughout the reset.\n\n## Warranty\n\n", "metadata": { "document_id": "doc-aaa8b734a101d7f34afb", "matched_child_id": "child-7", "matched_child_start": 610, "matched_child_end": 704, "returned_context": "full_parent" } } ], "details": { "query": "Where is tracked delivery available?", "limit": 1, "index_cache": "not_applicable", "model_cache": "unknown", "runtime_versions": { "bm25s": "0.2.14", "numpy": "2.5.4" } } }, { "name": "context_selection", "elapsed_ms": 0.02251300247735344, "hits": [ { "source_unit_id": "doc-aaa8b734a101d7f34afb", "rank": 1, "score": 2.4967360496520996, "evidence_ids": [], "preview": "# Orion device manual\n\n## Reset pairing\n\nHold the reset button for eight seconds to clear pairing. The status light flashes twice. Disconnect the companion app before starting. Keep USB-C power connected throughout the reset.\n\n## Warranty\n\n", "metadata": { "document_id": "doc-aaa8b734a101d7f34afb", "matched_child_id": "child-7", "matched_child_start": 610, "matched_child_end": 704, "returned_context": "full_parent" } } ], "details": { "adapter": "unicode-codepoints-v1", "unit": "characters", "exact": true, "count": 741, "budget": null, "scope": "rendered_retrieval_context", "skipped": [], "policy": "ranked-whole-passages-v1" } } ] } ], "observations": [ "Reviewed evidence is scored against the union of exact original-text spans.", "Child retrieval returns full parent source units. Evidence metrics describe parent context; they do not establish that the matching child contains every fact. Child candidate limits can return fewer than k unique parents." ], "limitations": [ "Answer correctness, citations, abstention and generation request tokens are not evaluated.", "Retriever scores describe this ranking; they are not probabilities of correctness.", "Chunk sizes count characters. Rendered context counts characters unless an explicit local tokenizer is identified; model request overhead is excluded.", "Character and text-line offsets refer to the extracted page/record/section, not byte offsets in the original file. Extraction does not verify layout fidelity." ] }, "resources": { "query_ms": { "mean_known": 0.16565724945394322, "known_count": 4, "missing_count": 0 }, "runtime_versions": { "bm25s": "0.2.14", "numpy": "2.5.4" }, "context_units": { "mean_known": 741.0, "known_count": 4, "missing_count": 0 }, "context_characters": { "mean_known": 713.0, "known_count": 4, "missing_count": 0 }, "context_unit": "characters", "index_ms": 0.9598130018275697, "vector_cache": "not_applicable", "model_cache": "unknown" }, "usage": { "source": "user_supplied_usage", "scope": "supplied_totals_not_inferred_from_repeats", "estimate": true, "cost": null, "currency": null, "pricing_identity": null, "input_tokens": null, "output_tokens": null }, "trials": [ { "trial": 0, "reused": false, "artifact_directory": "jobs/parent-child/trial-0/attempt-20261019T152652-c6c2bd1fda1c", "resources": { "query_ms": { "mean_known": 0.16565724945394322, "known_count": 4, "missing_count": 0 }, "runtime_versions": { "bm25s": "0.2.14", "numpy": "2.5.4" }, "context_units": { "mean_known": 741.0, "known_count": 4, "missing_count": 0 }, "context_characters": { "mean_known": 713.0, "known_count": 4, "missing_count": 0 }, "context_unit": "characters", "index_ms": 0.9598130018275697, "vector_cache": "not_applicable", "model_cache": "unknown" } } ], "repeat_rankings_changed": false, "evidence_scope": "exact_returned_spans", "summaries": { "dev": { "all": { "question_count": 4, "answerable_questions": 4, "sufficient_count": 4, "sufficiency_rate": 1.0, "mean_evidence_coverage": 1.0, "mean_reciprocal_rank": 1.0, "mean_precision_at_k": 1.0 } }, "heldout": { "all": { "question_count": 0, "answerable_questions": 0, "sufficient_count": 0, "sufficiency_rate": null, "mean_evidence_coverage": null, "mean_reciprocal_rank": null, "mean_precision_at_k": null } } }, "split_resources": { "dev": { "query_ms": { "mean_known": 0.16565724945394322, "known_count": 4, "missing_count": 0 }, "runtime_versions": { "bm25s": "0.2.14", "numpy": "2.5.4" }, "context_units": { "mean_known": 741.0, "known_count": 4, "missing_count": 0 }, "context_characters": { "mean_known": 713.0, "known_count": 4, "missing_count": 0 }, "context_unit": "characters", "index_ms": 0.9598130018275697, "vector_cache": "not_applicable", "model_cache": "unknown" }, "heldout": { "query_ms": { "mean_known": null, "known_count": 0, "missing_count": 0 }, "runtime_versions": {}, "context_units": { "mean_known": null, "known_count": 0, "missing_count": 0 }, "context_characters": { "mean_known": null, "known_count": 0, "missing_count": 0 }, "context_unit": null, "index_ms": 0.9598130018275697, "vector_cache": "unavailable", "model_cache": "unknown" } }, "criteria": { "dev": { "status": "not_requested", "checks": [] }, "heldout": { "status": "not_requested", "checks": [] } } }, { "name": "compress", "state": "complete", "error": null, "report": { "schema_version": 1, "kind": "inspection", "generated_at": "2026-10-19T15:26:52.054029+00:00", "evaluation_level": "reviewed_evaluation", "preparation": { "implementation": "portable-files-v1", "length_unit": "characters", "chunk_size": 100, "chunk_overlap": 10, "text_field": null, "include_patterns": [], "exclude_patterns": [], "limits": { "max_file_bytes": 20971520, "max_total_bytes": 104857600, "max_files": 1000, "max_records": 10000, "max_extracted_characters": 20971520 } }, "manifest": { "run_id": "eb9e9ebe6a0a", "schema_version": 2, "hash_algorithm": "canonical-json-v1", "evaluation_level": "reviewed_evaluation", "created_at": "2026-10-19T15:26:52+00:00", "product_version": "0.2.1", "python_version": "3.12.14", "platform": "Linux-6.18.33.2-microsoft-standard-WSL2-x86_64-with-glibc2.43", "dataset_hash": "6607e06b253048c606257174df51e82d8070a0a8f30cb605c181c8201303e4d4", "corpus_hash": "b4bdb92293aa66d4636c27dd25ca77a791dbbeea111d33eb4873d314acfa0386", "questions_hash": "c058b113c76f2d061279f04523dfe7348131314011b7de2045eb3c9726603a7e", "references_hash": "cd3ccaa312b66201e19ad814a9d61015802a174050757bd54fa1ed7a9584e338", "representation_hash": "a04060688ab87c64a6a3964da1e83b86e20db941c809b6ed19f6b91df32bf6a8", "configuration_hash": "3c821ba57282f475b99c66ae939034aad7d4bf4de8308da6a7f977050a1c0a9d", "input_preparation": { "implementation": "portable-files-v1", "length_unit": "characters", "chunk_size": 100, "chunk_overlap": 10, "text_field": null, "include_patterns": [], "exclude_patterns": [], "limits": { "max_file_bytes": 20971520, "max_total_bytes": 104857600, "max_files": 1000, "max_records": 10000, "max_extracted_characters": 20971520 } }, "config": { "name": "compress", "strategy": "bm25", "representation": "body", "k": 1, "model": null, "provider": null, "candidate_k": 20, "lexical_weight": 0.5, "rrf_constant": 60, "child_splitter": "none", "child_size": 500, "child_overlap": 50, "query_variants": {}, "reranker_model": null, "compression": { "mode": "sentences", "max_spans": 2, "max_output_tokens": 1024, "timeout_seconds": 60.0, "model": null, "provider": "ollama", "failure": "fail", "endpoint_hash": null, "prompt_version": "extractive-spans-v1" } }, "index_time_ms": 0.5941989984421525, "warnings": [ "Extractive compression can remove required facts or qualifications; evidence is rescored from only the returned spans." ] }, "evaluation": { "schema_version": 1, "method": "original-span-union-v1", "references_hash": "cd3ccaa312b66201e19ad814a9d61015802a174050757bd54fa1ed7a9584e338", "questions": [ { "question_id": "Q1", "split": "dev", "answerable": true, "tags": [], "required_evidence_ids": [ "POWER", "RESET" ], "found_evidence_ids": [ "RESET" ], "missing_evidence_ids": [ "POWER" ], "sufficient_evidence": false, "evidence_coverage": 0.5, "first_complete_evidence_rank": 1, "reciprocal_rank": 1.0, "precision_at_k": 1.0 }, { "question_id": "Q2", "split": "dev", "answerable": true, "tags": [], "required_evidence_ids": [ "EXCLUSION", "WARRANTY" ], "found_evidence_ids": [ "EXCLUSION", "WARRANTY" ], "missing_evidence_ids": [], "sufficient_evidence": true, "evidence_coverage": 1.0, "first_complete_evidence_rank": 1, "reciprocal_rank": 1.0, "precision_at_k": 1.0 }, { "question_id": "Q3", "split": "dev", "answerable": true, "tags": [], "required_evidence_ids": [ "CHARGE" ], "found_evidence_ids": [ "CHARGE" ], "missing_evidence_ids": [], "sufficient_evidence": true, "evidence_coverage": 1.0, "first_complete_evidence_rank": 1, "reciprocal_rank": 1.0, "precision_at_k": 1.0 }, { "question_id": "Q4", "split": "dev", "answerable": true, "tags": [], "required_evidence_ids": [ "DELIVERY" ], "found_evidence_ids": [ "DELIVERY" ], "missing_evidence_ids": [], "sufficient_evidence": true, "evidence_coverage": 1.0, "first_complete_evidence_rank": 1, "reciprocal_rank": 1.0, "precision_at_k": 1.0 } ], "summaries": { "dev": { "question_count": 4, "answerable_questions": 4, "sufficient_count": 3, "sufficiency_rate": 0.75, "mean_evidence_coverage": 0.875, "mean_reciprocal_rank": 1.0, "mean_precision_at_k": 1.0 }, "heldout": { "question_count": 0, "answerable_questions": 0, "sufficient_count": 0, "sufficiency_rate": null, "mean_evidence_coverage": null, "mean_reciprocal_rank": null, "mean_precision_at_k": null } }, "definitions": { "reciprocal_rank": "First rank prefix completing at least one required evidence item", "precision_at_k": "Returned hits overlapping required spans divided by configured k", "sufficiency": "All spans of every required item covered by the returned union", "unanswerable": "Excluded from retrieval quality rates; abstention is not evaluated" }, "scope": "exact_returned_spans" }, "input": { "name": "docs", "corpus_hash": "b4bdb92293aa66d4636c27dd25ca77a791dbbeea111d33eb4873d314acfa0386", "document_count": 1, "file_count": 1, "passage_count": 10, "source_characters": 713, "indexed_characters": 763, "skipped": [], "skipped_count": 0, "failed_count": 0, "warning_count": 0, "sources": [ { "source_unit_id": "doc-aaa8b734a101d7f34afb", "source_path": "manual.md", "location": {}, "location_label": "Source text", "heading_path": [], "loader": "utf8-text-v1", "file_sha256": "60d754e2f2aa039fe08127ae15f0775df6ea8309a1531920cbd9d94e46d405b9", "text_hash": "e24765cdef868e7e3b83177ad778e6eb80c5913fe7501e39f37302bc8092ca7b", "characters": 713 } ] }, "questions": [ { "question_id": "Q1", "question": "How long should I hold the reset button, and what power is required?", "hits": [ { "source_unit_id": "passage-5bdc4916bcb2462d0250", "rank": 1, "score": 2.2885582447052, "evidence_ids": [], "preview": "## Reset pairing\n\n", "metadata": { "document_id": "doc-aaa8b734a101d7f34afb", "compression_input_rank": 1 }, "context_spans": [ { "start": 0, "end": 18 }, { "start": 18, "end": 75 } ], "text": "## Reset pairing\n\n", "heading_path": [ "Orion device manual", "Reset pairing" ], "source_path": "manual.md", "location": {}, "location_label": "Source text", "loader": "utf8-text-v1", "start": 23, "end": 41, "origin_source_id": "doc-aaa8b734a101d7f34afb", "line_start": 3, "line_end": 4, "context_part": 1 }, { "source_unit_id": "passage-5bdc4916bcb2462d0250", "rank": 1, "score": 2.2885582447052, "evidence_ids": [], "preview": "## Reset pairing\n\n", "metadata": { "document_id": "doc-aaa8b734a101d7f34afb", "compression_input_rank": 1 }, "context_spans": [ { "start": 0, "end": 18 }, { "start": 18, "end": 75 } ], "text": "Hold the reset button for eight seconds to clear pairing.", "heading_path": [ "Orion device manual", "Reset pairing" ], "source_path": "manual.md", "location": {}, "location_label": "Source text", "loader": "utf8-text-v1", "start": 41, "end": 98, "origin_source_id": "doc-aaa8b734a101d7f34afb", "line_start": 5, "line_end": 5, "context_part": 2 } ], "query_time_ms": 0.5768959999841172, "context_characters": 75, "context_hash": "120f4463ddf7f8c302a3a3115862e534e83f6c693d0e22e56f8e1a86ffda4e39", "observations": [], "metrics": { "question_id": "Q1", "split": "dev", "answerable": true, "tags": [], "required_evidence_ids": [ "POWER", "RESET" ], "found_evidence_ids": [ "RESET" ], "missing_evidence_ids": [ "POWER" ], "sufficient_evidence": false, "evidence_coverage": 0.5, "first_complete_evidence_rank": 1, "reciprocal_rank": 1.0, "precision_at_k": 1.0 }, "context_measurement": { "adapter": "unicode-codepoints-v1", "unit": "characters", "exact": true, "count": 211, "budget": null, "scope": "rendered_retrieval_context", "skipped": [], "policy": "ranked-whole-passages-v1" }, "stages": [ { "name": "retrieval", "elapsed_ms": 0.11848500071209855, "hits": [ { "source_unit_id": "passage-5bdc4916bcb2462d0250", "rank": 1, "score": 2.2885582447052, "evidence_ids": [], "preview": "## Reset pairing\n\nHold the reset button for eight seconds to clear pairing. The status light ", "metadata": { "document_id": "doc-aaa8b734a101d7f34afb" } }, { "source_unit_id": "passage-a9bffaf7e82ff6d02b1e", "rank": 2, "score": 0.9809407591819763, "evidence_ids": [], "preview": "connected throughout the reset.\n\n", "metadata": { "document_id": "doc-aaa8b734a101d7f34afb" } }, { "source_unit_id": "passage-fc31aa14f18cf8b2a89a", "rank": 3, "score": 0.8127185106277466, "evidence_ids": [], "preview": "tus light flashes twice. Disconnect the companion app before starting. Keep USB-C power connected ", "metadata": { "document_id": "doc-aaa8b734a101d7f34afb" } }, { "source_unit_id": "passage-4be5e239b1ab47982a8e", "rank": 4, "score": 0.6282253265380859, "evidence_ids": [], "preview": "## Warranty\n\nThe warranty lasts two years. Keep your receipt. Accidental water damage is excluded. ", "metadata": { "document_id": "doc-aaa8b734a101d7f34afb" } }, { "source_unit_id": "passage-1518103967d65de40d95", "rank": 5, "score": 0.5550644397735596, "evidence_ids": [], "preview": "## Delivery\n\nStandard delivery takes five working days. Tracked delivery is available in Morocco. ", "metadata": { "document_id": "doc-aaa8b734a101d7f34afb" } }, { "source_unit_id": "passage-3657f06cb41040af19b1", "rank": 6, "score": 0.15764456987380981, "evidence_ids": [], "preview": " Morocco. Keep the tracking number until the parcel arrives.\n", "metadata": { "document_id": "doc-aaa8b734a101d7f34afb" } }, { "source_unit_id": "passage-6c50fe60990136f40ac3", "rank": 7, "score": 0.11353020370006561, "evidence_ids": [], "preview": "excluded. Contact the support team before returning a device.\n\n", "metadata": { "document_id": "doc-aaa8b734a101d7f34afb" } }, { "source_unit_id": "passage-a73c9da59294e4af482b", "rank": 8, "score": 0.08988720178604126, "evidence_ids": [], "preview": "n unused. Use only the supplied USB-C cable. Recycle batteries at an approved collection point.\n\n", "metadata": { "document_id": "doc-aaa8b734a101d7f34afb" } }, { "source_unit_id": "passage-32a3368b3b75076e202c", "rank": 9, "score": 0.08405247330665588, "evidence_ids": [], "preview": "## Battery care\n\nStore the device in a cool, dry place. Charge it every three months when unused. ", "metadata": { "document_id": "doc-aaa8b734a101d7f34afb" } }, { "source_unit_id": "passage-c341c5c87a667038141c", "rank": 10, "score": 0.0, "evidence_ids": [], "preview": "# Orion device manual\n\n", "metadata": { "document_id": "doc-aaa8b734a101d7f34afb" } } ], "details": { "query": "How long should I hold the reset button, and what power is required?", "limit": 20, "index_cache": "not_applicable", "model_cache": "unknown", "runtime_versions": { "bm25s": "0.2.14", "numpy": "2.5.4" } } }, { "name": "compression", "elapsed_ms": 0.3726879986061249, "hits": [ { "source_unit_id": "passage-5bdc4916bcb2462d0250", "rank": 1, "score": 2.2885582447052, "evidence_ids": [], "preview": "## Reset pairing\n\n", "metadata": { "document_id": "doc-aaa8b734a101d7f34afb", "compression_input_rank": 1 }, "context_spans": [ { "start": 0, "end": 18 }, { "start": 18, "end": 75 } ] }, { "source_unit_id": "passage-a9bffaf7e82ff6d02b1e", "rank": 2, "score": 0.9809407591819763, "evidence_ids": [], "preview": "connected throughout the reset.", "metadata": { "document_id": "doc-aaa8b734a101d7f34afb", "compression_input_rank": 2 }, "context_spans": [ { "start": 0, "end": 31 } ] }, { "source_unit_id": "passage-fc31aa14f18cf8b2a89a", "rank": 3, "score": 0.8127185106277466, "evidence_ids": [], "preview": " Disconnect the companion app before starting.", "metadata": { "document_id": "doc-aaa8b734a101d7f34afb", "compression_input_rank": 3 }, "context_spans": [ { "start": 24, "end": 70 }, { "start": 70, "end": 98 } ] }, { "source_unit_id": "passage-4be5e239b1ab47982a8e", "rank": 4, "score": 0.6282253265380859, "evidence_ids": [], "preview": "The warranty lasts two years.", "metadata": { "document_id": "doc-aaa8b734a101d7f34afb", "compression_input_rank": 4 }, "context_spans": [ { "start": 13, "end": 42 }, { "start": 61, "end": 98 } ] }, { "source_unit_id": "passage-1518103967d65de40d95", "rank": 5, "score": 0.5550644397735596, "evidence_ids": [], "preview": " Tracked delivery is available in Morocco.", "metadata": { "document_id": "doc-aaa8b734a101d7f34afb", "compression_input_rank": 5 }, "context_spans": [ { "start": 55, "end": 97 } ] }, { "source_unit_id": "passage-3657f06cb41040af19b1", "rank": 6, "score": 0.15764456987380981, "evidence_ids": [], "preview": " Keep the tracking number until the parcel arrives.", "metadata": { "document_id": "doc-aaa8b734a101d7f34afb", "compression_input_rank": 6 }, "context_spans": [ { "start": 9, "end": 60 } ] }, { "source_unit_id": "passage-6c50fe60990136f40ac3", "rank": 7, "score": 0.11353020370006561, "evidence_ids": [], "preview": " Contact the support team before returning a device.", "metadata": { "document_id": "doc-aaa8b734a101d7f34afb", "compression_input_rank": 7 }, "context_spans": [ { "start": 9, "end": 61 } ] }, { "source_unit_id": "passage-a73c9da59294e4af482b", "rank": 8, "score": 0.08988720178604126, "evidence_ids": [], "preview": " Use only the supplied USB-C cable.", "metadata": { "document_id": "doc-aaa8b734a101d7f34afb", "compression_input_rank": 8 }, "context_spans": [ { "start": 9, "end": 44 } ] }, { "source_unit_id": "passage-32a3368b3b75076e202c", "rank": 9, "score": 0.08405247330665588, "evidence_ids": [], "preview": "Store the device in a cool, dry place.", "metadata": { "document_id": "doc-aaa8b734a101d7f34afb", "compression_input_rank": 9 }, "context_spans": [ { "start": 17, "end": 55 } ] } ], "details": { "mode": "sentences", "max_spans_per_source": 2, "input_ids": [ "passage-5bdc4916bcb2462d0250", "passage-a9bffaf7e82ff6d02b1e", "passage-fc31aa14f18cf8b2a89a", "passage-4be5e239b1ab47982a8e", "passage-1518103967d65de40d95", "passage-3657f06cb41040af19b1", "passage-6c50fe60990136f40ac3", "passage-a73c9da59294e4af482b", "passage-32a3368b3b75076e202c", "passage-c341c5c87a667038141c" ], "order": "candidate rank, then original character offset", "budget_policy": "final context selection keeps whole compressed source groups", "failure_policy": "fail", "state": "complete", "selections": { "passage-5bdc4916bcb2462d0250": [ { "start": 0, "end": 18 }, { "start": 18, "end": 75 } ], "passage-a9bffaf7e82ff6d02b1e": [ { "start": 0, "end": 31 } ], "passage-fc31aa14f18cf8b2a89a": [ { "start": 24, "end": 70 }, { "start": 70, "end": 98 } ], "passage-4be5e239b1ab47982a8e": [ { "start": 13, "end": 42 }, { "start": 61, "end": 98 } ], "passage-1518103967d65de40d95": [ { "start": 55, "end": 97 } ], "passage-3657f06cb41040af19b1": [ { "start": 9, "end": 60 } ], "passage-6c50fe60990136f40ac3": [ { "start": 9, "end": 61 } ], "passage-a73c9da59294e4af482b": [ { "start": 9, "end": 44 } ], "passage-32a3368b3b75076e202c": [ { "start": 17, "end": 55 } ], "passage-c341c5c87a667038141c": [] } } }, { "name": "context_selection", "elapsed_ms": 0.03204099994036369, "hits": [ { "source_unit_id": "passage-5bdc4916bcb2462d0250", "rank": 1, "score": 2.2885582447052, "evidence_ids": [], "preview": "## Reset pairing\n\n", "metadata": { "document_id": "doc-aaa8b734a101d7f34afb", "compression_input_rank": 1 }, "context_spans": [ { "start": 0, "end": 18 }, { "start": 18, "end": 75 } ] } ], "details": { "adapter": "unicode-codepoints-v1", "unit": "characters", "exact": true, "count": 211, "budget": null, "scope": "rendered_retrieval_context", "skipped": [], "policy": "ranked-whole-passages-v1" } } ] }, { "question_id": "Q2", "question": "How long is the warranty and what damage is excluded?", "hits": [ { "source_unit_id": "passage-4be5e239b1ab47982a8e", "rank": 1, "score": 3.4753642082214355, "evidence_ids": [], "preview": "The warranty lasts two years.", "metadata": { "document_id": "doc-aaa8b734a101d7f34afb", "compression_input_rank": 1 }, "context_spans": [ { "start": 13, "end": 42 }, { "start": 61, "end": 98 } ], "text": "The warranty lasts two years.", "heading_path": [ "Orion device manual", "Warranty" ], "source_path": "manual.md", "location": {}, "location_label": "Source text", "loader": "utf8-text-v1", "start": 240, "end": 269, "origin_source_id": "doc-aaa8b734a101d7f34afb", "line_start": 9, "line_end": 9, "context_part": 1 }, { "source_unit_id": "passage-4be5e239b1ab47982a8e", "rank": 1, "score": 3.4753642082214355, "evidence_ids": [], "preview": "The warranty lasts two years.", "metadata": { "document_id": "doc-aaa8b734a101d7f34afb", "compression_input_rank": 1 }, "context_spans": [ { "start": 13, "end": 42 }, { "start": 61, "end": 98 } ], "text": " Accidental water damage is excluded.", "heading_path": [ "Orion device manual", "Warranty" ], "source_path": "manual.md", "location": {}, "location_label": "Source text", "loader": "utf8-text-v1", "start": 288, "end": 325, "origin_source_id": "doc-aaa8b734a101d7f34afb", "line_start": 9, "line_end": 9, "context_part": 2 } ], "query_time_ms": 0.41197199971065857, "context_characters": 66, "context_hash": "8a28741798b7a689f8895cdece7005ec1fc4d6c5e9f34e980de8343fee74b89b", "observations": [], "metrics": { "question_id": "Q2", "split": "dev", "answerable": true, "tags": [], "required_evidence_ids": [ "EXCLUSION", "WARRANTY" ], "found_evidence_ids": [ "EXCLUSION", "WARRANTY" ], "missing_evidence_ids": [], "sufficient_evidence": true, "evidence_coverage": 1.0, "first_complete_evidence_rank": 1, "reciprocal_rank": 1.0, "precision_at_k": 1.0 }, "context_measurement": { "adapter": "unicode-codepoints-v1", "unit": "characters", "exact": true, "count": 192, "budget": null, "scope": "rendered_retrieval_context", "skipped": [], "policy": "ranked-whole-passages-v1" }, "stages": [ { "name": "retrieval", "elapsed_ms": 0.09342700286651962, "hits": [ { "source_unit_id": "passage-4be5e239b1ab47982a8e", "rank": 1, "score": 3.4753642082214355, "evidence_ids": [], "preview": "## Warranty\n\nThe warranty lasts two years. Keep your receipt. Accidental water damage is excluded. ", "metadata": { "document_id": "doc-aaa8b734a101d7f34afb" } }, { "source_unit_id": "passage-1518103967d65de40d95", "rank": 2, "score": 1.1101288795471191, "evidence_ids": [], "preview": "## Delivery\n\nStandard delivery takes five working days. Tracked delivery is available in Morocco. ", "metadata": { "document_id": "doc-aaa8b734a101d7f34afb" } }, { "source_unit_id": "passage-6c50fe60990136f40ac3", "rank": 3, "score": 0.7659269571304321, "evidence_ids": [], "preview": "excluded. Contact the support team before returning a device.\n\n", "metadata": { "document_id": "doc-aaa8b734a101d7f34afb" } }, { "source_unit_id": "passage-3657f06cb41040af19b1", "rank": 4, "score": 0.15764456987380981, "evidence_ids": [], "preview": " Morocco. Keep the tracking number until the parcel arrives.\n", "metadata": { "document_id": "doc-aaa8b734a101d7f34afb" } }, { "source_unit_id": "passage-a9bffaf7e82ff6d02b1e", "rank": 5, "score": 0.14540082216262817, "evidence_ids": [], "preview": "connected throughout the reset.\n\n", "metadata": { "document_id": "doc-aaa8b734a101d7f34afb" } }, { "source_unit_id": "passage-5bdc4916bcb2462d0250", "rank": 6, "score": 0.13330140709877014, "evidence_ids": [], "preview": "## Reset pairing\n\nHold the reset button for eight seconds to clear pairing. The status light ", "metadata": { "document_id": "doc-aaa8b734a101d7f34afb" } }, { "source_unit_id": "passage-fc31aa14f18cf8b2a89a", "rank": 7, "score": 0.09311925619840622, "evidence_ids": [], "preview": "tus light flashes twice. Disconnect the companion app before starting. Keep USB-C power connected ", "metadata": { "document_id": "doc-aaa8b734a101d7f34afb" } }, { "source_unit_id": "passage-a73c9da59294e4af482b", "rank": 8, "score": 0.08988720178604126, "evidence_ids": [], "preview": "n unused. Use only the supplied USB-C cable. Recycle batteries at an approved collection point.\n\n", "metadata": { "document_id": "doc-aaa8b734a101d7f34afb" } }, { "source_unit_id": "passage-32a3368b3b75076e202c", "rank": 9, "score": 0.08405247330665588, "evidence_ids": [], "preview": "## Battery care\n\nStore the device in a cool, dry place. Charge it every three months when unused. ", "metadata": { "document_id": "doc-aaa8b734a101d7f34afb" } }, { "source_unit_id": "passage-c341c5c87a667038141c", "rank": 10, "score": 0.0, "evidence_ids": [], "preview": "# Orion device manual\n\n", "metadata": { "document_id": "doc-aaa8b734a101d7f34afb" } } ], "details": { "query": "How long is the warranty and what damage is excluded?", "limit": 20, "index_cache": "not_applicable", "model_cache": "unknown", "runtime_versions": { "bm25s": "0.2.14", "numpy": "2.5.4" } } }, { "name": "compression", "elapsed_ms": 0.24530600057914853, "hits": [ { "source_unit_id": "passage-4be5e239b1ab47982a8e", "rank": 1, "score": 3.4753642082214355, "evidence_ids": [], "preview": "The warranty lasts two years.", "metadata": { "document_id": "doc-aaa8b734a101d7f34afb", "compression_input_rank": 1 }, "context_spans": [ { "start": 13, "end": 42 }, { "start": 61, "end": 98 } ] }, { "source_unit_id": "passage-1518103967d65de40d95", "rank": 2, "score": 1.1101288795471191, "evidence_ids": [], "preview": " Tracked delivery is available in Morocco.", "metadata": { "document_id": "doc-aaa8b734a101d7f34afb", "compression_input_rank": 2 }, "context_spans": [ { "start": 55, "end": 97 } ] }, { "source_unit_id": "passage-6c50fe60990136f40ac3", "rank": 3, "score": 0.7659269571304321, "evidence_ids": [], "preview": "excluded.", "metadata": { "document_id": "doc-aaa8b734a101d7f34afb", "compression_input_rank": 3 }, "context_spans": [ { "start": 0, "end": 9 }, { "start": 9, "end": 61 } ] }, { "source_unit_id": "passage-3657f06cb41040af19b1", "rank": 4, "score": 0.15764456987380981, "evidence_ids": [], "preview": " Keep the tracking number until the parcel arrives.", "metadata": { "document_id": "doc-aaa8b734a101d7f34afb", "compression_input_rank": 4 }, "context_spans": [ { "start": 9, "end": 60 } ] }, { "source_unit_id": "passage-a9bffaf7e82ff6d02b1e", "rank": 5, "score": 0.14540082216262817, "evidence_ids": [], "preview": "connected throughout the reset.", "metadata": { "document_id": "doc-aaa8b734a101d7f34afb", "compression_input_rank": 5 }, "context_spans": [ { "start": 0, "end": 31 } ] }, { "source_unit_id": "passage-5bdc4916bcb2462d0250", "rank": 6, "score": 0.13330140709877014, "evidence_ids": [], "preview": "Hold the reset button for eight seconds to clear pairing.", "metadata": { "document_id": "doc-aaa8b734a101d7f34afb", "compression_input_rank": 6 }, "context_spans": [ { "start": 18, "end": 75 }, { "start": 75, "end": 93 } ] }, { "source_unit_id": "passage-fc31aa14f18cf8b2a89a", "rank": 7, "score": 0.09311925619840622, "evidence_ids": [], "preview": " Disconnect the companion app before starting.", "metadata": { "document_id": "doc-aaa8b734a101d7f34afb", "compression_input_rank": 7 }, "context_spans": [ { "start": 24, "end": 70 } ] }, { "source_unit_id": "passage-a73c9da59294e4af482b", "rank": 8, "score": 0.08988720178604126, "evidence_ids": [], "preview": " Use only the supplied USB-C cable.", "metadata": { "document_id": "doc-aaa8b734a101d7f34afb", "compression_input_rank": 8 }, "context_spans": [ { "start": 9, "end": 44 } ] }, { "source_unit_id": "passage-32a3368b3b75076e202c", "rank": 9, "score": 0.08405247330665588, "evidence_ids": [], "preview": "Store the device in a cool, dry place.", "metadata": { "document_id": "doc-aaa8b734a101d7f34afb", "compression_input_rank": 9 }, "context_spans": [ { "start": 17, "end": 55 } ] } ], "details": { "mode": "sentences", "max_spans_per_source": 2, "input_ids": [ "passage-4be5e239b1ab47982a8e", "passage-1518103967d65de40d95", "passage-6c50fe60990136f40ac3", "passage-3657f06cb41040af19b1", "passage-a9bffaf7e82ff6d02b1e", "passage-5bdc4916bcb2462d0250", "passage-fc31aa14f18cf8b2a89a", "passage-a73c9da59294e4af482b", "passage-32a3368b3b75076e202c", "passage-c341c5c87a667038141c" ], "order": "candidate rank, then original character offset", "budget_policy": "final context selection keeps whole compressed source groups", "failure_policy": "fail", "state": "complete", "selections": { "passage-4be5e239b1ab47982a8e": [ { "start": 13, "end": 42 }, { "start": 61, "end": 98 } ], "passage-1518103967d65de40d95": [ { "start": 55, "end": 97 } ], "passage-6c50fe60990136f40ac3": [ { "start": 0, "end": 9 }, { "start": 9, "end": 61 } ], "passage-3657f06cb41040af19b1": [ { "start": 9, "end": 60 } ], "passage-a9bffaf7e82ff6d02b1e": [ { "start": 0, "end": 31 } ], "passage-5bdc4916bcb2462d0250": [ { "start": 18, "end": 75 }, { "start": 75, "end": 93 } ], "passage-fc31aa14f18cf8b2a89a": [ { "start": 24, "end": 70 } ], "passage-a73c9da59294e4af482b": [ { "start": 9, "end": 44 } ], "passage-32a3368b3b75076e202c": [ { "start": 17, "end": 55 } ], "passage-c341c5c87a667038141c": [] } } }, { "name": "context_selection", "elapsed_ms": 0.026279998564859852, "hits": [ { "source_unit_id": "passage-4be5e239b1ab47982a8e", "rank": 1, "score": 3.4753642082214355, "evidence_ids": [], "preview": "The warranty lasts two years.", "metadata": { "document_id": "doc-aaa8b734a101d7f34afb", "compression_input_rank": 1 }, "context_spans": [ { "start": 13, "end": 42 }, { "start": 61, "end": 98 } ] } ], "details": { "adapter": "unicode-codepoints-v1", "unit": "characters", "exact": true, "count": 192, "budget": null, "scope": "rendered_retrieval_context", "skipped": [], "policy": "ranked-whole-passages-v1" } } ] }, { "question_id": "Q3", "question": "How often should I charge an unused device?", "hits": [ { "source_unit_id": "passage-32a3368b3b75076e202c", "rank": 1, "score": 1.5058517456054688, "evidence_ids": [], "preview": "Store the device in a cool, dry place.", "metadata": { "document_id": "doc-aaa8b734a101d7f34afb", "compression_input_rank": 1 }, "context_spans": [ { "start": 17, "end": 55 }, { "start": 55, "end": 97 } ], "text": "Store the device in a cool, dry place.", "heading_path": [ "Orion device manual", "Battery care" ], "source_path": "manual.md", "location": {}, "location_label": "Source text", "loader": "utf8-text-v1", "start": 396, "end": 434, "origin_source_id": "doc-aaa8b734a101d7f34afb", "line_start": 13, "line_end": 13, "context_part": 1 }, { "source_unit_id": "passage-32a3368b3b75076e202c", "rank": 1, "score": 1.5058517456054688, "evidence_ids": [], "preview": "Store the device in a cool, dry place.", "metadata": { "document_id": "doc-aaa8b734a101d7f34afb", "compression_input_rank": 1 }, "context_spans": [ { "start": 17, "end": 55 }, { "start": 55, "end": 97 } ], "text": " Charge it every three months when unused.", "heading_path": [ "Orion device manual", "Battery care" ], "source_path": "manual.md", "location": {}, "location_label": "Source text", "loader": "utf8-text-v1", "start": 434, "end": 476, "origin_source_id": "doc-aaa8b734a101d7f34afb", "line_start": 13, "line_end": 13, "context_part": 2 } ], "query_time_ms": 0.32620999991195276, "context_characters": 80, "context_hash": "1b19fd8fb3a391a16f76698e603b2c82984ff69139addd4edbd5c64470a82e6f", "observations": [], "metrics": { "question_id": "Q3", "split": "dev", "answerable": true, "tags": [], "required_evidence_ids": [ "CHARGE" ], "found_evidence_ids": [ "CHARGE" ], "missing_evidence_ids": [], "sufficient_evidence": true, "evidence_coverage": 1.0, "first_complete_evidence_rank": 1, "reciprocal_rank": 1.0, "precision_at_k": 1.0 }, "context_measurement": { "adapter": "unicode-codepoints-v1", "unit": "characters", "exact": true, "count": 214, "budget": null, "scope": "rendered_retrieval_context", "skipped": [], "policy": "ranked-whole-passages-v1" }, "stages": [ { "name": "retrieval", "elapsed_ms": 0.08580299981986172, "hits": [ { "source_unit_id": "passage-32a3368b3b75076e202c", "rank": 1, "score": 1.5058517456054688, "evidence_ids": [], "preview": "## Battery care\n\nStore the device in a cool, dry place. Charge it every three months when unused. ", "metadata": { "document_id": "doc-aaa8b734a101d7f34afb" } }, { "source_unit_id": "passage-a73c9da59294e4af482b", "rank": 2, "score": 1.2111560106277466, "evidence_ids": [], "preview": "n unused. Use only the supplied USB-C cable. Recycle batteries at an approved collection point.\n\n", "metadata": { "document_id": "doc-aaa8b734a101d7f34afb" } }, { "source_unit_id": "passage-c341c5c87a667038141c", "rank": 3, "score": 0.6842033267021179, "evidence_ids": [], "preview": "# Orion device manual\n\n", "metadata": { "document_id": "doc-aaa8b734a101d7f34afb" } }, { "source_unit_id": "passage-6c50fe60990136f40ac3", "rank": 4, "score": 0.5042375326156616, "evidence_ids": [], "preview": "excluded. Contact the support team before returning a device.\n\n", "metadata": { "document_id": "doc-aaa8b734a101d7f34afb" } }, { "source_unit_id": "passage-5bdc4916bcb2462d0250", "rank": 5, "score": 0.0, "evidence_ids": [], "preview": "## Reset pairing\n\nHold the reset button for eight seconds to clear pairing. The status light ", "metadata": { "document_id": "doc-aaa8b734a101d7f34afb" } }, { "source_unit_id": "passage-fc31aa14f18cf8b2a89a", "rank": 6, "score": 0.0, "evidence_ids": [], "preview": "tus light flashes twice. Disconnect the companion app before starting. Keep USB-C power connected ", "metadata": { "document_id": "doc-aaa8b734a101d7f34afb" } }, { "source_unit_id": "passage-a9bffaf7e82ff6d02b1e", "rank": 7, "score": 0.0, "evidence_ids": [], "preview": "connected throughout the reset.\n\n", "metadata": { "document_id": "doc-aaa8b734a101d7f34afb" } }, { "source_unit_id": "passage-4be5e239b1ab47982a8e", "rank": 8, "score": 0.0, "evidence_ids": [], "preview": "## Warranty\n\nThe warranty lasts two years. Keep your receipt. Accidental water damage is excluded. ", "metadata": { "document_id": "doc-aaa8b734a101d7f34afb" } }, { "source_unit_id": "passage-1518103967d65de40d95", "rank": 9, "score": 0.0, "evidence_ids": [], "preview": "## Delivery\n\nStandard delivery takes five working days. Tracked delivery is available in Morocco. ", "metadata": { "document_id": "doc-aaa8b734a101d7f34afb" } }, { "source_unit_id": "passage-3657f06cb41040af19b1", "rank": 10, "score": 0.0, "evidence_ids": [], "preview": " Morocco. Keep the tracking number until the parcel arrives.\n", "metadata": { "document_id": "doc-aaa8b734a101d7f34afb" } } ], "details": { "query": "How often should I charge an unused device?", "limit": 20, "index_cache": "not_applicable", "model_cache": "unknown", "runtime_versions": { "bm25s": "0.2.14", "numpy": "2.5.4" } } }, { "name": "compression", "elapsed_ms": 0.1759549995767884, "hits": [ { "source_unit_id": "passage-32a3368b3b75076e202c", "rank": 1, "score": 1.5058517456054688, "evidence_ids": [], "preview": "Store the device in a cool, dry place.", "metadata": { "document_id": "doc-aaa8b734a101d7f34afb", "compression_input_rank": 1 }, "context_spans": [ { "start": 17, "end": 55 }, { "start": 55, "end": 97 } ] }, { "source_unit_id": "passage-a73c9da59294e4af482b", "rank": 2, "score": 1.2111560106277466, "evidence_ids": [], "preview": "n unused.", "metadata": { "document_id": "doc-aaa8b734a101d7f34afb", "compression_input_rank": 2 }, "context_spans": [ { "start": 0, "end": 9 }, { "start": 44, "end": 95 } ] }, { "source_unit_id": "passage-c341c5c87a667038141c", "rank": 3, "score": 0.6842033267021179, "evidence_ids": [], "preview": "# Orion device manual\n\n", "metadata": { "document_id": "doc-aaa8b734a101d7f34afb", "compression_input_rank": 3 }, "context_spans": [ { "start": 0, "end": 23 } ] }, { "source_unit_id": "passage-6c50fe60990136f40ac3", "rank": 4, "score": 0.5042375326156616, "evidence_ids": [], "preview": " Contact the support team before returning a device.", "metadata": { "document_id": "doc-aaa8b734a101d7f34afb", "compression_input_rank": 4 }, "context_spans": [ { "start": 9, "end": 61 } ] } ], "details": { "mode": "sentences", "max_spans_per_source": 2, "input_ids": [ "passage-32a3368b3b75076e202c", "passage-a73c9da59294e4af482b", "passage-c341c5c87a667038141c", "passage-6c50fe60990136f40ac3", "passage-5bdc4916bcb2462d0250", "passage-fc31aa14f18cf8b2a89a", "passage-a9bffaf7e82ff6d02b1e", "passage-4be5e239b1ab47982a8e", "passage-1518103967d65de40d95", "passage-3657f06cb41040af19b1" ], "order": "candidate rank, then original character offset", "budget_policy": "final context selection keeps whole compressed source groups", "failure_policy": "fail", "state": "complete", "selections": { "passage-32a3368b3b75076e202c": [ { "start": 17, "end": 55 }, { "start": 55, "end": 97 } ], "passage-a73c9da59294e4af482b": [ { "start": 0, "end": 9 }, { "start": 44, "end": 95 } ], "passage-c341c5c87a667038141c": [ { "start": 0, "end": 23 } ], "passage-6c50fe60990136f40ac3": [ { "start": 9, "end": 61 } ], "passage-5bdc4916bcb2462d0250": [], "passage-fc31aa14f18cf8b2a89a": [], "passage-a9bffaf7e82ff6d02b1e": [], "passage-4be5e239b1ab47982a8e": [], "passage-1518103967d65de40d95": [], "passage-3657f06cb41040af19b1": [] } } }, { "name": "context_selection", "elapsed_ms": 0.02223199771833606, "hits": [ { "source_unit_id": "passage-32a3368b3b75076e202c", "rank": 1, "score": 1.5058517456054688, "evidence_ids": [], "preview": "Store the device in a cool, dry place.", "metadata": { "document_id": "doc-aaa8b734a101d7f34afb", "compression_input_rank": 1 }, "context_spans": [ { "start": 17, "end": 55 }, { "start": 55, "end": 97 } ] } ], "details": { "adapter": "unicode-codepoints-v1", "unit": "characters", "exact": true, "count": 214, "budget": null, "scope": "rendered_retrieval_context", "skipped": [], "policy": "ranked-whole-passages-v1" } } ] }, { "question_id": "Q4", "question": "Where is tracked delivery available?", "hits": [ { "source_unit_id": "passage-1518103967d65de40d95", "rank": 1, "score": 3.3280820846557617, "evidence_ids": [], "preview": "## Delivery\n\n", "metadata": { "document_id": "doc-aaa8b734a101d7f34afb", "compression_input_rank": 1 }, "context_spans": [ { "start": 0, "end": 13 }, { "start": 55, "end": 97 } ], "text": "## Delivery\n\n", "heading_path": [ "Orion device manual", "Delivery" ], "source_path": "manual.md", "location": {}, "location_label": "Source text", "loader": "utf8-text-v1", "start": 564, "end": 577, "origin_source_id": "doc-aaa8b734a101d7f34afb", "line_start": 15, "line_end": 16, "context_part": 1 }, { "source_unit_id": "passage-1518103967d65de40d95", "rank": 1, "score": 3.3280820846557617, "evidence_ids": [], "preview": "## Delivery\n\n", "metadata": { "document_id": "doc-aaa8b734a101d7f34afb", "compression_input_rank": 1 }, "context_spans": [ { "start": 0, "end": 13 }, { "start": 55, "end": 97 } ], "text": " Tracked delivery is available in Morocco.", "heading_path": [ "Orion device manual", "Delivery" ], "source_path": "manual.md", "location": {}, "location_label": "Source text", "loader": "utf8-text-v1", "start": 619, "end": 661, "origin_source_id": "doc-aaa8b734a101d7f34afb", "line_start": 17, "line_end": 17, "context_part": 2 } ], "query_time_ms": 0.4942289997416083, "context_characters": 55, "context_hash": "18aa5af5448804e043abe7858f6f406fd7039c5d1582235b870c9c42bc93d7a5", "observations": [], "metrics": { "question_id": "Q4", "split": "dev", "answerable": true, "tags": [], "required_evidence_ids": [ "DELIVERY" ], "found_evidence_ids": [ "DELIVERY" ], "missing_evidence_ids": [], "sufficient_evidence": true, "evidence_coverage": 1.0, "first_complete_evidence_rank": 1, "reciprocal_rank": 1.0, "precision_at_k": 1.0 }, "context_measurement": { "adapter": "unicode-codepoints-v1", "unit": "characters", "exact": true, "count": 181, "budget": null, "scope": "rendered_retrieval_context", "skipped": [], "policy": "ranked-whole-passages-v1" }, "stages": [ { "name": "retrieval", "elapsed_ms": 0.130978998640785, "hits": [ { "source_unit_id": "passage-1518103967d65de40d95", "rank": 1, "score": 3.3280820846557617, "evidence_ids": [], "preview": "## Delivery\n\nStandard delivery takes five working days. Tracked delivery is available in Morocco. ", "metadata": { "document_id": "doc-aaa8b734a101d7f34afb" } }, { "source_unit_id": "passage-4be5e239b1ab47982a8e", "rank": 2, "score": 0.5351060628890991, "evidence_ids": [], "preview": "## Warranty\n\nThe warranty lasts two years. Keep your receipt. Accidental water damage is excluded. ", "metadata": { "document_id": "doc-aaa8b734a101d7f34afb" } }, { "source_unit_id": "passage-c341c5c87a667038141c", "rank": 3, "score": 0.0, "evidence_ids": [], "preview": "# Orion device manual\n\n", "metadata": { "document_id": "doc-aaa8b734a101d7f34afb" } }, { "source_unit_id": "passage-5bdc4916bcb2462d0250", "rank": 4, "score": 0.0, "evidence_ids": [], "preview": "## Reset pairing\n\nHold the reset button for eight seconds to clear pairing. The status light ", "metadata": { "document_id": "doc-aaa8b734a101d7f34afb" } }, { "source_unit_id": "passage-fc31aa14f18cf8b2a89a", "rank": 5, "score": 0.0, "evidence_ids": [], "preview": "tus light flashes twice. Disconnect the companion app before starting. Keep USB-C power connected ", "metadata": { "document_id": "doc-aaa8b734a101d7f34afb" } }, { "source_unit_id": "passage-a9bffaf7e82ff6d02b1e", "rank": 6, "score": 0.0, "evidence_ids": [], "preview": "connected throughout the reset.\n\n", "metadata": { "document_id": "doc-aaa8b734a101d7f34afb" } }, { "source_unit_id": "passage-6c50fe60990136f40ac3", "rank": 7, "score": 0.0, "evidence_ids": [], "preview": "excluded. Contact the support team before returning a device.\n\n", "metadata": { "document_id": "doc-aaa8b734a101d7f34afb" } }, { "source_unit_id": "passage-32a3368b3b75076e202c", "rank": 8, "score": 0.0, "evidence_ids": [], "preview": "## Battery care\n\nStore the device in a cool, dry place. Charge it every three months when unused. ", "metadata": { "document_id": "doc-aaa8b734a101d7f34afb" } }, { "source_unit_id": "passage-a73c9da59294e4af482b", "rank": 9, "score": 0.0, "evidence_ids": [], "preview": "n unused. Use only the supplied USB-C cable. Recycle batteries at an approved collection point.\n\n", "metadata": { "document_id": "doc-aaa8b734a101d7f34afb" } }, { "source_unit_id": "passage-3657f06cb41040af19b1", "rank": 10, "score": 0.0, "evidence_ids": [], "preview": " Morocco. Keep the tracking number until the parcel arrives.\n", "metadata": { "document_id": "doc-aaa8b734a101d7f34afb" } } ], "details": { "query": "Where is tracked delivery available?", "limit": 20, "index_cache": "not_applicable", "model_cache": "unknown", "runtime_versions": { "bm25s": "0.2.14", "numpy": "2.5.4" } } }, { "name": "compression", "elapsed_ms": 0.16995300029520877, "hits": [ { "source_unit_id": "passage-1518103967d65de40d95", "rank": 1, "score": 3.3280820846557617, "evidence_ids": [], "preview": "## Delivery\n\n", "metadata": { "document_id": "doc-aaa8b734a101d7f34afb", "compression_input_rank": 1 }, "context_spans": [ { "start": 0, "end": 13 }, { "start": 55, "end": 97 } ] }, { "source_unit_id": "passage-4be5e239b1ab47982a8e", "rank": 2, "score": 0.5351060628890991, "evidence_ids": [], "preview": " Accidental water damage is excluded.", "metadata": { "document_id": "doc-aaa8b734a101d7f34afb", "compression_input_rank": 2 }, "context_spans": [ { "start": 61, "end": 98 } ] } ], "details": { "mode": "sentences", "max_spans_per_source": 2, "input_ids": [ "passage-1518103967d65de40d95", "passage-4be5e239b1ab47982a8e", "passage-c341c5c87a667038141c", "passage-5bdc4916bcb2462d0250", "passage-fc31aa14f18cf8b2a89a", "passage-a9bffaf7e82ff6d02b1e", "passage-6c50fe60990136f40ac3", "passage-32a3368b3b75076e202c", "passage-a73c9da59294e4af482b", "passage-3657f06cb41040af19b1" ], "order": "candidate rank, then original character offset", "budget_policy": "final context selection keeps whole compressed source groups", "failure_policy": "fail", "state": "complete", "selections": { "passage-1518103967d65de40d95": [ { "start": 0, "end": 13 }, { "start": 55, "end": 97 } ], "passage-4be5e239b1ab47982a8e": [ { "start": 61, "end": 98 } ], "passage-c341c5c87a667038141c": [], "passage-5bdc4916bcb2462d0250": [], "passage-fc31aa14f18cf8b2a89a": [], "passage-a9bffaf7e82ff6d02b1e": [], "passage-6c50fe60990136f40ac3": [], "passage-32a3368b3b75076e202c": [], "passage-a73c9da59294e4af482b": [], "passage-3657f06cb41040af19b1": [] } } }, { "name": "context_selection", "elapsed_ms": 0.02344399763387628, "hits": [ { "source_unit_id": "passage-1518103967d65de40d95", "rank": 1, "score": 3.3280820846557617, "evidence_ids": [], "preview": "## Delivery\n\n", "metadata": { "document_id": "doc-aaa8b734a101d7f34afb", "compression_input_rank": 1 }, "context_spans": [ { "start": 0, "end": 13 }, { "start": 55, "end": 97 } ] } ], "details": { "adapter": "unicode-codepoints-v1", "unit": "characters", "exact": true, "count": 181, "budget": null, "scope": "rendered_retrieval_context", "skipped": [], "policy": "ranked-whole-passages-v1" } } ] } ], "observations": [ "Reviewed evidence is scored against the union of exact original-text spans.", "Extractive compression can remove required facts or qualifications; evidence is rescored from only the returned spans." ], "limitations": [ "Answer correctness, citations, abstention and generation request tokens are not evaluated.", "Retriever scores describe this ranking; they are not probabilities of correctness.", "Chunk sizes count characters. Rendered context counts characters unless an explicit local tokenizer is identified; model request overhead is excluded.", "Character and text-line offsets refer to the extracted page/record/section, not byte offsets in the original file. Extraction does not verify layout fidelity." ] }, "resources": { "query_ms": { "mean_known": 0.4523267498370842, "known_count": 4, "missing_count": 0 }, "runtime_versions": { "bm25s": "0.2.14", "numpy": "2.5.4" }, "context_units": { "mean_known": 199.5, "known_count": 4, "missing_count": 0 }, "context_characters": { "mean_known": 69.0, "known_count": 4, "missing_count": 0 }, "context_unit": "characters", "index_ms": 0.5941989984421525, "vector_cache": "not_applicable", "model_cache": "unknown" }, "usage": { "source": "user_supplied_usage", "scope": "supplied_totals_not_inferred_from_repeats", "estimate": true, "cost": null, "currency": null, "pricing_identity": null, "input_tokens": null, "output_tokens": null }, "trials": [ { "trial": 0, "reused": false, "artifact_directory": "jobs/compress/trial-0/attempt-20261019T152652-fddca4a4d309", "resources": { "query_ms": { "mean_known": 0.4523267498370842, "known_count": 4, "missing_count": 0 }, "runtime_versions": { "bm25s": "0.2.14", "numpy": "2.5.4" }, "context_units": { "mean_known": 199.5, "known_count": 4, "missing_count": 0 }, "context_characters": { "mean_known": 69.0, "known_count": 4, "missing_count": 0 }, "context_unit": "characters", "index_ms": 0.5941989984421525, "vector_cache": "not_applicable", "model_cache": "unknown" } } ], "repeat_rankings_changed": false, "evidence_scope": "exact_returned_spans", "summaries": { "dev": { "all": { "question_count": 4, "answerable_questions": 4, "sufficient_count": 3, "sufficiency_rate": 0.75, "mean_evidence_coverage": 0.875, "mean_reciprocal_rank": 1.0, "mean_precision_at_k": 1.0 } }, "heldout": { "all": { "question_count": 0, "answerable_questions": 0, "sufficient_count": 0, "sufficiency_rate": null, "mean_evidence_coverage": null, "mean_reciprocal_rank": null, "mean_precision_at_k": null } } }, "split_resources": { "dev": { "query_ms": { "mean_known": 0.4523267498370842, "known_count": 4, "missing_count": 0 }, "runtime_versions": { "bm25s": "0.2.14", "numpy": "2.5.4" }, "context_units": { "mean_known": 199.5, "known_count": 4, "missing_count": 0 }, "context_characters": { "mean_known": 69.0, "known_count": 4, "missing_count": 0 }, "context_unit": "characters", "index_ms": 0.5941989984421525, "vector_cache": "not_applicable", "model_cache": "unknown" }, "heldout": { "query_ms": { "mean_known": null, "known_count": 0, "missing_count": 0 }, "runtime_versions": {}, "context_units": { "mean_known": null, "known_count": 0, "missing_count": 0 }, "context_characters": { "mean_known": null, "known_count": 0, "missing_count": 0 }, "context_unit": null, "index_ms": 0.5941989984421525, "vector_cache": "unavailable", "model_cache": "unknown" } }, "criteria": { "dev": { "status": "not_requested", "checks": [] }, "heldout": { "status": "not_requested", "checks": [] } } } ], "comparisons": [ { "baseline": "bm25", "participant": "parent-child", "quality_comparable": true, "scope": "exact_returned_spans", "questions": [ { "question_id": "Q1", "split": "dev", "tags": [], "answerable": true, "change": "improved", "gained": [ "POWER" ], "lost": [], "missing": [], "deltas": { "sufficiency_rate": 1.0, "mean_evidence_coverage": 0.5, "mean_reciprocal_rank": 0.0, "mean_precision_at_k": 0.0 } }, { "question_id": "Q2", "split": "dev", "tags": [], "answerable": true, "change": "stable_pass", "gained": [], "lost": [], "missing": [], "deltas": { "sufficiency_rate": 0.0, "mean_evidence_coverage": 0.0, "mean_reciprocal_rank": 0.0, "mean_precision_at_k": 0.0 } }, { "question_id": "Q3", "split": "dev", "tags": [], "answerable": true, "change": "stable_pass", "gained": [], "lost": [], "missing": [], "deltas": { "sufficiency_rate": 0.0, "mean_evidence_coverage": 0.0, "mean_reciprocal_rank": 0.0, "mean_precision_at_k": 0.0 } }, { "question_id": "Q4", "split": "dev", "tags": [], "answerable": true, "change": "stable_pass", "gained": [], "lost": [], "missing": [], "deltas": { "sufficiency_rate": 0.0, "mean_evidence_coverage": 0.0, "mean_reciprocal_rank": 0.0, "mean_precision_at_k": 0.0 } } ], "splits": { "dev": { "all": { "answerable_questions": 4, "changes": { "improved": 1, "regressed": 0, "stable_pass": 3, "stable_fail": 0 }, "intervals": { "sufficiency_rate": { "estimate": 0.25, "question_count": 4, "group_count": 4, "lower": 0.0, "upper": 0.75, "seed": 42, "samples": 2000, "method": "paired-cluster-percentile-bootstrap-v1", "numpy_version": "2.5.4", "confidence": 0.95, "note": "Small samples and reused public heldout data limit inference" }, "mean_evidence_coverage": { "estimate": 0.125, "question_count": 4, "group_count": 4, "lower": 0.0, "upper": 0.375, "seed": 42, "samples": 2000, "method": "paired-cluster-percentile-bootstrap-v1", "numpy_version": "2.5.4", "confidence": 0.95, "note": "Small samples and reused public heldout data limit inference" }, "mean_reciprocal_rank": { "estimate": 0.0, "question_count": 4, "group_count": 4, "lower": 0.0, "upper": 0.0, "seed": 42, "samples": 2000, "method": "paired-cluster-percentile-bootstrap-v1", "numpy_version": "2.5.4", "confidence": 0.95, "note": "Small samples and reused public heldout data limit inference" }, "mean_precision_at_k": { "estimate": 0.0, "question_count": 4, "group_count": 4, "lower": 0.0, "upper": 0.0, "seed": 42, "samples": 2000, "method": "paired-cluster-percentile-bootstrap-v1", "numpy_version": "2.5.4", "confidence": 0.95, "note": "Small samples and reused public heldout data limit inference" } } } }, "heldout": { "all": { "answerable_questions": 0, "changes": { "improved": 0, "regressed": 0, "stable_pass": 0, "stable_fail": 0 }, "intervals": { "sufficiency_rate": { "estimate": null, "question_count": 0, "group_count": 0, "lower": null, "upper": null, "seed": 42, "samples": 2000, "method": "paired-cluster-percentile-bootstrap-v1", "numpy_version": "2.5.4", "confidence": 0.95, "note": "No interval: fewer than two independent declared groups" }, "mean_evidence_coverage": { "estimate": null, "question_count": 0, "group_count": 0, "lower": null, "upper": null, "seed": 42, "samples": 2000, "method": "paired-cluster-percentile-bootstrap-v1", "numpy_version": "2.5.4", "confidence": 0.95, "note": "No interval: fewer than two independent declared groups" }, "mean_reciprocal_rank": { "estimate": null, "question_count": 0, "group_count": 0, "lower": null, "upper": null, "seed": 42, "samples": 2000, "method": "paired-cluster-percentile-bootstrap-v1", "numpy_version": "2.5.4", "confidence": 0.95, "note": "No interval: fewer than two independent declared groups" }, "mean_precision_at_k": { "estimate": null, "question_count": 0, "group_count": 0, "lower": null, "upper": null, "seed": 42, "samples": 2000, "method": "paired-cluster-percentile-bootstrap-v1", "numpy_version": "2.5.4", "confidence": 0.95, "note": "No interval: fewer than two independent declared groups" } } } } }, "interpretation": "Observed gains/losses under the declared policy; no automatic winner" }, { "baseline": "bm25", "participant": "compress", "quality_comparable": true, "scope": "exact_returned_spans", "questions": [ { "question_id": "Q1", "split": "dev", "tags": [], "answerable": true, "change": "stable_fail", "gained": [], "lost": [], "missing": [ "POWER" ], "deltas": { "sufficiency_rate": 0.0, "mean_evidence_coverage": 0.0, "mean_reciprocal_rank": 0.0, "mean_precision_at_k": 0.0 } }, { "question_id": "Q2", "split": "dev", "tags": [], "answerable": true, "change": "stable_pass", "gained": [], "lost": [], "missing": [], "deltas": { "sufficiency_rate": 0.0, "mean_evidence_coverage": 0.0, "mean_reciprocal_rank": 0.0, "mean_precision_at_k": 0.0 } }, { "question_id": "Q3", "split": "dev", "tags": [], "answerable": true, "change": "stable_pass", "gained": [], "lost": [], "missing": [], "deltas": { "sufficiency_rate": 0.0, "mean_evidence_coverage": 0.0, "mean_reciprocal_rank": 0.0, "mean_precision_at_k": 0.0 } }, { "question_id": "Q4", "split": "dev", "tags": [], "answerable": true, "change": "stable_pass", "gained": [], "lost": [], "missing": [], "deltas": { "sufficiency_rate": 0.0, "mean_evidence_coverage": 0.0, "mean_reciprocal_rank": 0.0, "mean_precision_at_k": 0.0 } } ], "splits": { "dev": { "all": { "answerable_questions": 4, "changes": { "improved": 0, "regressed": 0, "stable_pass": 3, "stable_fail": 1 }, "intervals": { "sufficiency_rate": { "estimate": 0.0, "question_count": 4, "group_count": 4, "lower": 0.0, "upper": 0.0, "seed": 42, "samples": 2000, "method": "paired-cluster-percentile-bootstrap-v1", "numpy_version": "2.5.4", "confidence": 0.95, "note": "Small samples and reused public heldout data limit inference" }, "mean_evidence_coverage": { "estimate": 0.0, "question_count": 4, "group_count": 4, "lower": 0.0, "upper": 0.0, "seed": 42, "samples": 2000, "method": "paired-cluster-percentile-bootstrap-v1", "numpy_version": "2.5.4", "confidence": 0.95, "note": "Small samples and reused public heldout data limit inference" }, "mean_reciprocal_rank": { "estimate": 0.0, "question_count": 4, "group_count": 4, "lower": 0.0, "upper": 0.0, "seed": 42, "samples": 2000, "method": "paired-cluster-percentile-bootstrap-v1", "numpy_version": "2.5.4", "confidence": 0.95, "note": "Small samples and reused public heldout data limit inference" }, "mean_precision_at_k": { "estimate": 0.0, "question_count": 4, "group_count": 4, "lower": 0.0, "upper": 0.0, "seed": 42, "samples": 2000, "method": "paired-cluster-percentile-bootstrap-v1", "numpy_version": "2.5.4", "confidence": 0.95, "note": "Small samples and reused public heldout data limit inference" } } } }, "heldout": { "all": { "answerable_questions": 0, "changes": { "improved": 0, "regressed": 0, "stable_pass": 0, "stable_fail": 0 }, "intervals": { "sufficiency_rate": { "estimate": null, "question_count": 0, "group_count": 0, "lower": null, "upper": null, "seed": 42, "samples": 2000, "method": "paired-cluster-percentile-bootstrap-v1", "numpy_version": "2.5.4", "confidence": 0.95, "note": "No interval: fewer than two independent declared groups" }, "mean_evidence_coverage": { "estimate": null, "question_count": 0, "group_count": 0, "lower": null, "upper": null, "seed": 42, "samples": 2000, "method": "paired-cluster-percentile-bootstrap-v1", "numpy_version": "2.5.4", "confidence": 0.95, "note": "No interval: fewer than two independent declared groups" }, "mean_reciprocal_rank": { "estimate": null, "question_count": 0, "group_count": 0, "lower": null, "upper": null, "seed": 42, "samples": 2000, "method": "paired-cluster-percentile-bootstrap-v1", "numpy_version": "2.5.4", "confidence": 0.95, "note": "No interval: fewer than two independent declared groups" }, "mean_precision_at_k": { "estimate": null, "question_count": 0, "group_count": 0, "lower": null, "upper": null, "seed": 42, "samples": 2000, "method": "paired-cluster-percentile-bootstrap-v1", "numpy_version": "2.5.4", "confidence": 0.95, "note": "No interval: fewer than two independent declared groups" } } } } }, "interpretation": "Observed gains/losses under the declared policy; no automatic winner" } ], "observations": [ "Keeping the baseline is valid. Choose using these cases and your requirements.", "Equal k does not imply equal context volume; resources are reported separately.", "Intervals pair questions or families. Repeated timings are not extra samples.", "These observations do not prove what caused a particular failure." ], "limitations": [ "Grids are exploratory. Intervals are not adjusted for multiple comparisons.", "Heldout reuse is tracked under this output root, without protection against edits.", "Vector reuse is observed. Model/OS cache warmth and external traces are unknown.", "Missing timing/cost is unavailable. Supplied usage/pricing yields labeled estimates.", "nDCG requires graded relevance gains. Answer quality is not evaluated." ] }