{ "bm25_reference": { "ndcg@10": 0.226, "recall@100": 0.572, "source": "MIRACL paper (TACL 2023), Table 5, full corpus" }, "checks": { "determinism": { "bge-v2-m3": { "batch_size": 32, "depth": 100, "difference_points": 0.0, "identical_rankings": 305, "n_queries": 305, "ndcg@10_first": 0.36008335712152306, "ndcg@10_repeat": 0.36008335712152306, "passed": true, "run_id": "quality-determinism-bge-v2-m3-20261010T140311Z-331ebf", "stage": "hybrid-71k", "tolerance_points": 0.1 }, "jina-v3.5": { "batch_size": null, "depth": 100, "difference_points": 0.0, "identical_rankings": 305, "n_queries": 305, "ndcg@10_first": 0.37660312839974847, "ndcg@10_repeat": 0.37660312839974847, "passed": true, "run_id": "quality-determinism-jina-v3.5-20261010T142422Z-07a25e", "stage": "hybrid-71k", "tolerance_points": 0.1 }, "qwen3-0.6b": { "batch_size": 8, "depth": 100, "difference_points": 0.0, "identical_rankings": 305, "n_queries": 305, "ndcg@10_first": 0.3646704503914691, "ndcg@10_repeat": 0.3646704503914691, "passed": true, "run_id": "quality-determinism-qwen3-0.6b-20261010T140745Z-5a7df0", "stage": "hybrid-71k", "tolerance_points": 0.1 } }, "dtype": { "bge-v2-m3": { "batch_size_gpu": 32, "difference_points": 0.0, "gpu_value_used_for_cpu": true, "identical_rankings": 1, "n_queries": 20, "ndcg@10_cpu_fp32": 0.7610539990914337, "ndcg@10_gpu": 0.7610539990914337, "run_id": "quality-dtype-bge-v2-m3-20261011T065250Z-875f4f", "stage": "hybrid-71k", "tolerance_points": 0.5 }, "gte-base": { "batch_size_gpu": 32, "difference_points": 0.0, "gpu_value_used_for_cpu": true, "identical_rankings": 2, "n_queries": 20, "ndcg@10_cpu_fp32": 0.6865317842440822, "ndcg@10_gpu": 0.6865317842440822, "run_id": "quality-dtype-gte-base-20261011T064302Z-16d7c9", "stage": "hybrid-71k", "tolerance_points": 0.5 }, "jina-v3.5": { "batch_size_gpu": null, "difference_points": 0.12749845543877925, "gpu_value_used_for_cpu": true, "identical_rankings": 0, "n_queries": 20, "ndcg@10_cpu_fp32": 0.7711561073178305, "ndcg@10_gpu": 0.7724310918722183, "run_id": "quality-dtype-jina-v3.5-20261011T073653Z-f68b38", "stage": "hybrid-71k", "tolerance_points": 0.5 }, "qwen3-0.6b": { "batch_size_gpu": 8, "difference_points": 0.0, "gpu_value_used_for_cpu": true, "identical_rankings": 0, "n_queries": 20, "ndcg@10_cpu_fp32": 0.7305128439846746, "ndcg@10_gpu": 0.7305128439846746, "run_id": "quality-dtype-qwen3-0.6b-20261011T081457Z-09da93", "stage": "hybrid-71k", "tolerance_points": 0.5 } }, "evaluation": { "difference": 0.0, "n_queries": 305, "ours_ndcg@10": 0.17498289422243518, "passed": true, "pytrec_eval_ndcg@10": 0.17498289422243518, "run_id": "validation-evaluation-hybrid-71k-20261010T035449Z-c16a8b", "run_sha256": "3032e2cc871b0b82c144f6e5358876c3647988d331d47f910ab9e1fa982b1b8a", "stage": "hybrid-71k", "tolerance": 0.0001 }, "pointwise": { "voyage-rerank-3": { "depths": [ 20, 100 ], "max_abs_difference": 0.0, "n_queries": 5, "pointwise": true, "run_id": "validation-voyage-rerank-3-pointwise-20261010T035432Z-e3c788", "tolerance": 0.001 }, "voyage-rerank-3-lite": { "depths": [ 20, 100 ], "max_abs_difference": 0.0, "n_queries": 5, "pointwise": true, "run_id": "validation-voyage-rerank-3-lite-pointwise-20261010T035437Z-91c411", "tolerance": 0.001 } } }, "costs": { "checks": { "inputs": { "api_run_info": { "voyage-rerank-3": { "keyword-71k": "078d3ba24a1ae0b1936c21bc1807f83de37dc1ddc284874dc0281452f91ed452", "keyword-full": "8a26b977508cb4368ea5a1ca640ba6fbb0eae3f6f382ec9853d33e2df0257b12", "vector-71k": "5cba92a73de0d83cb4d51a811210416a890044b013b17b14fd2ab9834d9f9923" }, "voyage-rerank-3-lite": { "keyword-71k": "16c2a8c27d467339786a93216f4bab1d835398ba8f9d6d38e7899ffc1038ccf4", "keyword-full": "76439f492fb9ad212df7a6ee280df00d946b44095384c3c320ead3af354d5a5c", "vector-71k": "e96f67e9d05f78af30263899deaba5afa847df395d2ee4b33c0c846285d3ddae" } }, "first_stage_runs": { "hybrid-71k": "3032e2cc871b0b82c144f6e5358876c3647988d331d47f910ab9e1fa982b1b8a", "keyword-71k": "a03aeed0d2bc63e1c79953090984accf978ae6b9c974d02fd8431c5d72e2642a", "keyword-full": "69458380199a02c657b5b5231188fb4ed3a6e5792340ba332213ff04db6cbaf8", "vector-71k": "ef12845ce3afbdc8d17d906e6859eba9e234a7355f41899830d55a6d472a850c" }, "latency": { "bge-v2-m3-cuda-desktop-se": "latency-bge-v2-m3-cuda-desktop-se-20261011T043232Z-d00b64", "first-stage-hybrid-71k-desktop-se": "latency-first-stage-hybrid-71k-desktop-se-20261010T152234Z-80ffe8", "gte-base-cuda-desktop-se": "latency-gte-base-cuda-desktop-se-20261011T043900Z-c4aece", "jina-v3.5-cuda-desktop-se": "latency-jina-v3.5-cuda-desktop-se-20261011T041221Z-de1b7f", "mxbai-large-v2-cuda-desktop-se": "latency-mxbai-large-v2-cuda-desktop-se-20261011T030622Z-f837da", "querit-4b-cuda-desktop-se": "latency-querit-4b-cuda-desktop-se-20261011T001254Z-186e05", "qwen3-0.6b-cuda-desktop-se": "latency-qwen3-0.6b-cuda-desktop-se-20261011T085531Z-bfd2bd", "qwen3-4b-cuda-desktop-se": "latency-qwen3-4b-cuda-desktop-se-20261011T020021Z-aa5503", "voyage-rerank-3-api-desktop-se-afternoon": "latency-voyage-rerank-3-api-desktop-se-afternoon-20261011T100517Z-9099a1", "voyage-rerank-3-api-desktop-se-evening": "latency-voyage-rerank-3-api-desktop-se-evening-20261010T162902Z-f848c5", "voyage-rerank-3-api-desktop-se-morning": "latency-voyage-rerank-3-api-desktop-se-morning-20261011T063414Z-47493a", "voyage-rerank-3-lite-api-desktop-se-afternoon": "latency-voyage-rerank-3-lite-api-desktop-se-afternoon-20261011T100852Z-9066d4", "voyage-rerank-3-lite-api-desktop-se-evening": "latency-voyage-rerank-3-lite-api-desktop-se-evening-20261010T163244Z-facd01", "voyage-rerank-3-lite-api-desktop-se-morning": "latency-voyage-rerank-3-lite-api-desktop-se-morning-20261011T063750Z-d5f19f" }, "reranker_runs": { "hybrid-71k--bge-v2-m3": "bc0959664514e50a32696682f63dd437e2b7a1e59cc43b3a1c55c67d25ce9b31", "hybrid-71k--gte-base": "1cf6cad186e086b7e7d00249e884845f562febf23e345adf63401c41b3776320", "hybrid-71k--jina-v3.5": "c7910a55cac133dec5345383d1a96760db613e6986b74fd17df77eef78d77344", "hybrid-71k--mxbai-large-v2": "3fbe579073f2568b3dfa0cb891d85ff05374476606ffda00ac9510479b8dcc31", "hybrid-71k--querit-4b": "1d95196e560d0f067af56dde53498fe1cbb9c00dd154837fa7d607e4654e359c", "hybrid-71k--qwen3-0.6b": "a8d8786a0e034ae5a164afc4e22a15c6fc488a6d436286b67fa8c6325d2bb5bf", "hybrid-71k--qwen3-4b": "4a9458d131e44d4eb0b5f523f19e0dda48dcc55970b40e15641deee340208bc7", "hybrid-71k--voyage-rerank-3": "3e949aeb397e8a053376542666e591d5993eea5b5e8847b14e62b2a01fb64cc2", "hybrid-71k--voyage-rerank-3-lite": "03fe1dc6f8ca69149121c99041f3755e0ec59ae75378c8579916e705dfb0d809", "keyword-71k--bge-v2-m3": "58946ee5d98c7dbec37b8c83f5fb9da6b777799bac053bc398a7abdd65ffcf0f", "keyword-71k--gte-base": "d5bcb9f174ffc10c382a8004dced7a2fd9d2d2418285d5ebf757f18b8e99017c", "keyword-71k--jina-v3.5": "4fb39f2c415a380e314f33da59d96e5049a041eca1289dbfee7d99c59f6e36bc", "keyword-71k--mxbai-large-v2": "022572dd1a703819362027e28224a2474cd0e68b19e3ff8862b3b082ff8dc301", "keyword-71k--querit-4b": "875c9efc5d166172e0af96213a0c5b85e0f58e595aec76c147a2c0f52098efef", "keyword-71k--qwen3-0.6b": "45d0a510488a6676fd880278eca4751ceb9e38c1d00518a916ca1d0c90e9d25e", "keyword-71k--qwen3-4b": "6b77b343195d77f0b483234f07b62ad6e4d345a64f491df1718314d7ffd6bb96", "keyword-71k--voyage-rerank-3": "0d152a740dc8496a3c74775169ee6dc9fb035486ac5998398b8ce112a54fdb88", "keyword-71k--voyage-rerank-3-lite": "21f25d03dfb31ff7cee28d8f1f750aaaf5449829918e3370e5498ffced329ea4", "keyword-full--bge-v2-m3": "a4de05a7d199359656a92686ccd866c32c0b6e176cb935b4b02064c2a502e329", "keyword-full--gte-base": "1e19e8f947bf47bb8295595d59a7fc2d63a65944fa17aa9dfcdccc278fbe3714", "keyword-full--mxbai-large-v2": "eb5656eac01ce129aa660125a560e589390eb723a08a2489c2d50f0a98ff886c", "keyword-full--querit-4b": "b00e2def7a609bd8de313be8c154cf15f6765c1dd0ed3b8a0588547788bd5c14", "keyword-full--qwen3-0.6b": "f531df892644d9cbf4040b0298082d5269a6216aa477c20ed85f31a71937ce8d", "keyword-full--qwen3-4b": "ca03597659b4ce2c9399e8148c982b25230617ec3706815baad8148756c9d46d", "keyword-full--voyage-rerank-3": "79dce8b26fbe40258bbf1456d3e22290bc58a118acb3398ebb53809eb7fcf30d", "keyword-full--voyage-rerank-3-lite": "fa9a109cf5f0819671a3c1d23f3ad6a561ef9f93a034e54c0c74832188521d3c", "vector-71k--bge-v2-m3": "9d1b0a2cf8aed81e2d64ebb73110d29313a52a3a80703e0f12e1cb4889558edf", "vector-71k--gte-base": "1485207854146931ecf4de29077da81364499762eeb9b60328c5a8d5c163a1d4", "vector-71k--jina-v3.5": "4fc5abd57717d7af953bfb166e0aa05148384dbd721708f5f32687b3c51479f9", "vector-71k--mxbai-large-v2": "a45f2ffef2584bbdc3e46a2c777dce55df8eecd2cfbfaeee63a833915e4b560b", "vector-71k--querit-4b": "fd8c37435fda57a221b18217bc872a051e5a01cd41e9e41a7b93a22d8068b44e", "vector-71k--qwen3-0.6b": "2b0ddf7465bd930a575774428dc5b3e7d5961248d2aa9a5a8a5ba629ff9ba467", "vector-71k--qwen3-4b": "9c1e387eadb56ae1ce76b6ba5891cacba211514e2e7195967665aefd10f695f2", "vector-71k--voyage-rerank-3": "8ded261a994412dbecaca17d94e5a49cceae4a9de6ed2deebd74bc21c7d6b87e", "vector-71k--voyage-rerank-3-lite": "60493d5095ad819f685557a3405237e188e10e58bd07708000d3cda080411c93" } } }, "context": { "hybrid-71k--bge-v2-m3": { "hit@5_reranked": 0.5114754098360655, "k_needed_hit": 26, "k_needed_recall": 32, "llm_eur_saved_per_1000": { "anthropic/claude-sonnet-5.5": 7.694756907160391, "deepseek/deepseek-flash": 0.5771067680370293 }, "llm_eur_saved_per_1000_peak": { "deepseek/deepseek-flash": 1.1542135360740586 }, "llm_eur_saved_per_1000_recall": { "anthropic/claude-sonnet-5.5": 9.89325888063479, "deepseek/deepseek-flash": 0.7419944160476091 }, "llm_eur_saved_per_1000_recall_peak": { "deepseek/deepseek-flash": 1.4839888320952181 }, "recall@5_reranked": 0.36545667447306796, "saved_passages_hit": 21, "saved_passages_recall": 27 }, "hybrid-71k--gte-base": { "hit@5_reranked": 0.4852459016393443, "k_needed_hit": 20, "k_needed_recall": 27, "llm_eur_saved_per_1000": { "anthropic/claude-sonnet-5.5": 5.496254933685994, "deepseek/deepseek-flash": 0.41221912002644945 }, "llm_eur_saved_per_1000_peak": { "deepseek/deepseek-flash": 0.8244382400528989 }, "llm_eur_saved_per_1000_recall": { "anthropic/claude-sonnet-5.5": 8.061173902739457, "deepseek/deepseek-flash": 0.6045880427054593 }, "llm_eur_saved_per_1000_recall_peak": { "deepseek/deepseek-flash": 1.2091760854109186 }, "recall@5_reranked": 0.35117096018735366, "saved_passages_hit": 15, "saved_passages_recall": 22 }, "hybrid-71k--jina-v3.5": { "hit@5_reranked": 0.5180327868852459, "k_needed_hit": 30, "k_needed_recall": 38, "llm_eur_saved_per_1000": { "anthropic/claude-sonnet-5.5": 9.160424889476658, "deepseek/deepseek-flash": 0.6870318667107492 }, "llm_eur_saved_per_1000_peak": { "deepseek/deepseek-flash": 1.3740637334214985 }, "llm_eur_saved_per_1000_recall": { "anthropic/claude-sonnet-5.5": 12.091760854109186, "deepseek/deepseek-flash": 0.9068820640581889 }, "llm_eur_saved_per_1000_recall_peak": { "deepseek/deepseek-flash": 1.8137641281163779 }, "recall@5_reranked": 0.3731693989071038, "saved_passages_hit": 25, "saved_passages_recall": 33 }, "hybrid-71k--mxbai-large-v2": { "hit@5_reranked": 0.4819672131147541, "k_needed_hit": 20, "k_needed_recall": 23, "llm_eur_saved_per_1000": { "anthropic/claude-sonnet-5.5": 5.496254933685994, "deepseek/deepseek-flash": 0.41221912002644945 }, "llm_eur_saved_per_1000_peak": { "deepseek/deepseek-flash": 0.8244382400528989 }, "llm_eur_saved_per_1000_recall": { "anthropic/claude-sonnet-5.5": 6.595505920423192, "deepseek/deepseek-flash": 0.4946629440317394 }, "llm_eur_saved_per_1000_recall_peak": { "deepseek/deepseek-flash": 0.9893258880634788 }, "recall@5_reranked": 0.3360421545667447, "saved_passages_hit": 15, "saved_passages_recall": 18 }, "hybrid-71k--querit-4b": { "hit@5_reranked": 0.49836065573770494, "k_needed_hit": 24, "k_needed_recall": 29, "llm_eur_saved_per_1000": { "anthropic/claude-sonnet-5.5": 6.961922916002259, "deepseek/deepseek-flash": 0.5221442187001694 }, "llm_eur_saved_per_1000_peak": { "deepseek/deepseek-flash": 1.0442884374003387 }, "llm_eur_saved_per_1000_recall": { "anthropic/claude-sonnet-5.5": 8.794007893897591, "deepseek/deepseek-flash": 0.6595505920423194 }, "llm_eur_saved_per_1000_recall_peak": { "deepseek/deepseek-flash": 1.3191011840846387 }, "recall@5_reranked": 0.35725995316159254, "saved_passages_hit": 19, "saved_passages_recall": 24 }, "hybrid-71k--qwen3-0.6b": { "hit@5_reranked": 0.5081967213114754, "k_needed_hit": 24, "k_needed_recall": 33, "llm_eur_saved_per_1000": { "anthropic/claude-sonnet-5.5": 6.961922916002259, "deepseek/deepseek-flash": 0.5221442187001694 }, "llm_eur_saved_per_1000_peak": { "deepseek/deepseek-flash": 1.0442884374003387 }, "llm_eur_saved_per_1000_recall": { "anthropic/claude-sonnet-5.5": 10.259675876213857, "deepseek/deepseek-flash": 0.769475690716039 }, "llm_eur_saved_per_1000_recall_peak": { "deepseek/deepseek-flash": 1.538951381432078 }, "recall@5_reranked": 0.36669008587041374, "saved_passages_hit": 19, "saved_passages_recall": 28 }, "hybrid-71k--qwen3-4b": { "hit@5_reranked": 0.5180327868852459, "k_needed_hit": 30, "k_needed_recall": 38, "llm_eur_saved_per_1000": { "anthropic/claude-sonnet-5.5": 9.160424889476658, "deepseek/deepseek-flash": 0.6870318667107492 }, "llm_eur_saved_per_1000_peak": { "deepseek/deepseek-flash": 1.3740637334214985 }, "llm_eur_saved_per_1000_recall": { "anthropic/claude-sonnet-5.5": 12.091760854109186, "deepseek/deepseek-flash": 0.9068820640581889 }, "llm_eur_saved_per_1000_recall_peak": { "deepseek/deepseek-flash": 1.8137641281163779 }, "recall@5_reranked": 0.3769086651053864, "saved_passages_hit": 25, "saved_passages_recall": 33 }, "hybrid-71k--voyage-rerank-3": { "hit@5_reranked": 0.5081967213114754, "k_needed_hit": 24, "k_needed_recall": 37, "llm_eur_saved_per_1000": { "anthropic/claude-sonnet-5.5": 6.961922916002259, "deepseek/deepseek-flash": 0.5221442187001694 }, "llm_eur_saved_per_1000_peak": { "deepseek/deepseek-flash": 1.0442884374003387 }, "llm_eur_saved_per_1000_recall": { "anthropic/claude-sonnet-5.5": 11.725343858530119, "deepseek/deepseek-flash": 0.8794007893897589 }, "llm_eur_saved_per_1000_recall_peak": { "deepseek/deepseek-flash": 1.7588015787795177 }, "recall@5_reranked": 0.371928181108509, "saved_passages_hit": 19, "saved_passages_recall": 32 }, "hybrid-71k--voyage-rerank-3-lite": { "hit@5_reranked": 0.5016393442622951, "k_needed_hit": 24, "k_needed_recall": 32, "llm_eur_saved_per_1000": { "anthropic/claude-sonnet-5.5": 6.961922916002259, "deepseek/deepseek-flash": 0.5221442187001694 }, "llm_eur_saved_per_1000_peak": { "deepseek/deepseek-flash": 1.0442884374003387 }, "llm_eur_saved_per_1000_recall": { "anthropic/claude-sonnet-5.5": 9.89325888063479, "deepseek/deepseek-flash": 0.7419944160476091 }, "llm_eur_saved_per_1000_recall_peak": { "deepseek/deepseek-flash": 1.4839888320952181 }, "recall@5_reranked": 0.3614988290398127, "saved_passages_hit": 19, "saved_passages_recall": 27 }, "keyword-71k--bge-v2-m3": { "hit@5_reranked": 0.33114754098360655, "k_needed_hit": 60, "k_needed_recall": 57, "llm_eur_saved_per_1000": { "anthropic/claude-sonnet-5.5": 20.534752280415777, "deepseek/deepseek-flash": 1.5401064210311834 }, "llm_eur_saved_per_1000_peak": { "deepseek/deepseek-flash": 3.080212842062367 }, "llm_eur_saved_per_1000_recall": { "anthropic/claude-sonnet-5.5": 19.41467488330219, "deepseek/deepseek-flash": 1.456100616247664 }, "llm_eur_saved_per_1000_recall_peak": { "deepseek/deepseek-flash": 2.912201232495328 }, "recall@5_reranked": 0.20192818110850896, "saved_passages_hit": 55, "saved_passages_recall": 52 }, "keyword-71k--gte-base": { "hit@5_reranked": 0.31475409836065577, "k_needed_hit": 57, "k_needed_recall": 48, "llm_eur_saved_per_1000": { "anthropic/claude-sonnet-5.5": 19.41467488330219, "deepseek/deepseek-flash": 1.456100616247664 }, "llm_eur_saved_per_1000_peak": { "deepseek/deepseek-flash": 2.912201232495328 }, "llm_eur_saved_per_1000_recall": { "anthropic/claude-sonnet-5.5": 16.054442691961427, "deepseek/deepseek-flash": 1.204083201897107 }, "llm_eur_saved_per_1000_recall_peak": { "deepseek/deepseek-flash": 2.408166403794214 }, "recall@5_reranked": 0.18635441061670568, "saved_passages_hit": 52, "saved_passages_recall": 43 }, "keyword-71k--jina-v3.5": { "hit@5_reranked": 0.3442622950819672, "k_needed_hit": 75, "k_needed_recall": 62, "llm_eur_saved_per_1000": { "anthropic/claude-sonnet-5.5": 26.13513926598372, "deepseek/deepseek-flash": 1.9601354449487787 }, "llm_eur_saved_per_1000_peak": { "deepseek/deepseek-flash": 3.9202708898975573 }, "llm_eur_saved_per_1000_recall": { "anthropic/claude-sonnet-5.5": 21.28147054515817, "deepseek/deepseek-flash": 1.5961102908868627 }, "llm_eur_saved_per_1000_recall_peak": { "deepseek/deepseek-flash": 3.1922205817737255 }, "recall@5_reranked": 0.21266978922716628, "saved_passages_hit": 70, "saved_passages_recall": 57 }, "keyword-71k--mxbai-large-v2": { "hit@5_reranked": 0.30491803278688523, "k_needed_hit": 53, "k_needed_recall": 47, "llm_eur_saved_per_1000": { "anthropic/claude-sonnet-5.5": 17.921238353817408, "deepseek/deepseek-flash": 1.3440928765363056 }, "llm_eur_saved_per_1000_peak": { "deepseek/deepseek-flash": 2.688185753072611 }, "llm_eur_saved_per_1000_recall": { "anthropic/claude-sonnet-5.5": 15.68108355959023, "deepseek/deepseek-flash": 1.176081266969267 }, "llm_eur_saved_per_1000_recall_peak": { "deepseek/deepseek-flash": 2.352162533938534 }, "recall@5_reranked": 0.18309133489461357, "saved_passages_hit": 48, "saved_passages_recall": 42 }, "keyword-71k--querit-4b": { "hit@5_reranked": 0.3180327868852459, "k_needed_hit": 58, "k_needed_recall": 56, "llm_eur_saved_per_1000": { "anthropic/claude-sonnet-5.5": 19.78803401567339, "deepseek/deepseek-flash": 1.484102551175504 }, "llm_eur_saved_per_1000_peak": { "deepseek/deepseek-flash": 2.968205102351008 }, "llm_eur_saved_per_1000_recall": { "anthropic/claude-sonnet-5.5": 19.041315750931, "deepseek/deepseek-flash": 1.4280986813198246 }, "llm_eur_saved_per_1000_recall_peak": { "deepseek/deepseek-flash": 2.8561973626396493 }, "recall@5_reranked": 0.19563622170179545, "saved_passages_hit": 53, "saved_passages_recall": 51 }, "keyword-71k--qwen3-0.6b": { "hit@5_reranked": 0.33114754098360655, "k_needed_hit": 60, "k_needed_recall": 57, "llm_eur_saved_per_1000": { "anthropic/claude-sonnet-5.5": 20.534752280415777, "deepseek/deepseek-flash": 1.5401064210311834 }, "llm_eur_saved_per_1000_peak": { "deepseek/deepseek-flash": 3.080212842062367 }, "llm_eur_saved_per_1000_recall": { "anthropic/claude-sonnet-5.5": 19.41467488330219, "deepseek/deepseek-flash": 1.456100616247664 }, "llm_eur_saved_per_1000_recall_peak": { "deepseek/deepseek-flash": 2.912201232495328 }, "recall@5_reranked": 0.199032006245121, "saved_passages_hit": 55, "saved_passages_recall": 52 }, "keyword-71k--qwen3-4b": { "hit@5_reranked": 0.3344262295081967, "k_needed_hit": 62, "k_needed_recall": 60, "llm_eur_saved_per_1000": { "anthropic/claude-sonnet-5.5": 21.28147054515817, "deepseek/deepseek-flash": 1.5961102908868627 }, "llm_eur_saved_per_1000_peak": { "deepseek/deepseek-flash": 3.1922205817737255 }, "llm_eur_saved_per_1000_recall": { "anthropic/claude-sonnet-5.5": 20.534752280415777, "deepseek/deepseek-flash": 1.5401064210311834 }, "llm_eur_saved_per_1000_recall_peak": { "deepseek/deepseek-flash": 3.080212842062367 }, "recall@5_reranked": 0.21034348165495703, "saved_passages_hit": 57, "saved_passages_recall": 55 }, "keyword-71k--voyage-rerank-3": { "hit@5_reranked": 0.32786885245901637, "k_needed_hit": 60, "k_needed_recall": 57, "llm_eur_saved_per_1000": { "anthropic/claude-sonnet-5.5": 20.534752280415777, "deepseek/deepseek-flash": 1.5401064210311834 }, "llm_eur_saved_per_1000_peak": { "deepseek/deepseek-flash": 3.080212842062367 }, "llm_eur_saved_per_1000_recall": { "anthropic/claude-sonnet-5.5": 19.41467488330219, "deepseek/deepseek-flash": 1.456100616247664 }, "llm_eur_saved_per_1000_recall_peak": { "deepseek/deepseek-flash": 2.912201232495328 }, "recall@5_reranked": 0.20348165495706477, "saved_passages_hit": 55, "saved_passages_recall": 52 }, "keyword-71k--voyage-rerank-3-lite": { "hit@5_reranked": 0.3114754098360656, "k_needed_hit": 57, "k_needed_recall": 55, "llm_eur_saved_per_1000": { "anthropic/claude-sonnet-5.5": 19.41467488330219, "deepseek/deepseek-flash": 1.456100616247664 }, "llm_eur_saved_per_1000_peak": { "deepseek/deepseek-flash": 2.912201232495328 }, "llm_eur_saved_per_1000_recall": { "anthropic/claude-sonnet-5.5": 18.6679566185598, "deepseek/deepseek-flash": 1.400096746391985 }, "llm_eur_saved_per_1000_recall_peak": { "deepseek/deepseek-flash": 2.80019349278397 }, "recall@5_reranked": 0.1942232630757221, "saved_passages_hit": 52, "saved_passages_recall": 50 }, "keyword-full--bge-v2-m3": { "hit@5_reranked": 0.2360655737704918, "k_needed_hit": 72, "k_needed_recall": 71, "llm_eur_saved_per_1000": { "anthropic/claude-sonnet-5.5": 31.868469252843873, "deepseek/deepseek-flash": 2.3901351939632907 }, "llm_eur_saved_per_1000_peak": { "deepseek/deepseek-flash": 4.780270387926581 }, "llm_eur_saved_per_1000_recall": { "anthropic/claude-sonnet-5.5": 31.39282045802531, "deepseek/deepseek-flash": 2.354461534351898 }, "llm_eur_saved_per_1000_recall_peak": { "deepseek/deepseek-flash": 4.708923068703796 }, "recall@5_reranked": 0.14453551912568305, "saved_passages_hit": 67, "saved_passages_recall": 66 }, "keyword-full--gte-base": { "hit@5_reranked": 0.22295081967213115, "k_needed_hit": 61, "k_needed_recall": 54, "llm_eur_saved_per_1000": { "anthropic/claude-sonnet-5.5": 26.636332509839654, "deepseek/deepseek-flash": 1.9977249382379736 }, "llm_eur_saved_per_1000_peak": { "deepseek/deepseek-flash": 3.995449876475947 }, "llm_eur_saved_per_1000_recall": { "anthropic/claude-sonnet-5.5": 23.306790946109697, "deepseek/deepseek-flash": 1.748009320958227 }, "llm_eur_saved_per_1000_recall_peak": { "deepseek/deepseek-flash": 3.496018641916454 }, "recall@5_reranked": 0.13473067915690867, "saved_passages_hit": 56, "saved_passages_recall": 49 }, "keyword-full--jina-v3.5": { "status": "not measurable at depth 100" }, "keyword-full--mxbai-large-v2": { "hit@5_reranked": 0.23278688524590163, "k_needed_hit": 71, "k_needed_recall": 65, "llm_eur_saved_per_1000": { "anthropic/claude-sonnet-5.5": 31.39282045802531, "deepseek/deepseek-flash": 2.354461534351898 }, "llm_eur_saved_per_1000_peak": { "deepseek/deepseek-flash": 4.708923068703796 }, "llm_eur_saved_per_1000_recall": { "anthropic/claude-sonnet-5.5": 28.538927689113915, "deepseek/deepseek-flash": 2.1404195766835437 }, "llm_eur_saved_per_1000_recall_peak": { "deepseek/deepseek-flash": 4.280839153367087 }, "recall@5_reranked": 0.14073380171740826, "saved_passages_hit": 66, "saved_passages_recall": 60 }, "keyword-full--querit-4b": { "hit@5_reranked": 0.23934426229508196, "k_needed_hit": 76, "k_needed_recall": 71, "llm_eur_saved_per_1000": { "anthropic/claude-sonnet-5.5": 33.77106443211814, "deepseek/deepseek-flash": 2.53282983240886 }, "llm_eur_saved_per_1000_peak": { "deepseek/deepseek-flash": 5.06565966481772 }, "llm_eur_saved_per_1000_recall": { "anthropic/claude-sonnet-5.5": 31.39282045802531, "deepseek/deepseek-flash": 2.354461534351898 }, "llm_eur_saved_per_1000_recall_peak": { "deepseek/deepseek-flash": 4.708923068703796 }, "recall@5_reranked": 0.14455893832943012, "saved_passages_hit": 71, "saved_passages_recall": 66 }, "keyword-full--qwen3-0.6b": { "hit@5_reranked": 0.23934426229508196, "k_needed_hit": 76, "k_needed_recall": 71, "llm_eur_saved_per_1000": { "anthropic/claude-sonnet-5.5": 33.77106443211814, "deepseek/deepseek-flash": 2.53282983240886 }, "llm_eur_saved_per_1000_peak": { "deepseek/deepseek-flash": 5.06565966481772 }, "llm_eur_saved_per_1000_recall": { "anthropic/claude-sonnet-5.5": 31.39282045802531, "deepseek/deepseek-flash": 2.354461534351898 }, "llm_eur_saved_per_1000_recall_peak": { "deepseek/deepseek-flash": 4.708923068703796 }, "recall@5_reranked": 0.14483996877439498, "saved_passages_hit": 71, "saved_passages_recall": 66 }, "keyword-full--qwen3-4b": { "hit@5_reranked": 0.2459016393442623, "k_needed_hit": 78, "k_needed_recall": 76, "llm_eur_saved_per_1000": { "anthropic/claude-sonnet-5.5": 34.722362021755266, "deepseek/deepseek-flash": 2.604177151631645 }, "llm_eur_saved_per_1000_peak": { "deepseek/deepseek-flash": 5.20835430326329 }, "llm_eur_saved_per_1000_recall": { "anthropic/claude-sonnet-5.5": 33.77106443211814, "deepseek/deepseek-flash": 2.53282983240886 }, "llm_eur_saved_per_1000_recall_peak": { "deepseek/deepseek-flash": 5.06565966481772 }, "recall@5_reranked": 0.15325526932084307, "saved_passages_hit": 73, "saved_passages_recall": 71 }, "keyword-full--voyage-rerank-3": { "hit@5_reranked": 0.2360655737704918, "k_needed_hit": 72, "k_needed_recall": 71, "llm_eur_saved_per_1000": { "anthropic/claude-sonnet-5.5": 31.868469252843873, "deepseek/deepseek-flash": 2.3901351939632907 }, "llm_eur_saved_per_1000_peak": { "deepseek/deepseek-flash": 4.780270387926581 }, "llm_eur_saved_per_1000_recall": { "anthropic/claude-sonnet-5.5": 31.39282045802531, "deepseek/deepseek-flash": 2.354461534351898 }, "llm_eur_saved_per_1000_recall_peak": { "deepseek/deepseek-flash": 4.708923068703796 }, "recall@5_reranked": 0.1456830601092896, "saved_passages_hit": 67, "saved_passages_recall": 66 }, "keyword-full--voyage-rerank-3-lite": { "hit@5_reranked": 0.22950819672131148, "k_needed_hit": 71, "k_needed_recall": 64, "llm_eur_saved_per_1000": { "anthropic/claude-sonnet-5.5": 31.39282045802531, "deepseek/deepseek-flash": 2.354461534351898 }, "llm_eur_saved_per_1000_peak": { "deepseek/deepseek-flash": 4.708923068703796 }, "llm_eur_saved_per_1000_recall": { "anthropic/claude-sonnet-5.5": 28.063278894295355, "deepseek/deepseek-flash": 2.104745917072151 }, "llm_eur_saved_per_1000_recall_peak": { "deepseek/deepseek-flash": 4.209491834144302 }, "recall@5_reranked": 0.13882903981264635, "saved_passages_hit": 66, "saved_passages_recall": 59 }, "vector-71k--bge-v2-m3": { "hit@5_reranked": 0.8524590163934426, "k_needed_hit": 8, "k_needed_recall": 7, "llm_eur_saved_per_1000": { "anthropic/claude-sonnet-5.5": 0.8922471860800565, "deepseek/deepseek-flash": 0.06691853895600426 }, "llm_eur_saved_per_1000_peak": { "deepseek/deepseek-flash": 0.13383707791200852 }, "llm_eur_saved_per_1000_recall": { "anthropic/claude-sonnet-5.5": 0.5948314573867043, "deepseek/deepseek-flash": 0.04461235930400283 }, "llm_eur_saved_per_1000_recall_peak": { "deepseek/deepseek-flash": 0.08922471860800565 }, "recall@5_reranked": 0.632305490502212, "saved_passages_hit": 3, "saved_passages_recall": 2 }, "vector-71k--gte-base": { "hit@5_reranked": 0.7836065573770492, "k_needed_hit": 4, "k_needed_recall": 5, "llm_eur_saved_per_1000": { "anthropic/claude-sonnet-5.5": 0.0, "deepseek/deepseek-flash": 0.0 }, "llm_eur_saved_per_1000_peak": { "deepseek/deepseek-flash": 0.0 }, "llm_eur_saved_per_1000_recall": { "anthropic/claude-sonnet-5.5": 0.0, "deepseek/deepseek-flash": 0.0 }, "llm_eur_saved_per_1000_recall_peak": { "deepseek/deepseek-flash": 0.0 }, "recall@5_reranked": 0.552311995836586, "saved_passages_hit": 0, "saved_passages_recall": 0 }, "vector-71k--jina-v3.5": { "hit@5_reranked": 0.8524590163934426, "k_needed_hit": 8, "k_needed_recall": 8, "llm_eur_saved_per_1000": { "anthropic/claude-sonnet-5.5": 0.8922471860800565, "deepseek/deepseek-flash": 0.06691853895600426 }, "llm_eur_saved_per_1000_peak": { "deepseek/deepseek-flash": 0.13383707791200852 }, "llm_eur_saved_per_1000_recall": { "anthropic/claude-sonnet-5.5": 0.8922471860800565, "deepseek/deepseek-flash": 0.06691853895600426 }, "llm_eur_saved_per_1000_recall_peak": { "deepseek/deepseek-flash": 0.13383707791200852 }, "recall@5_reranked": 0.6440658339838669, "saved_passages_hit": 3, "saved_passages_recall": 3 }, "vector-71k--mxbai-large-v2": { "hit@5_reranked": 0.7639344262295082, "k_needed_hit": 4, "k_needed_recall": 5, "llm_eur_saved_per_1000": { "anthropic/claude-sonnet-5.5": 0.0, "deepseek/deepseek-flash": 0.0 }, "llm_eur_saved_per_1000_peak": { "deepseek/deepseek-flash": 0.0 }, "llm_eur_saved_per_1000_recall": { "anthropic/claude-sonnet-5.5": 0.0, "deepseek/deepseek-flash": 0.0 }, "llm_eur_saved_per_1000_recall_peak": { "deepseek/deepseek-flash": 0.0 }, "recall@5_reranked": 0.532352328909706, "saved_passages_hit": 0, "saved_passages_recall": 0 }, "vector-71k--querit-4b": { "hit@5_reranked": 0.8295081967213115, "k_needed_hit": 6, "k_needed_recall": 7, "llm_eur_saved_per_1000": { "anthropic/claude-sonnet-5.5": 0.29741572869335214, "deepseek/deepseek-flash": 0.022306179652001414 }, "llm_eur_saved_per_1000_peak": { "deepseek/deepseek-flash": 0.04461235930400283 }, "llm_eur_saved_per_1000_recall": { "anthropic/claude-sonnet-5.5": 0.5948314573867043, "deepseek/deepseek-flash": 0.04461235930400283 }, "llm_eur_saved_per_1000_recall_peak": { "deepseek/deepseek-flash": 0.08922471860800565 }, "recall@5_reranked": 0.6261930783242259, "saved_passages_hit": 1, "saved_passages_recall": 2 }, "vector-71k--qwen3-0.6b": { "hit@5_reranked": 0.8721311475409836, "k_needed_hit": 9, "k_needed_recall": 8, "llm_eur_saved_per_1000": { "anthropic/claude-sonnet-5.5": 1.1896629147734086, "deepseek/deepseek-flash": 0.08922471860800565 }, "llm_eur_saved_per_1000_peak": { "deepseek/deepseek-flash": 0.1784494372160113 }, "llm_eur_saved_per_1000_recall": { "anthropic/claude-sonnet-5.5": 0.8922471860800565, "deepseek/deepseek-flash": 0.06691853895600426 }, "llm_eur_saved_per_1000_recall_peak": { "deepseek/deepseek-flash": 0.13383707791200852 }, "recall@5_reranked": 0.6599557637262555, "saved_passages_hit": 4, "saved_passages_recall": 3 }, "vector-71k--qwen3-4b": { "hit@5_reranked": 0.8819672131147541, "k_needed_hit": 10, "k_needed_recall": 9, "llm_eur_saved_per_1000": { "anthropic/claude-sonnet-5.5": 1.4870786434667609, "deepseek/deepseek-flash": 0.11153089826000706 }, "llm_eur_saved_per_1000_peak": { "deepseek/deepseek-flash": 0.22306179652001412 }, "llm_eur_saved_per_1000_recall": { "anthropic/claude-sonnet-5.5": 1.1896629147734086, "deepseek/deepseek-flash": 0.08922471860800565 }, "llm_eur_saved_per_1000_recall_peak": { "deepseek/deepseek-flash": 0.1784494372160113 }, "recall@5_reranked": 0.6840202966432474, "saved_passages_hit": 5, "saved_passages_recall": 4 }, "vector-71k--voyage-rerank-3": { "hit@5_reranked": 0.8524590163934426, "k_needed_hit": 8, "k_needed_recall": 8, "llm_eur_saved_per_1000": { "anthropic/claude-sonnet-5.5": 0.8922471860800565, "deepseek/deepseek-flash": 0.06691853895600426 }, "llm_eur_saved_per_1000_peak": { "deepseek/deepseek-flash": 0.13383707791200852 }, "llm_eur_saved_per_1000_recall": { "anthropic/claude-sonnet-5.5": 0.8922471860800565, "deepseek/deepseek-flash": 0.06691853895600426 }, "llm_eur_saved_per_1000_recall_peak": { "deepseek/deepseek-flash": 0.13383707791200852 }, "recall@5_reranked": 0.6480783242258652, "saved_passages_hit": 3, "saved_passages_recall": 3 }, "vector-71k--voyage-rerank-3-lite": { "hit@5_reranked": 0.839344262295082, "k_needed_hit": 7, "k_needed_recall": 7, "llm_eur_saved_per_1000": { "anthropic/claude-sonnet-5.5": 0.5948314573867043, "deepseek/deepseek-flash": 0.04461235930400283 }, "llm_eur_saved_per_1000_peak": { "deepseek/deepseek-flash": 0.08922471860800565 }, "llm_eur_saved_per_1000_recall": { "anthropic/claude-sonnet-5.5": 0.5948314573867043, "deepseek/deepseek-flash": 0.04461235930400283 }, "llm_eur_saved_per_1000_recall_peak": { "deepseek/deepseek-flash": 0.08922471860800565 }, "recall@5_reranked": 0.6118293000260213, "saved_passages_hit": 2, "saved_passages_recall": 2 } }, "costs": { "api": { "voyage-rerank-3": { "hybrid-71k": { "k@100": { "eur_per_1000": 0.9534382927178939, "n_results": 3, "source": "latency", "tokenizer_deviation_pct": 2.0234357978206052, "tokenizer_estimate": 21800.83606557377, "tokens_per_query": 21368.45901639344, "tokens_per_query_max": 21368.45901639344, "tokens_per_query_min": 21368.45901639344 }, "k@20": { "eur_per_1000": 0.18955097240061675, "n_results": 3, "source": "latency", "tokenizer_deviation_pct": 2.0316337193777123, "tokenizer_estimate": 4334.524590163935, "tokens_per_query": 4248.216393442623, "tokens_per_query_max": 4248.216393442623, "tokens_per_query_min": 4248.216393442623 } }, "keyword-71k": { "k@100": { "eur_per_1000": 0.9684090466103685, "n_queries": 304, "source": "quality run", "tokens_per_query": 21703.98355263158 } }, "keyword-full": { "k@100": { "eur_per_1000": 1.2264181488770114, "n_queries": 304, "source": "quality run", "tokens_per_query": 27486.48355263158 } }, "vector-71k": { "k@100": { "eur_per_1000": 0.7805761257874149, "n_queries": 305, "source": "quality run", "tokens_per_query": 17494.27213114754 } } }, "voyage-rerank-3-lite": { "hybrid-71k": { "k@100": { "eur_per_1000": 0.3813753170871576, "n_results": 3, "source": "latency", "tokenizer_deviation_pct": 2.0234357978206052, "tokenizer_estimate": 21800.83606557377, "tokens_per_query": 21368.45901639344, "tokens_per_query_max": 21368.45901639344, "tokens_per_query_min": 21368.45901639344 }, "k@20": { "eur_per_1000": 0.07582038896024669, "n_results": 3, "source": "latency", "tokenizer_deviation_pct": 2.0316337193777123, "tokenizer_estimate": 4334.524590163935, "tokens_per_query": 4248.216393442623, "tokens_per_query_max": 4248.216393442623, "tokens_per_query_min": 4248.216393442623 } }, "keyword-71k": { "k@100": { "eur_per_1000": 0.3873636186441474, "n_queries": 304, "source": "quality run", "tokens_per_query": 21703.98355263158 } }, "keyword-full": { "k@100": { "eur_per_1000": 0.4905672595508046, "n_queries": 304, "source": "quality run", "tokens_per_query": 27486.48355263158 } }, "vector-71k": { "k@100": { "eur_per_1000": 0.31223045031496593, "n_queries": 305, "source": "quality run", "tokens_per_query": 17494.27213114754 } } } }, "break_even": { "bge-v2-m3-cuda-desktop-se-k@100": { "approximation": true, "break_even_vs_rerank-3": 224450.81305678052, "break_even_vs_rerank-3-lite": 561127.0326419512, "capacity_50": 2665384.942904679, "capacity_full": 5330769.885809358, "eur_per_month": 214.0, "k": 100, "model": "bge-v2-m3", "server": "gex45", "status_chat": "reachable", "status_chat_vs_rerank-3-lite": "reachable", "status_live": "not reachable", "status_live_vs_rerank-3-lite": "not reachable" }, "bge-v2-m3-cuda-desktop-se-k@20": { "approximation": true, "break_even_vs_rerank-3": 1128983.9207351052, "break_even_vs_rerank-3-lite": 2822459.8018377633, "capacity_50": 15595448.798455572, "capacity_full": 31190897.596911144, "eur_per_month": 214.0, "k": 20, "model": "bge-v2-m3", "server": "gex45", "status_chat": "reachable", "status_chat_vs_rerank-3-lite": "reachable", "status_live": "reachable", "status_live_vs_rerank-3-lite": "reachable" }, "gte-base-cuda-desktop-se-k@100": { "approximation": true, "break_even_vs_rerank-3": 224450.81305678052, "break_even_vs_rerank-3-lite": 561127.0326419512, "capacity_50": 5488996.783029616, "capacity_full": 10977993.566059232, "eur_per_month": 214.0, "k": 100, "model": "gte-base", "server": "gex45", "status_chat": "reachable", "status_chat_vs_rerank-3-lite": "reachable", "status_live": "reachable", "status_live_vs_rerank-3-lite": "reachable" }, "gte-base-cuda-desktop-se-k@20": { "approximation": true, "break_even_vs_rerank-3": 1128983.9207351052, "break_even_vs_rerank-3-lite": 2822459.8018377633, "capacity_50": 32669886.874571107, "capacity_full": 65339773.749142215, "eur_per_month": 214.0, "k": 20, "model": "gte-base", "server": "gex45", "status_chat": "reachable", "status_chat_vs_rerank-3-lite": "reachable", "status_live": "reachable", "status_live_vs_rerank-3-lite": "reachable" }, "jina-v3.5-cuda-desktop-se-k@100": { "approximation": true, "break_even_vs_rerank-3": 224450.81305678052, "break_even_vs_rerank-3-lite": 561127.0326419512, "capacity_50": 418675.2916504048, "capacity_full": 837350.5833008096, "eur_per_month": 214.0, "k": 100, "model": "jina-v3.5", "server": "gex45", "status_chat": "not reachable", "status_chat_vs_rerank-3-lite": "not reachable", "status_live": "not reachable", "status_live_vs_rerank-3-lite": "not reachable" }, "jina-v3.5-cuda-desktop-se-k@20": { "approximation": true, "break_even_vs_rerank-3": 1128983.9207351052, "break_even_vs_rerank-3-lite": 2822459.8018377633, "capacity_50": 6375494.62611351, "capacity_full": 12750989.25222702, "eur_per_month": 214.0, "k": 20, "model": "jina-v3.5", "server": "gex45", "status_chat": "reachable", "status_chat_vs_rerank-3-lite": "reachable", "status_live": "reachable", "status_live_vs_rerank-3-lite": "reachable" }, "mxbai-large-v2-cuda-desktop-se-k@100": { "approximation": true, "break_even_vs_rerank-3": 224450.81305678052, "break_even_vs_rerank-3-lite": 561127.0326419512, "capacity_50": 568343.8878708056, "capacity_full": 1136687.7757416111, "eur_per_month": 214.0, "k": 100, "model": "mxbai-large-v2", "server": "gex45", "status_chat": "not reachable", "status_chat_vs_rerank-3-lite": "not reachable", "status_live": "not reachable", "status_live_vs_rerank-3-lite": "not reachable" }, "mxbai-large-v2-cuda-desktop-se-k@20": { "approximation": true, "break_even_vs_rerank-3": 1128983.9207351052, "break_even_vs_rerank-3-lite": 2822459.8018377633, "capacity_50": 2271543.9541040836, "capacity_full": 4543087.908208167, "eur_per_month": 214.0, "k": 20, "model": "mxbai-large-v2", "server": "gex45", "status_chat": "reachable", "status_chat_vs_rerank-3-lite": "not reachable", "status_live": "not reachable", "status_live_vs_rerank-3-lite": "not reachable" }, "querit-4b-cuda-desktop-se-k@100": { "approximation": true, "break_even_vs_rerank-3": 224450.81305678052, "break_even_vs_rerank-3-lite": 561127.0326419512, "capacity_50": 98718.78887982092, "capacity_full": 197437.57775964183, "eur_per_month": 214.0, "k": 100, "model": "querit-4b", "server": "gex45", "status_chat": "not reachable", "status_chat_vs_rerank-3-lite": "not reachable", "status_live": "not reachable", "status_live_vs_rerank-3-lite": "not reachable" }, "querit-4b-cuda-desktop-se-k@20": { "approximation": true, "break_even_vs_rerank-3": 1128983.9207351052, "break_even_vs_rerank-3-lite": 2822459.8018377633, "capacity_50": 489421.7829323102, "capacity_full": 978843.5658646204, "eur_per_month": 214.0, "k": 20, "model": "querit-4b", "server": "gex45", "status_chat": "not reachable", "status_chat_vs_rerank-3-lite": "not reachable", "status_live": "not reachable", "status_live_vs_rerank-3-lite": "not reachable" }, "qwen3-0.6b-cuda-desktop-se-k@100": { "approximation": true, "break_even_vs_rerank-3": 224450.81305678052, "break_even_vs_rerank-3-lite": 561127.0326419512, "capacity_50": 1001956.3006692568, "capacity_full": 2003912.6013385137, "eur_per_month": 214.0, "k": 100, "model": "qwen3-0.6b", "server": "gex45", "status_chat": "not reachable", "status_chat_vs_rerank-3-lite": "not reachable", "status_live": "not reachable", "status_live_vs_rerank-3-lite": "not reachable" }, "qwen3-0.6b-cuda-desktop-se-k@20": { "approximation": true, "break_even_vs_rerank-3": 1128983.9207351052, "break_even_vs_rerank-3-lite": 2822459.8018377633, "capacity_50": 3482179.5100111053, "capacity_full": 6964359.020022211, "eur_per_month": 214.0, "k": 20, "model": "qwen3-0.6b", "server": "gex45", "status_chat": "reachable", "status_chat_vs_rerank-3-lite": "reachable", "status_live": "not reachable", "status_live_vs_rerank-3-lite": "not reachable" }, "qwen3-4b-cuda-desktop-se-k@100": { "approximation": true, "break_even_vs_rerank-3": 224450.81305678052, "break_even_vs_rerank-3-lite": 561127.0326419512, "capacity_50": 333222.3453825137, "capacity_full": 666444.6907650274, "eur_per_month": 214.0, "k": 100, "model": "qwen3-4b", "server": "gex45", "status_chat": "not reachable", "status_chat_vs_rerank-3-lite": "not reachable", "status_live": "not reachable", "status_live_vs_rerank-3-lite": "not reachable" }, "qwen3-4b-cuda-desktop-se-k@20": { "approximation": true, "break_even_vs_rerank-3": 1128983.9207351052, "break_even_vs_rerank-3-lite": 2822459.8018377633, "capacity_50": 1402217.4492120992, "capacity_full": 2804434.8984241984, "eur_per_month": 214.0, "k": 20, "model": "qwen3-4b", "server": "gex45", "status_chat": "reachable", "status_chat_vs_rerank-3-lite": "not reachable", "status_live": "not reachable", "status_live_vs_rerank-3-lite": "not reachable" } }, "first_stage_p95_ms": 67.2347182946396, "fx": { "date": "2026-10-09", "note": "ECB reference rate of Friday 2026-10-09, the last one before the measurement days (Saturday 2026-10-10, Sunday 2026-10-11); the ECB publishes no rate on weekends", "source": "ECB reference rate", "usd_per_eur": 1.1206, "weekday": "Friday" }, "mean_passage_tokens": { "hybrid-71k": 205.30344262295083, "keyword-71k": 209.1931218675811, "keyword-full": 266.5060197368421, "vector-71k": 166.64203278688524 }, "prices": { "budgets": { "chat_rerank_p95_ms": 2000, "live_search_total_p95_ms": 500, "note": "assumptions: live search a few hundred milliseconds in total, chat 1 to 2 s for the reranking step" }, "fx": { "date": "2026-10-09", "source": "ECB reference rate", "usd_per_eur": 1.1206 }, "hetzner": { "gex45": { "date": "2026-10-09", "eur_per_month": 212.3, "extra_eur_per_month": 1.7, "gpu": "RTX PRO 4000 Blackwell SFF 24 GB", "note": "price as checked on 2026-10-09; not re-checkable on 2026-10-11 (JavaScript-only page)", "setup_eur": 209.0, "source": "hetzner.com GPU server overview, website-price-api.hetzner.com" } }, "llm": [ { "date": "2026-10-11", "id": "anthropic/claude-sonnet-5.5", "source": "https://claude.com/pricing", "usd_per_million_input": 2.0 }, { "date": "2026-10-11", "id": "deepseek/deepseek-flash", "note": "DeepSeek-V4.1-Flash, cache miss, off-peak; legacy name deepseek-v4-flash is billed at this price", "source": "https://api-docs.deepseek.com/quick_start/pricing", "usd_per_million_input": 0.15, "usd_per_million_input_peak": 0.3 } ], "voyage": { "rerank-3": { "date": "2026-10-11", "source": "https://docs.voyageai.com/docs/pricing", "usd_per_million_tokens": 0.05 }, "rerank-3-lite": { "date": "2026-10-11", "source": "https://docs.voyageai.com/docs/pricing", "usd_per_million_tokens": 0.02 } } }, "prices_sha256": "ed1be9a6716182ba4f567fc56b5d8a10d11a33c8ccf901872d97fb2ae2f4c93d", "usd_per_eur": 1.1206, "voyage_tokenizer_revision": "2d2c29517b5c3bbf130c879fda12b1678e598158" }, "created_utc": "2026-10-11T16:42:12+00:00", "notes": { "gpu_approximation": "GPU latency was measured on an RTX 5070 Ti, not on the GEX45 (RTX PRO 4000 Blackwell SFF 24 GB): its break-even values are an approximation", "passage_tokens": "mean_passage_tokens: per stage, voyageai/rerank-3 at revision 2d2c29517b5c3bbf130c879fda12b1678e598158 on title and text of the top 100 candidates of all queries", "saving": "saved passages and the LLM saving are computed for Hit@5 (k_needed_hit) and Recall@5 (k_needed_recall); an LLM with a peak price is also priced at it (keys ending in _peak)", "setup_fee": "the GEX45 monthly price is 214.00 EUR; the one-off setup fee of 209 EUR is not part of it" }, "run_id": "costs-round1-20261011T164212Z-95734e" }, "cpu_indicative": { "conditions": { "cpu": "AMD Ryzen 9 5950X 16-Core Processor", "memory_cap": { "gb": 30, "source": "DEVIATIONS.md 2026-10-11" }, "n_queries": 10, "passes": 1, "threads": 8, "warmup_calls": 2 }, "models": { "bge-v2-m3": { "created_utc": "2026-10-11T04:59:51+00:00", "k@100": { "mean_ms": 61409.83661879727, "n": 10, "p50_ms": 62146.97118299955, "p95_ms": 62642.68078599707 }, "k@20": { "mean_ms": 9977.319651597645, "n": 10, "p50_ms": 10241.914545491454, "p95_ms": 12916.846485299175 }, "loadavg_before": [ 7.18212890625, 5.61181640625, 3.3896484375 ], "run_id": "latency-bge-v2-m3-cpu-desktop-se-cpu8-20261011T045951Z-76c895" }, "gte-base": { "created_utc": "2026-10-11T04:45:37+00:00", "k@100": { "mean_ms": 28298.464546899777, "n": 10, "p50_ms": 28612.30463501124, "p95_ms": 28819.95504853985 }, "k@20": { "mean_ms": 4223.670570898685, "n": 10, "p50_ms": 4512.1684889891185, "p95_ms": 5357.161884418747 }, "loadavg_before": [ 1.12451171875, 1.54296875, 1.46630859375 ], "run_id": "latency-gte-base-cpu-desktop-se-cpu8-20261011T044537Z-764ee9" }, "jina-v3.5": { "created_utc": "2026-10-11T06:06:24+00:00", "k@100": { "mean_ms": 188551.24795570446, "n": 10, "p50_ms": 195979.08959601773, "p95_ms": 283557.6599151027 }, "k@20": { "mean_ms": 19275.705510302214, "n": 10, "p50_ms": 18282.998731010593, "p95_ms": 34405.58793949019 }, "loadavg_before": [ 6.3681640625, 7.09912109375, 7.091796875 ], "run_id": "latency-jina-v3.5-cpu-desktop-se-cpu8-20261011T060624Z-016a59" }, "qwen3-0.6b": { "created_utc": "2026-10-11T09:24:40+00:00", "k@100": { "status": "not measurable in 30 GB" }, "k@20": { "mean_ms": 44060.33865789941, "n": 10, "p50_ms": 35533.5564614943, "p95_ms": 98836.73847516466 }, "loadavg_before": [ 0.7578125, 0.5498046875, 1.03759765625 ], "run_id": "latency-qwen3-0.6b-cpu-desktop-se-cpu8-20261011T092440Z-101896" } } }, "first_stage": { "hybrid-71k": { "contamination": [ "MIRACLRetrieval", "MIRACLRetrievalHardNegatives", "MIRACLReranking" ], "counts": { "queries_with_fewer_than_100": 0, "queries_without_hits": 0 }, "created_utc": "2026-10-10T00:07:29+00:00", "exploratory": false, "latency": { "mean_ms": 38.601024595694874, "n": 915, "p50_ms": 35.02669600129593, "p95_ms": 55.96083429700229 }, "latency_created_utc": "2026-10-10T15:22:34+00:00", "latency_run_id": "latency-first-stage-hybrid-71k-desktop-se-20261010T152234Z-80ffe8", "metrics": { "ceiling_ndcg@10": { "mean": 0.46115157793366746 }, "hit@10": { "mean": 0.4360655737704918 }, "hit@5": { "ci_high": 0.3901639344262295, "ci_low": 0.28524590163934427, "mean": 0.3377049180327869 }, "judged@10": { "mean": 0.12688524590163935 }, "mrr@10": { "mean": 0.18533437418683318 }, "ndcg@10": { "ci_high": 0.20373374554993806, "ci_low": 0.1476800243657697, "mean": 0.17498289422243518 }, "ndcg_condensed@10": { "mean": 0.38735964250551674 }, "recall@10": { "mean": 0.26903200624512097 }, "recall@100": { "mean": 0.4350039032006245 }, "recall@20": { "mean": 0.32904761904761903 }, "recall@5": { "mean": 0.19079625292740046 } }, "query_embedding": { "batch_size": 1, "device": "cuda", "dtype": "float16", "mean_ms": 10.062300876788719, "model": "Snowflake/snowflake-arctic-embed-l-v2.0", "n": 915, "p50_ms": 9.828702997765504, "p95_ms": 11.273883997637313, "revision": "ac6544c8a46e00af67e330e85a9028c66b8cfd9a", "warmup": 20 }, "run_id": "first_stage-hybrid-71k-20261010T000729Z-108e4c" }, "keyword-71k": { "contamination": [], "counts": { "queries_with_fewer_than_100": 3, "queries_without_hits": 1 }, "created_utc": "2026-10-10T00:07:21+00:00", "exploratory": false, "latency": { "mean_ms": 38.32853186802107, "n": 915, "p50_ms": 36.75935500359628, "p95_ms": 55.03031899861525 }, "latency_created_utc": "2026-10-10T15:21:26+00:00", "latency_run_id": "latency-first-stage-keyword-71k-desktop-se-20261010T152126Z-330905", "metrics": { "ceiling_ndcg@10": { "mean": 0.2638523417479578 }, "hit@10": { "mean": 0.18688524590163935 }, "hit@5": { "ci_high": 0.1737704918032787, "ci_low": 0.09836065573770492, "mean": 0.13442622950819672 }, "judged@10": { "mean": 0.07081967213114755 }, "mrr@10": { "mean": 0.09334894613583139 }, "ndcg@10": { "ci_high": 0.09605649644661068, "ci_low": 0.05430560617418975, "mean": 0.07428238494756044 }, "ndcg_condensed@10": { "mean": 0.20465572369341348 }, "recall@10": { "mean": 0.10108508977361437 }, "recall@100": { "mean": 0.23757220921155345 }, "recall@20": { "mean": 0.14451990632318498 }, "recall@5": { "mean": 0.06959406713505073 } }, "query_embedding": null, "run_id": "first_stage-keyword-71k-20261010T000721Z-52e3c2" }, "keyword-71k-ltr": { "contamination": [], "counts": { "queries_with_fewer_than_100": 3, "queries_without_hits": 1 }, "created_utc": "2026-10-10T16:21:29+00:00", "exploratory": true, "latency": null, "latency_created_utc": null, "latency_run_id": null, "metrics": { "ceiling_ndcg@10": { "mean": 0.5014375832052528 }, "hit@10": { "mean": 0.3114754098360656 }, "hit@5": { "ci_high": 0.2754098360655738, "ci_low": 0.18360655737704917, "mean": 0.22950819672131148 }, "judged@10": { "mean": 0.1321311475409836 }, "mrr@10": { "mean": 0.14824225865209473 }, "ndcg@10": { "ci_high": 0.14879409253958079, "ci_low": 0.09836524572260068, "mean": 0.12245521303522566 }, "ndcg_condensed@10": { "mean": 0.37037365727189425 }, "recall@10": { "mean": 0.17128024980483997 }, "recall@100": { "mean": 0.4625696070778038 }, "recall@20": { "mean": 0.2454215456674473 }, "recall@5": { "mean": 0.11541764246682279 } }, "query_embedding": null, "run_id": "first_stage-keyword-71k-ltr-20261010T162129Z-6dea2f" }, "keyword-full": { "contamination": [], "counts": { "queries_with_fewer_than_100": 1, "queries_without_hits": 1 }, "created_utc": "2026-10-10T01:48:27+00:00", "exploratory": false, "latency": { "mean_ms": 160.51128488536568, "n": 915, "p50_ms": 56.46387599699665, "p95_ms": 764.8886294016844 }, "latency_created_utc": "2026-10-10T15:25:12+00:00", "latency_run_id": "latency-first-stage-keyword-full-desktop-se-20261010T152512Z-e352d2", "metrics": { "ceiling_ndcg@10": { "mean": 0.18570729421947257 }, "hit@10": { "mean": 0.14098360655737704 }, "hit@5": { "ci_high": 0.14426229508196722, "ci_low": 0.07540983606557378, "mean": 0.10819672131147541 }, "judged@10": { "mean": 0.05114754098360656 }, "mrr@10": { "mean": 0.0804072339318241 }, "ndcg@10": { "ci_high": 0.08247661951521222, "ci_low": 0.042417668801672095, "mean": 0.0614607226564852 }, "ndcg_condensed@10": { "mean": 0.14786485122088336 }, "recall@10": { "mean": 0.07630757220921154 }, "recall@100": { "mean": 0.1662217017954723 }, "recall@20": { "mean": 0.10241217798594845 }, "recall@5": { "mean": 0.057572209211553475 } }, "query_embedding": null, "run_id": "first_stage-keyword-full-20261010T014827Z-cc6118" }, "keyword-plain-71k": { "contamination": [], "counts": { "queries_with_fewer_than_100": 287, "queries_without_hits": 0 }, "created_utc": "2026-10-10T00:07:33+00:00", "exploratory": false, "latency": { "mean_ms": 27.576348629857723, "n": 915, "p50_ms": 21.44671000132803, "p95_ms": 76.7096120980568 }, "latency_created_utc": "2026-10-10T15:25:38+00:00", "latency_run_id": "latency-first-stage-keyword-plain-71k-desktop-se-20261010T152538Z-2237d2", "metrics": { "ceiling_ndcg@10": { "mean": 0.015728790487001005 }, "hit@10": { "mean": 0.03278688524590164 }, "hit@5": { "ci_high": 0.04918032786885246, "ci_low": 0.013114754098360656, "mean": 0.029508196721311476 }, "judged@10": { "mean": 0.007540983606557377 }, "mrr@10": { "mean": 0.025819672131147543 }, "ndcg@10": { "ci_high": 0.020784422170357714, "ci_low": 0.004857981133367159, "mean": 0.012118580232116774 }, "ndcg_condensed@10": { "mean": 0.013986005307201258 }, "recall@10": { "mean": 0.010140515222482435 }, "recall@100": { "mean": 0.011779859484777517 }, "recall@20": { "mean": 0.011779859484777517 }, "recall@5": { "mean": 0.008501170960187352 } }, "query_embedding": null, "run_id": "first_stage-keyword-plain-71k-20261010T000733Z-b317ea" }, "vector-71k": { "contamination": [ "MIRACLRetrieval", "MIRACLRetrievalHardNegatives", "MIRACLReranking" ], "counts": { "queries_with_fewer_than_100": 0, "queries_without_hits": 0 }, "created_utc": "2026-10-10T00:07:25+00:00", "exploratory": false, "latency": { "mean_ms": 33.95838939454063, "n": 915, "p50_ms": 31.73468599561602, "p95_ms": 41.02490899240365 }, "latency_created_utc": "2026-10-10T15:21:58+00:00", "latency_run_id": "latency-first-stage-vector-71k-desktop-se-20261010T152158Z-b8b773", "metrics": { "ceiling_ndcg@10": { "mean": 0.9716164506014489 }, "hit@10": { "mean": 0.8852459016393442 }, "hit@5": { "ci_high": 0.8622950819672132, "ci_low": 0.7770491803278688, "mean": 0.819672131147541 }, "judged@10": { "mean": 0.3737704918032787 }, "mrr@10": { "mean": 0.6627582617746551 }, "ndcg@10": { "ci_high": 0.6265068816908699, "ci_low": 0.5532981146569897, "mean": 0.5903147920817311 }, "ndcg_condensed@10": { "mean": 0.8171232816985772 }, "recall@10": { "mean": 0.7042583918813426 }, "recall@100": { "mean": 0.9652354931043456 }, "recall@20": { "mean": 0.8045771532656778 }, "recall@5": { "mean": 0.5768214936247723 } }, "query_embedding": { "batch_size": 1, "device": "cuda", "dtype": "float16", "mean_ms": 10.062300876788719, "model": "Snowflake/snowflake-arctic-embed-l-v2.0", "n": 915, "p50_ms": 9.828702997765504, "p95_ms": 11.273883997637313, "revision": "ac6544c8a46e00af67e330e85a9028c66b8cfd9a", "warmup": 20 }, "run_id": "first_stage-vector-71k-20261010T000725Z-1fc254" } }, "hardware": { "gpu": "NVIDIA GeForce RTX 5070 Ti" }, "judgments": { "checks": { "answered": 72, "control_agreed": 13, "control_agreement": 0.8666666666666667, "control_answered": 15, "control_total": 15, "control_warning": false, "judge": { "date": "2026-10-10", "kind": "llm", "labels_sha256": "01e4c73defadd585a3855136c2be7144863f5a9e66c2ed80d55e7285b328e124", "model": "claude-opus-5-5", "pairs_sha256": "9d23d6678b11205cbc58a320ff49f7475b3fcd0155e589853c4e5971cd86b67d", "prompt_sha256": "5fea598f8f99864a5ad740e2d20196e280266990ea4ba45fbb430d7bb482ee3f" }, "sample_sha256": "70d6de4659f40174054f1ca7f2b423287acb2860de0cb4b0e4f384855fcfd1dc", "seed": 20261009, "sources": { "control-irrelevant": { "corpus": "71k", "corpus_sha256": "8e33b4da3abaf1bf7101d1d68f50d4c3c59fda65010153d95a7412a5c0d1196b", "qrels_sha256": "5d4a0d7ac42c7839a0513ba6deb6b27fc62803f318be8a7c411e10dbbde7578c" }, "control-relevant": { "corpus": "71k", "corpus_sha256": "8e33b4da3abaf1bf7101d1d68f50d4c3c59fda65010153d95a7412a5c0d1196b", "qrels_sha256": "5d4a0d7ac42c7839a0513ba6deb6b27fc62803f318be8a7c411e10dbbde7578c" }, "hybrid-71k:jina-v3.5": { "model": "jina-v3.5", "run_sha256": "c7910a55cac133dec5345383d1a96760db613e6986b74fd17df77eef78d77344", "stage": "hybrid-71k" }, "hybrid-71k:qwen3-4b": { "model": "qwen3-4b", "run_sha256": "4a9458d131e44d4eb0b5f523f19e0dda48dcc55970b40e15641deee340208bc7", "stage": "hybrid-71k" }, "hybrid-71k:voyage-rerank-3": { "model": "voyage-rerank-3", "run_sha256": "3e949aeb397e8a053376542666e591d5993eea5b5e8847b14e62b2a01fb64cc2", "stage": "hybrid-71k" }, "keyword-full:qwen3-4b": { "model": "qwen3-4b", "run_sha256": "ca03597659b4ce2c9399e8148c982b25230617ec3706815baad8148756c9d46d", "stage": "keyword-full" } }, "unclear": 0 }, "created_utc": "2026-10-10T18:21:00+00:00", "metrics": { "control-irrelevant": { "judged": 7, "rate": 0.2857142857142857, "relevant": 2, "unclear": 0, "wilson_high": 0.6410709098517873, "wilson_low": 0.0822171657090155 }, "control-relevant": { "judged": 8, "rate": 1.0, "relevant": 8, "unclear": 0, "wilson_high": 1.0, "wilson_low": 0.6755843804891231 }, "hybrid-71k:jina-v3.5": { "judged": 15, "rate": 0.4, "relevant": 6, "unclear": 0, "wilson_high": 0.642535572434923, "wilson_low": 0.19824228082594664 }, "hybrid-71k:qwen3-4b": { "judged": 15, "rate": 0.4, "relevant": 6, "unclear": 0, "wilson_high": 0.642535572434923, "wilson_low": 0.19824228082594664 }, "hybrid-71k:voyage-rerank-3": { "judged": 15, "rate": 0.4666666666666667, "relevant": 7, "unclear": 0, "wilson_high": 0.6988336984894921, "wilson_low": 0.24809225259746442 }, "keyword-full:qwen3-4b": { "judged": 15, "rate": 0.06666666666666667, "relevant": 1, "unclear": 0, "wilson_high": 0.2981707755242399, "wilson_low": 0.011866588606194856 } }, "notes": { "judge_bias": "an LLM judge may favour the same passages as LLM-based rerankers; the control pairs measure agreement with the MIRACL annotators, not this bias", "unclear_answers": "unclear answers are left out of the rates and counted separately, per group and in total" }, "run_id": "judgments-judged-sample-20261010T182100Z-ca8dbb" }, "latency": { "api:desktop-se:afternoon": { "voyage-rerank-3": { "created_utc": "2026-10-11T10:05:17+00:00", "finished_local": "2026-10-11T12:05:17+02:00", "k@100": { "mean_ms": 351.57125757164414, "n": 301, "p50_ms": 341.25486901029944, "p50_ms_per_pass": [ 341.25486901029944 ], "p95_ms": 456.9225680024829 }, "k@20": { "mean_ms": 288.2902025973203, "n": 305, "p50_ms": 278.45161501318216, "p50_ms_per_pass": [ 278.45161501318216 ], "p95_ms": 366.03759081335744 }, "run_id": "latency-voyage-rerank-3-api-desktop-se-afternoon-20261011T100517Z-9099a1", "tcp_connect_ms": 12.186575986561365, "training_opt_out": "confirmed", "training_opt_out_checked": "2026-10-11", "weekday": "Sunday" }, "voyage-rerank-3-lite": { "created_utc": "2026-10-11T10:08:52+00:00", "finished_local": "2026-10-11T12:08:52+02:00", "k@100": { "mean_ms": 351.9095244924309, "n": 305, "p50_ms": 344.37996201450005, "p50_ms_per_pass": [ 344.37996201450005 ], "p95_ms": 435.10518818511633 }, "k@20": { "mean_ms": 294.4341976490429, "n": 305, "p50_ms": 287.0092080265749, "p50_ms_per_pass": [ 287.0092080265749 ], "p95_ms": 356.0759455896914 }, "run_id": "latency-voyage-rerank-3-lite-api-desktop-se-afternoon-20261011T100852Z-9066d4", "tcp_connect_ms": 12.089155497960746, "training_opt_out": "confirmed", "training_opt_out_checked": "2026-10-11", "weekday": "Sunday" } }, "api:desktop-se:evening": { "voyage-rerank-3": { "created_utc": "2026-10-10T16:29:02+00:00", "finished_local": "2026-10-10T18:29:02+02:00", "k@100": { "mean_ms": 359.9083460498047, "n": 301, "p50_ms": 342.206155997701, "p50_ms_per_pass": [ 342.206155997701 ], "p95_ms": 504.94668100145645 }, "k@20": { "mean_ms": 288.0009136493337, "n": 305, "p50_ms": 278.48049098975025, "p50_ms_per_pass": [ 278.48049098975025 ], "p95_ms": 355.46070059936034 }, "run_id": "latency-voyage-rerank-3-api-desktop-se-evening-20261010T162902Z-f848c5", "tcp_connect_ms": 12.122179003199562, "training_opt_out": "unconfirmed", "training_opt_out_checked": "2026-10-09", "weekday": "Saturday" }, "voyage-rerank-3-lite": { "created_utc": "2026-10-10T16:32:44+00:00", "finished_local": "2026-10-10T18:32:44+02:00", "k@100": { "mean_ms": 369.36725611129555, "n": 305, "p50_ms": 353.2199600012973, "p50_ms_per_pass": [ 353.2199600012973 ], "p95_ms": 481.75052179140044 }, "k@20": { "mean_ms": 297.00609048528855, "n": 305, "p50_ms": 289.27212199778296, "p50_ms_per_pass": [ 289.27212199778296 ], "p95_ms": 352.49391680117697 }, "run_id": "latency-voyage-rerank-3-lite-api-desktop-se-evening-20261010T163244Z-facd01", "tcp_connect_ms": 12.181218007754069, "training_opt_out": "unconfirmed", "training_opt_out_checked": "2026-10-09", "weekday": "Saturday" } }, "api:desktop-se:morning": { "voyage-rerank-3": { "created_utc": "2026-10-11T06:34:14+00:00", "finished_local": "2026-10-11T08:34:14+02:00", "k@100": { "mean_ms": 350.45696019239114, "n": 301, "p50_ms": 340.6027519959025, "p50_ms_per_pass": [ 340.6027519959025 ], "p95_ms": 433.673435007222 }, "k@20": { "mean_ms": 286.0286203151966, "n": 305, "p50_ms": 278.1783709942829, "p50_ms_per_pass": [ 278.1783709942829 ], "p95_ms": 328.9324154029601 }, "run_id": "latency-voyage-rerank-3-api-desktop-se-morning-20261011T063414Z-47493a", "tcp_connect_ms": 12.221959492308088, "training_opt_out": "unconfirmed", "training_opt_out_checked": "2026-10-09", "weekday": "Sunday" }, "voyage-rerank-3-lite": { "created_utc": "2026-10-11T06:37:50+00:00", "finished_local": "2026-10-11T08:37:50+02:00", "k@100": { "mean_ms": 356.2167529388881, "n": 305, "p50_ms": 350.03936299472116, "p50_ms_per_pass": [ 350.03936299472116 ], "p95_ms": 414.25477258162573 }, "k@20": { "mean_ms": 293.16903815335655, "n": 305, "p50_ms": 286.7322310048621, "p50_ms_per_pass": [ 286.7322310048621 ], "p95_ms": 361.3657616020646 }, "run_id": "latency-voyage-rerank-3-lite-api-desktop-se-morning-20261011T063750Z-d5f19f", "tcp_connect_ms": 11.974098495556973, "training_opt_out": "unconfirmed", "training_opt_out_checked": "2026-10-09", "weekday": "Sunday" } }, "cuda:desktop-se": { "bge-v2-m3": { "created_utc": "2026-10-11T04:32:32+00:00", "k@100": { "mean_ms": 486.2337064858058, "n": 915, "p50_ms": 496.5486779983621, "p50_ms_per_pass": [ 512.6457090082113, 496.0023900202941, 494.9810719990637 ], "p95_ms": 516.5834453160642 }, "k@20": { "mean_ms": 83.10116731801548, "n": 915, "p50_ms": 85.87239900953136, "p50_ms_per_pass": [ 84.49011397897266, 87.4492930015549, 87.9313990008086 ], "p95_ms": 107.95813818695024 }, "k@50": { "mean_ms": 238.89692419894354, "n": 915, "p50_ms": 259.28083798498847, "p50_ms_per_pass": [ 259.2426020128187, 259.23776798299514, 259.2926849902142 ], "p95_ms": 264.3223113118438 }, "run_id": "latency-bge-v2-m3-cuda-desktop-se-20261011T043232Z-d00b64" }, "gte-base": { "created_utc": "2026-10-11T04:39:00+00:00", "k@100": { "mean_ms": 236.1087191755797, "n": 915, "p50_ms": 244.47898898506537, "p50_ms_per_pass": [ 244.489878998138, 244.49402201571502, 244.46565899415873 ], "p95_ms": 247.25005190703087 }, "k@20": { "mean_ms": 39.66955885019464, "n": 915, "p50_ms": 41.41516599338502, "p50_ms_per_pass": [ 41.41516599338502, 41.490216011879966, 41.36783702415414 ], "p95_ms": 50.12729699665215 }, "k@50": { "mean_ms": 112.82657481716625, "n": 915, "p50_ms": 123.08858399046585, "p50_ms_per_pass": [ 122.99052599701099, 123.09000597451814, 123.14969999715686 ], "p95_ms": 125.39473618380725 }, "run_id": "latency-gte-base-cuda-desktop-se-20261011T043900Z-c4aece" }, "jina-v3.5": { "created_utc": "2026-10-11T04:12:21+00:00", "k@100": { "mean_ms": 3095.477631104546, "n": 915, "p50_ms": 2957.4151929991785, "p50_ms_per_pass": [ 2956.885023013456, 2957.7737099898513, 2957.4151929991785 ], "p95_ms": 5466.718614599085 }, "k@20": { "mean_ms": 203.27834560344368, "n": 915, "p50_ms": 186.30112300161272, "p50_ms_per_pass": [ 186.18972599506378, 186.40104599762708, 186.34896699222736 ], "p95_ms": 337.6763823936926 }, "k@50": { "mean_ms": 905.8828080700242, "n": 915, "p50_ms": 835.7520879944786, "p50_ms_per_pass": [ 835.4137660062406, 835.9546060091816, 835.2819999854546 ], "p95_ms": 1588.7781331024598 }, "run_id": "latency-jina-v3.5-cuda-desktop-se-20261011T041221Z-de1b7f" }, "mxbai-large-v2": { "created_utc": "2026-10-11T03:06:22+00:00", "k@100": { "mean_ms": 2280.3095584527573, "n": 915, "p50_ms": 2095.3549850091804, "p50_ms_per_pass": [ 2094.8534989729524, 2095.3549850091804, 2096.0386439983267 ], "p95_ms": 4342.567619209876 }, "k@20": { "mean_ms": 570.5370559343429, "n": 915, "p50_ms": 515.1620070100762, "p50_ms_per_pass": [ 515.0743809936102, 515.1913350273389, 514.9787080008537 ], "p95_ms": 996.3384669885272 }, "k@50": { "mean_ms": 1355.6866292745992, "n": 915, "p50_ms": 1220.2538960264064, "p50_ms_per_pass": [ 1221.4215340209194, 1220.2538960264064, 1219.4071479898412 ], "p95_ms": 2221.4463036943926 }, "run_id": "latency-mxbai-large-v2-cuda-desktop-se-20261011T030622Z-f837da" }, "querit-4b": { "created_utc": "2026-10-11T00:12:54+00:00", "k@100": { "mean_ms": 13128.199957737883, "n": 915, "p50_ms": 13118.437324999832, "p50_ms_per_pass": [ 13125.792626990005, 13120.229057996767, 13108.918849990005 ], "p95_ms": 13134.274961202755 }, "k@20": { "mean_ms": 2648.022718227162, "n": 915, "p50_ms": 2639.4875170080923, "p50_ms_per_pass": [ 2644.4881490024272, 2641.3081909995526, 2637.334397993982 ], "p95_ms": 2687.7645314962137 }, "k@50": { "mean_ms": 6586.41369046368, "n": 915, "p50_ms": 6578.078530001221, "p50_ms_per_pass": [ 6576.649719005218, 6576.906135014724, 6581.500080006663 ], "p95_ms": 6651.63715671515 }, "run_id": "latency-querit-4b-cuda-desktop-se-20261011T001254Z-186e05" }, "qwen3-0.6b": { "created_utc": "2026-10-11T08:55:31+00:00", "k@100": { "mean_ms": 1293.469584586011, "n": 915, "p50_ms": 1166.8686180200893, "p50_ms_per_pass": [ 1175.879354996141, 1163.237714994466, 1146.6391159920022 ], "p95_ms": 2647.571201398386 }, "k@20": { "mean_ms": 372.18069782848926, "n": 915, "p50_ms": 322.7822380140424, "p50_ms_per_pass": [ 326.3335970113985, 322.7822380140424, 319.6459099999629 ], "p95_ms": 692.6114955916995 }, "k@50": { "mean_ms": 768.7205537607197, "n": 915, "p50_ms": 693.5476949729491, "p50_ms_per_pass": [ 688.9273619744927, 685.8392379945144, 694.2495509865694 ], "p95_ms": 1268.9767349831525 }, "run_id": "latency-qwen3-0.6b-cuda-desktop-se-20261011T085531Z-bfd2bd" }, "qwen3-4b": { "created_utc": "2026-10-11T02:00:21+00:00", "k@100": { "mean_ms": 3889.2949946447657, "n": 915, "p50_ms": 3799.245742993662, "p50_ms_per_pass": [ 3797.9197569948155, 3799.245742993662, 3800.098120002076 ], "p95_ms": 5509.06642670161 }, "k@20": { "mean_ms": 924.250372670956, "n": 915, "p50_ms": 864.0762289869599, "p50_ms_per_pass": [ 864.396727993153, 863.6506010079756, 864.0762289869599 ], "p95_ms": 1418.8344406022225 }, "k@50": { "mean_ms": 2047.7207791860167, "n": 915, "p50_ms": 1976.3904010178521, "p50_ms_per_pass": [ 1974.6302219864447, 1976.3904010178521, 1976.4167079993058 ], "p95_ms": 2918.79690430942 }, "run_id": "latency-qwen3-4b-cuda-desktop-se-20261011T020021Z-aa5503" } } }, "measured": "2026-10-11", "measured_from": "2026-10-09", "models": { "bge-v2-m3": { "architecture": "encoder", "commercial_use": "yes", "contamination": [ "MIRACLReranking", "MIRACLRetrieval", "MIRACLRetrievalHardNegatives" ], "contamination_known": true, "hf_id": "BAAI/bge-reranker-v2-m3", "license": "Apache-2.0", "params_m": 568, "rated": true, "revision": "953dc6f6f85a1b2dbfca4c34a2796e7dde08d41e", "validation": { "deciding_reference": "model_card", "mean_spearman": 0.9996338560136244, "min_spearman": 0.9931991809667617, "n_queries": 20, "passed": true, "reason": null, "run_id": "validation-bge-v2-m3-20261010T033230Z-0cbba3", "status": "ok", "threshold": 0.99 } }, "gte-base": { "architecture": "encoder", "commercial_use": "yes", "contamination": [], "contamination_known": false, "hf_id": "Alibaba-NLP/gte-multilingual-reranker-base", "license": "Apache-2.0", "params_m": 306, "rated": true, "revision": "8215cf04918ba6f7b6a62bb44238ce2953d8831c", "validation": { "deciding_reference": "transformers457", "mean_spearman": 0.9999698466367711, "min_spearman": 0.9998829859538008, "n_queries": 20, "passed": true, "reason": null, "run_id": "validation-gte-base-20261010T033154Z-7ae9bd", "status": "ok", "threshold": 0.99 } }, "jina-v3.5": { "architecture": "listwise", "commercial_use": "license required", "contamination": [], "contamination_known": false, "hf_id": "jinaai/jina-reranker-v3.5", "license": "CC-BY-NC-4.0", "params_m": 600, "rated": true, "revision": "e8a93f33f0b22108f8c2364f8484ce3422552fbc", "validation": { "deciding_reference": "transformers457", "mean_spearman": 0.9999999999999998, "min_spearman": 0.9999999999999999, "n_queries": 20, "passed": true, "reason": null, "run_id": "validation-jina-v3.5-20261010T033454Z-79be40", "status": "ok", "threshold": 0.99 } }, "mxbai-large-v2": { "architecture": "llm", "commercial_use": "yes", "contamination": [], "contamination_known": true, "hf_id": "mixedbread-ai/mxbai-rerank-large-v2", "license": "Apache-2.0", "params_m": 1500, "rated": true, "revision": "ca7e1ee484c37c0ddd8d178a9a5c33cec575c5e6", "validation": { "deciding_reference": "vendor_package", "mean_spearman": 0.9991151682705217, "min_spearman": 0.9965865768232341, "n_queries": 20, "passed": true, "reason": null, "run_id": "validation-mxbai-large-v2-20261010T040457Z-eac0e2", "status": "ok", "threshold": 0.99 } }, "querit-4b": { "architecture": "llm", "commercial_use": "yes", "contamination": [ "MIRACLReranking" ], "contamination_known": true, "hf_id": "Querit/Querit-4B", "license": "Apache-2.0 (model metadata only)", "params_m": 4000, "rated": true, "revision": "e557bb5ce6555f16b33dbb7d2ca62e8d27c14c13", "validation": { "deciding_reference": "transformers457", "mean_spearman": 1.0, "min_spearman": 0.9999999999999999, "n_queries": 20, "passed": true, "reason": null, "run_id": "validation-querit-4b-20261010T035119Z-0e6de8", "status": "ok", "threshold": 0.99 } }, "qwen3-0.6b": { "architecture": "llm", "commercial_use": "yes", "contamination": [ "MIRACLReranking" ], "contamination_known": true, "hf_id": "Qwen/Qwen3-Reranker-0.6B", "license": "Apache-2.0", "params_m": 600, "rated": true, "revision": "e61197ed45024b0ed8a2d74b80b4d909f1255473", "validation": { "deciding_reference": "model_card", "mean_spearman": 0.9999303916435401, "min_spearman": 0.999741967270338, "n_queries": 20, "passed": true, "reason": null, "run_id": "validation-qwen3-0.6b-20261010T033614Z-8c7b96", "status": "ok", "threshold": 0.99 } }, "qwen3-4b": { "architecture": "llm", "commercial_use": "yes", "contamination": [ "MIRACLReranking" ], "contamination_known": true, "hf_id": "Qwen/Qwen3-Reranker-4B", "license": "Apache-2.0", "params_m": 4000, "rated": true, "revision": "22e683669bc0f0bd69640a1354a6d0aebcfeede5", "validation": { "deciding_reference": "model_card", "mean_spearman": 0.9999260397248848, "min_spearman": 0.9995799304304103, "n_queries": 20, "passed": true, "reason": null, "run_id": "validation-qwen3-4b-20261010T034214Z-70f71a", "status": "ok", "threshold": 0.99 } }, "voyage-rerank-3": { "architecture": "api", "commercial_use": "yes (paid API)", "contamination": [], "contamination_known": false, "hf_id": "", "license": "Voyage AI terms of service", "params_m": 0, "rated": true, "revision": "2026-10-10", "validation": null }, "voyage-rerank-3-lite": { "architecture": "api", "commercial_use": "yes (paid API)", "contamination": [], "contamination_known": false, "hf_id": "", "license": "Voyage AI terms of service", "params_m": 0, "rated": true, "revision": "2026-10-10", "validation": null } }, "pitfall": { "checks": { "exploratory": { "vs_keyword-71k": [ "keyword-71k-ltr" ] }, "holm_families": { "vs_correct": [ "no-instruction", "right-padding", "max256-correct", "max256-tail-cut" ], "vs_keyword-plain-71k": [ "keyword-71k" ] }, "manual_correct_vs_sentence_transformers_points": 0.05088816735001056, "manual_correct_vs_sentence_transformers_tolerance_points": 0.2, "sentence_transformers_result": "quality-hybrid-71k--qwen3-0.6b-20261010T140052Z-630b08" }, "created_utc": "2026-10-10T16:21:29+00:00", "metrics": { "qwen3": { "correct": { "ndcg@10": { "ci_high": 0.40752976091595533, "ci_low": 0.3219647843769938, "mean": 0.3651793320649692 } }, "max256-correct": { "delta_vs_correct_points": -0.678963179482378, "ndcg@10": { "ci_high": 0.39976499294660767, "ci_low": 0.3162039922945238, "mean": 0.35838970027014544 }, "p_vs_correct_holm": 0.04279572042795721, "wording_vs_correct": "measurably worse" }, "max256-tail-cut": { "delta_vs_correct_points": -16.029970456824746, "ndcg@10": { "ci_high": 0.2390141440629392, "ci_low": 0.1716763257799117, "mean": 0.20487962749672176 }, "p_vs_correct_holm": 0.00039996000399960006, "wording_vs_correct": "clearly worse" }, "no-instruction": { "delta_vs_correct_points": 1.0376198573177065, "ndcg@10": { "ci_high": 0.4182490407665978, "ci_low": 0.33147474003716565, "mean": 0.3755555306381463 }, "p_vs_correct_holm": 0.04279572042795721, "wording_vs_correct": "measurably better" }, "right-padding": { "delta_vs_correct_points": -22.946530706613004, "ndcg@10": { "ci_high": 0.16331167507635358, "ci_low": 0.10991854835400777, "mean": 0.13571402499883917 }, "p_vs_correct_holm": 0.00039996000399960006, "wording_vs_correct": "clearly worse" } }, "typesense": { "delta_points": 6.216380471544366, "delta_vs_keyword-71k_points": 4.817282808766523, "exploratory": true, "keyword-71k": { "ci_high": 0.09605649644661068, "ci_low": 0.05430560617418975, "mean": 0.07428238494756044 }, "keyword-71k-ltr": { "ci_high": 0.14879409253958079, "ci_low": 0.09836524572260068, "mean": 0.12245521303522566 }, "keyword-plain-71k": { "ci_high": 0.020784422170357714, "ci_low": 0.004857981133367159, "mean": 0.012118580232116774 }, "p": 9.999000099990002e-05, "p_vs_keyword-71k": 9.999000099990002e-05 } }, "model": "qwen3-0.6b", "run_id": "pitfall-qwen3-0.6b-and-typesense-20261010T162129Z-e19e60", "stage": "hybrid-71k" }, "plan": { "commit": "dec01291a2ce67e259385f3a69145b6b5f041f50", "timestamp_file": "PLAN.md.ots" }, "quality": { "hybrid-71k": { "best": "qwen3-4b", "created_utc": "2026-10-12T08:05:07+00:00", "models": { "bge-v2-m3": { "comparison": { "contamination": [ "MIRACLReranking", "MIRACLRetrieval", "MIRACLRetrievalHardNegatives" ], "contamination_known": true, "delta_best_minus_model_points": 1.8893224920613128, "delta_vs_first_stage_points": 18.51004628990879, "ndcg@10": 0.36008335712152306, "p_vs_best_holm": 0.0221977802219778, "p_vs_first_stage_holm": 0.0008999100089991002, "wording_best_vs_model": "measurably better", "wording_vs_first_stage": "clearly better" }, "created_utc": "2026-10-10T14:00:51+00:00", "depth@100": { "hit@10": { "mean": 0.5344262295081967 }, "hit@5": { "ci_high": 0.5672131147540984, "ci_low": 0.4557377049180328, "mean": 0.5114754098360655 }, "judged@10": { "mean": 0.2318032786885246 }, "mrr@10": { "mean": 0.4143377569607078 }, "ndcg@10": { "ci_high": 0.40337042214914215, "ci_low": 0.316580551630309, "mean": 0.36008335712152306 }, "ndcg_condensed@10": { "mean": 0.4158026723259443 }, "recall@10": { "mean": 0.4077985948477752 }, "recall@100": { "mean": 0.4350039032006245 }, "recall@20": { "mean": 0.4246760343481655 }, "recall@5": { "mean": 0.36545667447306796 } }, "depth@20": { "hit@10": { "mean": 0.4852459016393443 }, "hit@5": { "ci_high": 0.5344262295081967, "ci_low": 0.42295081967213116, "mean": 0.4786885245901639 }, "judged@10": { "mean": 0.1714754098360656 }, "mrr@10": { "mean": 0.402080405932865 }, "ndcg@10": { "ci_high": 0.3572896929905259, "ci_low": 0.27448655465035726, "mean": 0.31570222031762757 }, "ndcg_condensed@10": { "mean": 0.4049005899749167 }, "recall@10": { "mean": 0.32546448087431695 }, "recall@100": { "mean": 0.4350039032006245 }, "recall@20": { "mean": 0.32904761904761903 }, "recall@5": { "mean": 0.3121779859484778 } }, "depth@50": { "hit@10": { "mean": 0.521311475409836 }, "hit@5": { "ci_high": 0.5573770491803278, "ci_low": 0.4459016393442623, "mean": 0.5016393442622951 }, "judged@10": { "mean": 0.21311475409836064 }, "mrr@10": { "mean": 0.4124173822534478 }, "ndcg@10": { "ci_high": 0.3859164073897641, "ci_low": 0.3011918683440086, "mean": 0.34362146924940334 }, "ndcg_condensed@10": { "mean": 0.4133391543621477 }, "recall@10": { "mean": 0.3764090554254489 }, "recall@100": { "mean": 0.4350039032006245 }, "recall@20": { "mean": 0.38811865729898515 }, "recall@5": { "mean": 0.34523809523809523 } }, "run_id": "quality-hybrid-71k--bge-v2-m3-20261010T140051Z-46dcf5" }, "gte-base": { "comparison": { "contamination": [], "contamination_known": false, "delta_best_minus_model_points": 4.072329027371851, "delta_vs_first_stage_points": 16.32703975459825, "ndcg@10": 0.3382532917684177, "p_vs_best_holm": 0.0007999200079992001, "p_vs_first_stage_holm": 0.0008999100089991002, "wording_best_vs_model": "clearly better", "wording_vs_first_stage": "clearly better" }, "created_utc": "2026-10-10T14:00:51+00:00", "depth@100": { "hit@10": { "mean": 0.5180327868852459 }, "hit@5": { "ci_high": 0.5409836065573771, "ci_low": 0.42950819672131146, "mean": 0.4852459016393443 }, "judged@10": { "mean": 0.2249180327868852 }, "mrr@10": { "mean": 0.38870543845953687 }, "ndcg@10": { "ci_high": 0.380730736331945, "ci_low": 0.2965280656652474, "mean": 0.3382532917684177 }, "ndcg_condensed@10": { "mean": 0.403892805556557 }, "recall@10": { "mean": 0.3869242779078845 }, "recall@100": { "mean": 0.4350039032006245 }, "recall@20": { "mean": 0.40893052302888366 }, "recall@5": { "mean": 0.35117096018735366 } }, "depth@20": { "hit@10": { "mean": 0.4786885245901639 }, "hit@5": { "ci_high": 0.521311475409836, "ci_low": 0.4098360655737705, "mean": 0.46557377049180326 }, "judged@10": { "mean": 0.17049180327868851 }, "mrr@10": { "mean": 0.38132058287795995 }, "ndcg@10": { "ci_high": 0.3431720353876738, "ci_low": 0.2616289319406668, "mean": 0.30222054605140375 }, "ndcg_condensed@10": { "mean": 0.3986028532803199 }, "recall@10": { "mean": 0.3196955503512881 }, "recall@100": { "mean": 0.4350039032006245 }, "recall@20": { "mean": 0.32904761904761903 }, "recall@5": { "mean": 0.3023965651834504 } }, "depth@50": { "hit@10": { "mean": 0.5147540983606558 }, "hit@5": { "ci_high": 0.5344262295081967, "ci_low": 0.42295081967213116, "mean": 0.4786885245901639 }, "judged@10": { "mean": 0.21114754098360658 }, "mrr@10": { "mean": 0.38910616705698675 }, "ndcg@10": { "ci_high": 0.3672679305029315, "ci_low": 0.28545287514836065, "mean": 0.32616612826363844 }, "ndcg_condensed@10": { "mean": 0.40368732179906697 }, "recall@10": { "mean": 0.3643871975019516 }, "recall@100": { "mean": 0.4350039032006245 }, "recall@20": { "mean": 0.3842622950819672 }, "recall@5": { "mean": 0.3289305230288837 } }, "run_id": "quality-hybrid-71k--gte-base-20261010T140051Z-7180df" }, "jina-v3.5": { "comparison": { "contamination": [], "contamination_known": false, "delta_best_minus_model_points": 0.2373453642387724, "delta_vs_first_stage_points": 20.16202341773133, "ndcg@10": 0.37660312839974847, "p_vs_best_holm": 1.0, "p_vs_first_stage_holm": 0.0008999100089991002, "wording_best_vs_model": "on par", "wording_vs_first_stage": "clearly better" }, "created_utc": "2026-10-10T14:00:52+00:00", "depth@100": { "hit@10": { "mean": 0.5409836065573771 }, "hit@5": { "ci_high": 0.5737704918032787, "ci_low": 0.46229508196721314, "mean": 0.5180327868852459 }, "judged@10": { "mean": 0.22426229508196724 }, "mrr@10": { "mean": 0.4402172781680978 }, "ndcg@10": { "ci_high": 0.42111354523236333, "ci_low": 0.33174636796680995, "mean": 0.37660312839974847 }, "ndcg_condensed@10": { "mean": 0.42716691736164847 }, "recall@10": { "mean": 0.41007025761124116 }, "recall@100": { "mean": 0.4350039032006245 }, "recall@20": { "mean": 0.426448087431694 }, "recall@5": { "mean": 0.3731693989071038 } }, "depth@20": { "hit@10": { "mean": 0.4852459016393443 }, "hit@5": { "ci_high": 0.5344262295081967, "ci_low": 0.42295081967213116, "mean": 0.4786885245901639 }, "judged@10": { "mean": 0.17114754098360657 }, "mrr@10": { "mean": 0.4123263075722092 }, "ndcg@10": { "ci_high": 0.3653403073400263, "ci_low": 0.28081773832357393, "mean": 0.32304259505980254 }, "ndcg_condensed@10": { "mean": 0.4142163186886747 }, "recall@10": { "mean": 0.3262060889929742 }, "recall@100": { "mean": 0.4350039032006245 }, "recall@20": { "mean": 0.32904761904761903 }, "recall@5": { "mean": 0.31616705698672914 } }, "depth@50": { "hit@10": { "mean": 0.5311475409836065 }, "hit@5": { "ci_high": 0.5672131147540984, "ci_low": 0.4557377049180328, "mean": 0.5114754098360655 }, "judged@10": { "mean": 0.21114754098360658 }, "mrr@10": { "mean": 0.42845563361956807 }, "ndcg@10": { "ci_high": 0.39856321936303185, "ci_low": 0.31336769276708204, "mean": 0.35632762512353494 }, "ndcg_condensed@10": { "mean": 0.42184864023118657 }, "recall@10": { "mean": 0.382263856362217 }, "recall@100": { "mean": 0.4350039032006245 }, "recall@20": { "mean": 0.3947853239656518 }, "recall@5": { "mean": 0.35393442622950816 } }, "run_id": "quality-hybrid-71k--jina-v3.5-20261010T140052Z-7a3767" }, "mxbai-large-v2": { "comparison": { "contamination": [], "contamination_known": true, "delta_best_minus_model_points": 4.810090219205715, "delta_vs_first_stage_points": 15.589278562764386, "ndcg@10": 0.33087567985007904, "p_vs_best_holm": 0.0007999200079992001, "p_vs_first_stage_holm": 0.0008999100089991002, "wording_best_vs_model": "clearly better", "wording_vs_first_stage": "clearly better" }, "created_utc": "2026-10-10T14:00:52+00:00", "depth@100": { "hit@10": { "mean": 0.5278688524590164 }, "hit@5": { "ci_high": 0.5377049180327869, "ci_low": 0.4262295081967213, "mean": 0.4819672131147541 }, "judged@10": { "mean": 0.20852459016393446 }, "mrr@10": { "mean": 0.37970465781941193 }, "ndcg@10": { "ci_high": 0.37178025675123, "ci_low": 0.2896679045818184, "mean": 0.33087567985007904 }, "ndcg_condensed@10": { "mean": 0.4180121281279578 }, "recall@10": { "mean": 0.3938095238095238 }, "recall@100": { "mean": 0.4350039032006245 }, "recall@20": { "mean": 0.41797814207650275 }, "recall@5": { "mean": 0.3360421545667447 } }, "depth@20": { "hit@10": { "mean": 0.4852459016393443 }, "hit@5": { "ci_high": 0.521311475409836, "ci_low": 0.4098360655737705, "mean": 0.46557377049180326 }, "judged@10": { "mean": 0.16852459016393445 }, "mrr@10": { "mean": 0.3813505074160812 }, "ndcg@10": { "ci_high": 0.34029516991695286, "ci_low": 0.26025747582144254, "mean": 0.3003036590571089 }, "ndcg_condensed@10": { "mean": 0.40515290123240477 }, "recall@10": { "mean": 0.3251912568306011 }, "recall@100": { "mean": 0.4350039032006245 }, "recall@20": { "mean": 0.32904761904761903 }, "recall@5": { "mean": 0.3003590944574551 } }, "depth@50": { "hit@10": { "mean": 0.521311475409836 }, "hit@5": { "ci_high": 0.5442622950819672, "ci_low": 0.43278688524590164, "mean": 0.4885245901639344 }, "judged@10": { "mean": 0.20032786885245904 }, "mrr@10": { "mean": 0.3821194379391101 }, "ndcg@10": { "ci_high": 0.3601186851851023, "ci_low": 0.2798782735736859, "mean": 0.3204013317761722 }, "ndcg_condensed@10": { "mean": 0.41301599836340525 }, "recall@10": { "mean": 0.37220140515222483 }, "recall@100": { "mean": 0.4350039032006245 }, "recall@20": { "mean": 0.3867213114754098 }, "recall@5": { "mean": 0.33079625292740045 } }, "run_id": "quality-hybrid-71k--mxbai-large-v2-20261010T140052Z-b98d5c" }, "querit-4b": { "comparison": { "contamination": [ "MIRACLReranking" ], "contamination_known": true, "delta_best_minus_model_points": 1.6730573537623195, "delta_vs_first_stage_points": 18.726311428207783, "ndcg@10": 0.362246008504513, "p_vs_best_holm": 0.098990100989901, "p_vs_first_stage_holm": 0.0008999100089991002, "wording_best_vs_model": "on par", "wording_vs_first_stage": "clearly better" }, "created_utc": "2026-10-10T14:00:53+00:00", "depth@100": { "hit@10": { "mean": 0.5245901639344263 }, "hit@5": { "ci_high": 0.5540983606557377, "ci_low": 0.4426229508196721, "mean": 0.49836065573770494 }, "judged@10": { "mean": 0.22131147540983606 }, "mrr@10": { "mean": 0.42163283892792086 }, "ndcg@10": { "ci_high": 0.40600109820040436, "ci_low": 0.3185421184349729, "mean": 0.362246008504513 }, "ndcg_condensed@10": { "mean": 0.4289074217349377 }, "recall@10": { "mean": 0.39870413739266203 }, "recall@100": { "mean": 0.4350039032006245 }, "recall@20": { "mean": 0.4190944574551132 }, "recall@5": { "mean": 0.35725995316159254 } }, "depth@20": { "hit@10": { "mean": 0.4786885245901639 }, "hit@5": { "ci_high": 0.521311475409836, "ci_low": 0.4098360655737705, "mean": 0.46557377049180326 }, "judged@10": { "mean": 0.16983606557377048 }, "mrr@10": { "mean": 0.40784933645589383 }, "ndcg@10": { "ci_high": 0.35808383056892196, "ci_low": 0.274296837022535, "mean": 0.3161083815907691 }, "ndcg_condensed@10": { "mean": 0.41400721211315655 }, "recall@10": { "mean": 0.320624512099922 }, "recall@100": { "mean": 0.4350039032006245 }, "recall@20": { "mean": 0.32904761904761903 }, "recall@5": { "mean": 0.3048711943793911 } }, "depth@50": { "hit@10": { "mean": 0.5180327868852459 }, "hit@5": { "ci_high": 0.5475409836065573, "ci_low": 0.4360655737704918, "mean": 0.4918032786885246 }, "judged@10": { "mean": 0.2081967213114754 }, "mrr@10": { "mean": 0.4183827738745772 }, "ndcg@10": { "ci_high": 0.3901997491647364, "ci_low": 0.3041222086047349, "mean": 0.3470618053247245 }, "ndcg_condensed@10": { "mean": 0.42439398088919744 }, "recall@10": { "mean": 0.3737314597970336 }, "recall@100": { "mean": 0.4350039032006245 }, "recall@20": { "mean": 0.3893989071038251 }, "recall@5": { "mean": 0.33794691647150665 } }, "run_id": "quality-hybrid-71k--querit-4b-20261010T140053Z-1a1656" }, "qwen3-0.6b": { "comparison": { "contamination": [ "MIRACLReranking" ], "contamination_known": true, "delta_best_minus_model_points": 1.4306131650667075, "delta_vs_first_stage_points": 18.968755616903394, "ndcg@10": 0.3646704503914691, "p_vs_best_holm": 0.18198180181981802, "p_vs_first_stage_holm": 0.0008999100089991002, "wording_best_vs_model": "on par", "wording_vs_first_stage": "clearly better" }, "created_utc": "2026-10-10T14:00:52+00:00", "depth@100": { "hit@10": { "mean": 0.5409836065573771 }, "hit@5": { "ci_high": 0.5639344262295082, "ci_low": 0.4524590163934426, "mean": 0.5081967213114754 }, "judged@10": { "mean": 0.23278688524590163 }, "mrr@10": { "mean": 0.42804189435336976 }, "ndcg@10": { "ci_high": 0.4069485885629406, "ci_low": 0.3214945232820251, "mean": 0.3646704503914691 }, "ndcg_condensed@10": { "mean": 0.4140002262898512 }, "recall@10": { "mean": 0.4091569086651054 }, "recall@100": { "mean": 0.4350039032006245 }, "recall@20": { "mean": 0.43067915690866504 }, "recall@5": { "mean": 0.36669008587041374 } }, "depth@20": { "hit@10": { "mean": 0.4819672131147541 }, "hit@5": { "ci_high": 0.5311475409836065, "ci_low": 0.419672131147541, "mean": 0.47540983606557374 }, "judged@10": { "mean": 0.1737704918032787 }, "mrr@10": { "mean": 0.41082357533177205 }, "ndcg@10": { "ci_high": 0.3591312949496711, "ci_low": 0.27497392121112196, "mean": 0.3168942360370384 }, "ndcg_condensed@10": { "mean": 0.4055902213220764 }, "recall@10": { "mean": 0.32152224824355974 }, "recall@100": { "mean": 0.4350039032006245 }, "recall@20": { "mean": 0.32904761904761903 }, "recall@5": { "mean": 0.31192818110850895 } }, "depth@50": { "hit@10": { "mean": 0.5311475409836065 }, "hit@5": { "ci_high": 0.5639344262295082, "ci_low": 0.4524590163934426, "mean": 0.5081967213114754 }, "judged@10": { "mean": 0.21672131147540985 }, "mrr@10": { "mean": 0.4275956284153005 }, "ndcg@10": { "ci_high": 0.3924800131497411, "ci_low": 0.3087793397717643, "mean": 0.35126229657161756 }, "ndcg_condensed@10": { "mean": 0.41356988927259075 }, "recall@10": { "mean": 0.3814832162373146 }, "recall@100": { "mean": 0.4350039032006245 }, "recall@20": { "mean": 0.39377049180327867 }, "recall@5": { "mean": 0.35076502732240433 } }, "run_id": "quality-hybrid-71k--qwen3-0.6b-20261010T140052Z-630b08" }, "qwen3-4b": { "comparison": { "contamination": [ "MIRACLReranking" ], "contamination_known": true, "delta_vs_first_stage_points": 20.3993687819701, "ndcg@10": 0.3789765820421362, "p_vs_first_stage_holm": 0.0008999100089991002, "wording_vs_first_stage": "clearly better" }, "created_utc": "2026-10-10T14:00:52+00:00", "depth@100": { "hit@10": { "mean": 0.5377049180327869 }, "hit@5": { "ci_high": 0.5737704918032787, "ci_low": 0.46229508196721314, "mean": 0.5180327868852459 }, "judged@10": { "mean": 0.2249180327868852 }, "mrr@10": { "mean": 0.43776346604215455 }, "ndcg@10": { "ci_high": 0.42327084848945346, "ci_low": 0.3335835133656596, "mean": 0.3789765820421362 }, "ndcg_condensed@10": { "mean": 0.43030236372933583 }, "recall@10": { "mean": 0.4120296643247463 }, "recall@100": { "mean": 0.4350039032006245 }, "recall@20": { "mean": 0.4311943793911008 }, "recall@5": { "mean": 0.3769086651053864 } }, "depth@20": { "hit@10": { "mean": 0.4852459016393443 }, "hit@5": { "ci_high": 0.5344262295081967, "ci_low": 0.42295081967213116, "mean": 0.4786885245901639 }, "judged@10": { "mean": 0.1718032786885246 }, "mrr@10": { "mean": 0.41980874316939887 }, "ndcg@10": { "ci_high": 0.36818598598312297, "ci_low": 0.28345387256428095, "mean": 0.32587402485603323 }, "ndcg_condensed@10": { "mean": 0.41296471351181036 }, "recall@10": { "mean": 0.32701795472287276 }, "recall@100": { "mean": 0.4350039032006245 }, "recall@20": { "mean": 0.32904761904761903 }, "recall@5": { "mean": 0.31575331772053084 } }, "depth@50": { "hit@10": { "mean": 0.5311475409836065 }, "hit@5": { "ci_high": 0.5704918032786885, "ci_low": 0.45901639344262296, "mean": 0.5147540983606558 }, "judged@10": { "mean": 0.20885245901639346 }, "mrr@10": { "mean": 0.4353135571168357 }, "ndcg@10": { "ci_high": 0.40605130097266307, "ci_low": 0.3181130532518667, "mean": 0.36252903677420156 }, "ndcg_condensed@10": { "mean": 0.4261417451192491 }, "recall@10": { "mean": 0.385480093676815 }, "recall@100": { "mean": 0.4350039032006245 }, "recall@20": { "mean": 0.39439500390320065 }, "recall@5": { "mean": 0.35669008587041373 } }, "run_id": "quality-hybrid-71k--qwen3-4b-20261010T140052Z-ea48ec" }, "voyage-rerank-3": { "comparison": { "contamination": [], "contamination_known": false, "delta_best_minus_model_points": 0.02168988170075381, "delta_vs_first_stage_points": 20.37767890026935, "ndcg@10": 0.37875968322512865, "p_vs_best_holm": 1.0, "p_vs_first_stage_holm": 0.0008999100089991002, "wording_best_vs_model": "on par", "wording_vs_first_stage": "clearly better" }, "created_utc": "2026-10-10T14:00:53+00:00", "depth@100": { "hit@10": { "mean": 0.5344262295081967 }, "hit@5": { "ci_high": 0.5639344262295082, "ci_low": 0.4524590163934426, "mean": 0.5081967213114754 }, "judged@10": { "mean": 0.22098360655737706 }, "mrr@10": { "mean": 0.4371935987509758 }, "ndcg@10": { "ci_high": 0.42435078803782683, "ci_low": 0.3338204410058941, "mean": 0.37875968322512865 }, "ndcg_condensed@10": { "mean": 0.43627723931454465 }, "recall@10": { "mean": 0.41048399687743947 }, "recall@100": { "mean": 0.4350039032006245 }, "recall@20": { "mean": 0.4251912568306011 }, "recall@5": { "mean": 0.371928181108509 } }, "depth@20": { "hit@10": { "mean": 0.4852459016393443 }, "hit@5": { "ci_high": 0.5344262295081967, "ci_low": 0.42295081967213116, "mean": 0.4786885245901639 }, "judged@10": { "mean": 0.16950819672131148 }, "mrr@10": { "mean": 0.41112412177985946 }, "ndcg@10": { "ci_high": 0.36532199743824795, "ci_low": 0.2797767348359059, "mean": 0.32233633333764605 }, "ndcg_condensed@10": { "mean": 0.4167127961661786 }, "recall@10": { "mean": 0.3253005464480874 }, "recall@100": { "mean": 0.4350039032006245 }, "recall@20": { "mean": 0.32904761904761903 }, "recall@5": { "mean": 0.3159953161592506 } }, "depth@50": { "hit@10": { "mean": 0.5278688524590164 }, "hit@5": { "ci_high": 0.5540983606557377, "ci_low": 0.4426229508196721, "mean": 0.49836065573770494 }, "judged@10": { "mean": 0.20622950819672134 }, "mrr@10": { "mean": 0.42998959146500126 }, "ndcg@10": { "ci_high": 0.40108130856867885, "ci_low": 0.3139666211240512, "mean": 0.357706963347204 }, "ndcg_condensed@10": { "mean": 0.429391218382141 }, "recall@10": { "mean": 0.37971896955503515 }, "recall@100": { "mean": 0.4350039032006245 }, "recall@20": { "mean": 0.3910694769711163 }, "recall@5": { "mean": 0.34988290398126465 } }, "run_id": "quality-hybrid-71k--voyage-rerank-3-20261010T140053Z-7dfa4f" }, "voyage-rerank-3-lite": { "comparison": { "contamination": [], "contamination_known": false, "delta_best_minus_model_points": 1.259636873862946, "delta_vs_first_stage_points": 19.139731908107155, "ndcg@10": 0.36638021330350673, "p_vs_best_holm": 0.3020697930206979, "p_vs_first_stage_holm": 0.0008999100089991002, "wording_best_vs_model": "on par", "wording_vs_first_stage": "clearly better" }, "created_utc": "2026-10-10T14:00:53+00:00", "depth@100": { "hit@10": { "mean": 0.5344262295081967 }, "hit@5": { "ci_high": 0.5573770491803278, "ci_low": 0.4459016393442623, "mean": 0.5016393442622951 }, "judged@10": { "mean": 0.219016393442623 }, "mrr@10": { "mean": 0.42319021597710127 }, "ndcg@10": { "ci_high": 0.410862503601567, "ci_low": 0.3224374035525684, "mean": 0.36638021330350673 }, "ndcg_condensed@10": { "mean": 0.4270414960062045 }, "recall@10": { "mean": 0.40626854020296643 }, "recall@100": { "mean": 0.4350039032006245 }, "recall@20": { "mean": 0.42535519125683063 }, "recall@5": { "mean": 0.3614988290398127 } }, "depth@20": { "hit@10": { "mean": 0.4852459016393443 }, "hit@5": { "ci_high": 0.5278688524590164, "ci_low": 0.4163934426229508, "mean": 0.4721311475409836 }, "judged@10": { "mean": 0.16852459016393442 }, "mrr@10": { "mean": 0.40027712724434034 }, "ndcg@10": { "ci_high": 0.35668888981512986, "ci_low": 0.27314854799720745, "mean": 0.3145864667230756 }, "ndcg_condensed@10": { "mean": 0.41226837421356566 }, "recall@10": { "mean": 0.32409836065573766 }, "recall@100": { "mean": 0.4350039032006245 }, "recall@20": { "mean": 0.32904761904761903 }, "recall@5": { "mean": 0.3097423887587822 } }, "depth@50": { "hit@10": { "mean": 0.5278688524590164 }, "hit@5": { "ci_high": 0.5475409836065573, "ci_low": 0.4360655737704918, "mean": 0.4918032786885246 }, "judged@10": { "mean": 0.20754098360655737 }, "mrr@10": { "mean": 0.4198842050481394 }, "ndcg@10": { "ci_high": 0.39251805545155116, "ci_low": 0.3065070697208379, "mean": 0.3493652399524116 }, "ndcg_condensed@10": { "mean": 0.4215704197265436 }, "recall@10": { "mean": 0.3781576893052303 }, "recall@100": { "mean": 0.4350039032006245 }, "recall@20": { "mean": 0.39030444964871197 }, "recall@5": { "mean": 0.34128024980483995 } }, "run_id": "quality-hybrid-71k--voyage-rerank-3-lite-20261010T140053Z-50bf1f" } }, "not_measured": {}, "run_id": "quality-summary-hybrid-71k-20261012T080507Z-95eada", "winner": null }, "keyword-71k": { "best": "qwen3-4b", "created_utc": "2026-10-12T08:05:07+00:00", "models": { "bge-v2-m3": { "comparison": { "contamination": [ "MIRACLReranking", "MIRACLRetrieval", "MIRACLRetrievalHardNegatives" ], "contamination_known": true, "delta_best_minus_model_points": 1.1284801096421548, "delta_vs_first_stage_points": 13.368436930619065, "ndcg@10": 0.20796675425375108, "p_vs_best_holm": 0.0917908209179082, "p_vs_first_stage_holm": 0.0008999100089991002, "wording_best_vs_model": "on par", "wording_vs_first_stage": "clearly better" }, "created_utc": "2026-10-10T14:00:47+00:00", "depth@100": { "hit@10": { "mean": 0.34098360655737703 }, "hit@5": { "ci_high": 0.3836065573770492, "ci_low": 0.2786885245901639, "mean": 0.33114754098360655 }, "judged@10": { "mean": 0.15180327868852458 }, "mrr@10": { "mean": 0.27283112151964606 }, "ndcg@10": { "ci_high": 0.2458495512308586, "ci_low": 0.17092672191628222, "mean": 0.20796675425375108 }, "ndcg_condensed@10": { "mean": 0.23699463564986453 }, "recall@10": { "mean": 0.21669008587041372 }, "recall@100": { "mean": 0.23757220921155345 }, "recall@20": { "mean": 0.2292661982825917 }, "recall@5": { "mean": 0.20192818110850896 } }, "depth@20": { "hit@10": { "mean": 0.25245901639344265 }, "hit@5": { "ci_high": 0.29508196721311475, "ci_low": 0.19672131147540983, "mean": 0.2459016393442623 }, "judged@10": { "mean": 0.0977049180327869 }, "mrr@10": { "mean": 0.2098360655737705 }, "ndcg@10": { "ci_high": 0.17978968256960412, "ci_low": 0.11630895432720136, "mean": 0.14698249246964376 }, "ndcg_condensed@10": { "mean": 0.2217611735343921 }, "recall@10": { "mean": 0.14241217798594846 }, "recall@100": { "mean": 0.23757220921155345 }, "recall@20": { "mean": 0.14451990632318498 }, "recall@5": { "mean": 0.1389383294301327 } }, "depth@50": { "hit@10": { "mean": 0.28852459016393445 }, "hit@5": { "ci_high": 0.32786885245901637, "ci_low": 0.2262295081967213, "mean": 0.2754098360655738 }, "judged@10": { "mean": 0.12590163934426232 }, "mrr@10": { "mean": 0.22826437678896697 }, "ndcg@10": { "ci_high": 0.20525158862484802, "ci_low": 0.13605064565715067, "mean": 0.1700005959710838 }, "ndcg_condensed@10": { "mean": 0.22726357369241995 }, "recall@10": { "mean": 0.17491022638563622 }, "recall@100": { "mean": 0.23757220921155345 }, "recall@20": { "mean": 0.186775956284153 }, "recall@5": { "mean": 0.1624199843871975 } }, "run_id": "quality-keyword-71k--bge-v2-m3-20261010T140047Z-7230a0" }, "gte-base": { "comparison": { "contamination": [], "contamination_known": false, "delta_best_minus_model_points": 1.6916138653092134, "delta_vs_first_stage_points": 12.805303174952003, "ndcg@10": 0.2023354166970805, "p_vs_best_holm": 0.0811918808119188, "p_vs_first_stage_holm": 0.0008999100089991002, "wording_best_vs_model": "on par", "wording_vs_first_stage": "clearly better" }, "created_utc": "2026-10-10T14:00:47+00:00", "depth@100": { "hit@10": { "mean": 0.34098360655737703 }, "hit@5": { "ci_high": 0.36721311475409835, "ci_low": 0.26229508196721313, "mean": 0.31475409836065577 }, "judged@10": { "mean": 0.14688524590163934 }, "mrr@10": { "mean": 0.27446526151444184 }, "ndcg@10": { "ci_high": 0.2396010385199934, "ci_low": 0.1660895361060079, "mean": 0.2023354166970805 }, "ndcg_condensed@10": { "mean": 0.23124379564285252 }, "recall@10": { "mean": 0.20865729898516786 }, "recall@100": { "mean": 0.23757220921155345 }, "recall@20": { "mean": 0.22311475409836065 }, "recall@5": { "mean": 0.18635441061670568 } }, "depth@20": { "hit@10": { "mean": 0.24918032786885247 }, "hit@5": { "ci_high": 0.28852459016393445, "ci_low": 0.19344262295081968, "mean": 0.23934426229508196 }, "judged@10": { "mean": 0.09868852459016395 }, "mrr@10": { "mean": 0.2094574551131928 }, "ndcg@10": { "ci_high": 0.17540194525595906, "ci_low": 0.1136425527212194, "mean": 0.14360815643714964 }, "ndcg_condensed@10": { "mean": 0.2186462434810603 }, "recall@10": { "mean": 0.13975800156128024 }, "recall@100": { "mean": 0.23757220921155345 }, "recall@20": { "mean": 0.14451990632318498 }, "recall@5": { "mean": 0.1314519906323185 } }, "depth@50": { "hit@10": { "mean": 0.28524590163934427 }, "hit@5": { "ci_high": 0.31475409836065577, "ci_low": 0.21639344262295082, "mean": 0.26557377049180325 }, "judged@10": { "mean": 0.12852459016393444 }, "mrr@10": { "mean": 0.23554514702055687 }, "ndcg@10": { "ci_high": 0.20294926428747556, "ci_low": 0.1349914195441349, "mean": 0.1680840305728503 }, "ndcg_condensed@10": { "mean": 0.22392731101674432 }, "recall@10": { "mean": 0.16950039032006242 }, "recall@100": { "mean": 0.23757220921155345 }, "recall@20": { "mean": 0.18065573770491805 }, "recall@5": { "mean": 0.15504293520686963 } }, "run_id": "quality-keyword-71k--gte-base-20261010T140047Z-9a9cec" }, "jina-v3.5": { "comparison": { "contamination": [], "contamination_known": false, "delta_best_minus_model_points": 0.31027737584549964, "delta_vs_first_stage_points": 14.186639664415718, "ndcg@10": 0.21614878159171763, "p_vs_best_holm": 0.7073292670732927, "p_vs_first_stage_holm": 0.0008999100089991002, "wording_best_vs_model": "on par", "wording_vs_first_stage": "clearly better" }, "created_utc": "2026-10-10T14:00:47+00:00", "depth@100": { "hit@10": { "mean": 0.3475409836065574 }, "hit@5": { "ci_high": 0.39672131147540984, "ci_low": 0.29180327868852457, "mean": 0.3442622950819672 }, "judged@10": { "mean": 0.14688524590163934 }, "mrr@10": { "mean": 0.28298724954462656 }, "ndcg@10": { "ci_high": 0.254476750006573, "ci_low": 0.1785537516126157, "mean": 0.21614878159171763 }, "ndcg_condensed@10": { "mean": 0.23672540897280736 }, "recall@10": { "mean": 0.22171740827478534 }, "recall@100": { "mean": 0.23757220921155345 }, "recall@20": { "mean": 0.2332006245120999 }, "recall@5": { "mean": 0.21266978922716628 } }, "depth@20": { "hit@10": { "mean": 0.24918032786885247 }, "hit@5": { "ci_high": 0.29508196721311475, "ci_low": 0.19672131147540983, "mean": 0.2459016393442623 }, "judged@10": { "mean": 0.09836065573770492 }, "mrr@10": { "mean": 0.21677595628415303 }, "ndcg@10": { "ci_high": 0.18257909496675145, "ci_low": 0.11804569640219416, "mean": 0.14981444359463866 }, "ndcg_condensed@10": { "mean": 0.222838268801342 }, "recall@10": { "mean": 0.1402263856362217 }, "recall@100": { "mean": 0.23757220921155345 }, "recall@20": { "mean": 0.14451990632318498 }, "recall@5": { "mean": 0.1367525370804059 } }, "depth@50": { "hit@10": { "mean": 0.29508196721311475 }, "hit@5": { "ci_high": 0.33450819672131027, "ci_low": 0.2360655737704918, "mean": 0.28524590163934427 }, "judged@10": { "mean": 0.12819672131147541 }, "mrr@10": { "mean": 0.24120218579234973 }, "ndcg@10": { "ci_high": 0.21325194649198842, "ci_low": 0.1432600064312427, "mean": 0.17766862311306839 }, "ndcg_condensed@10": { "mean": 0.2279328904592979 }, "recall@10": { "mean": 0.17729117876658856 }, "recall@100": { "mean": 0.23757220921155345 }, "recall@20": { "mean": 0.18568306010928962 }, "recall@5": { "mean": 0.1697423887587822 } }, "run_id": "quality-keyword-71k--jina-v3.5-20261010T140047Z-8b7d0a" }, "mxbai-large-v2": { "comparison": { "contamination": [], "contamination_known": true, "delta_best_minus_model_points": 2.675093145080634, "delta_vs_first_stage_points": 11.821823895180584, "ndcg@10": 0.19250062389936629, "p_vs_best_holm": 0.0007999200079992001, "p_vs_first_stage_holm": 0.0008999100089991002, "wording_best_vs_model": "clearly better", "wording_vs_first_stage": "clearly better" }, "created_utc": "2026-10-10T14:00:48+00:00", "depth@100": { "hit@10": { "mean": 0.3377049180327869 }, "hit@5": { "ci_high": 0.35737704918032787, "ci_low": 0.25245901639344265, "mean": 0.30491803278688523 }, "judged@10": { "mean": 0.13540983606557377 }, "mrr@10": { "mean": 0.25129586260733805 }, "ndcg@10": { "ci_high": 0.22838494964584355, "ci_low": 0.15797767812830196, "mean": 0.19250062389936629 }, "ndcg_condensed@10": { "mean": 0.23748583993581746 }, "recall@10": { "mean": 0.21056986729117877 }, "recall@100": { "mean": 0.23757220921155345 }, "recall@20": { "mean": 0.22557377049180327 }, "recall@5": { "mean": 0.18309133489461357 } }, "depth@20": { "hit@10": { "mean": 0.25245901639344265 }, "hit@5": { "ci_high": 0.28524590163934427, "ci_low": 0.1901639344262295, "mean": 0.2360655737704918 }, "judged@10": { "mean": 0.09770491803278689 }, "mrr@10": { "mean": 0.1976541764246682 }, "ndcg@10": { "ci_high": 0.17065091365999221, "ci_low": 0.10982308331723394, "mean": 0.13945017873057144 }, "ndcg_condensed@10": { "mean": 0.21936887982533065 }, "recall@10": { "mean": 0.14233411397345822 }, "recall@100": { "mean": 0.23757220921155345 }, "recall@20": { "mean": 0.14451990632318498 }, "recall@5": { "mean": 0.13234192037470727 } }, "depth@50": { "hit@10": { "mean": 0.29180327868852457 }, "hit@5": { "ci_high": 0.3180327868852459, "ci_low": 0.21967213114754097, "mean": 0.26885245901639343 }, "judged@10": { "mean": 0.11967213114754098 }, "mrr@10": { "mean": 0.22091465001301067 }, "ndcg@10": { "ci_high": 0.19661370910885906, "ci_low": 0.13021071513619178, "mean": 0.1627127208847503 }, "ndcg_condensed@10": { "mean": 0.22615493711535406 }, "recall@10": { "mean": 0.1737626854020297 }, "recall@100": { "mean": 0.23757220921155345 }, "recall@20": { "mean": 0.18393442622950817 }, "recall@5": { "mean": 0.1549726775956284 } }, "run_id": "quality-keyword-71k--mxbai-large-v2-20261010T140048Z-7e33c3" }, "querit-4b": { "comparison": { "contamination": [ "MIRACLReranking" ], "contamination_known": true, "delta_best_minus_model_points": 0.9983119466368956, "delta_vs_first_stage_points": 13.498605093624322, "ndcg@10": 0.20926843588380367, "p_vs_best_holm": 0.27917208279172084, "p_vs_first_stage_holm": 0.0008999100089991002, "wording_best_vs_model": "on par", "wording_vs_first_stage": "clearly better" }, "created_utc": "2026-10-10T14:00:48+00:00", "depth@100": { "hit@10": { "mean": 0.3344262295081967 }, "hit@5": { "ci_high": 0.3704918032786885, "ci_low": 0.26557377049180325, "mean": 0.3180327868852459 }, "judged@10": { "mean": 0.14557377049180328 }, "mrr@10": { "mean": 0.2747905282331512 }, "ndcg@10": { "ci_high": 0.24805679216996104, "ci_low": 0.1715720843659151, "mean": 0.20926843588380367 }, "ndcg_condensed@10": { "mean": 0.24143574352371056 }, "recall@10": { "mean": 0.21141295862607334 }, "recall@100": { "mean": 0.23757220921155345 }, "recall@20": { "mean": 0.23156128024980482 }, "recall@5": { "mean": 0.19563622170179545 } }, "depth@20": { "hit@10": { "mean": 0.24918032786885247 }, "hit@5": { "ci_high": 0.28524590163934427, "ci_low": 0.1901639344262295, "mean": 0.2360655737704918 }, "judged@10": { "mean": 0.09770491803278689 }, "mrr@10": { "mean": 0.20995706479313037 }, "ndcg@10": { "ci_high": 0.17939743494661564, "ci_low": 0.1150408254144826, "mean": 0.14611059614274993 }, "ndcg_condensed@10": { "mean": 0.2239232893286737 }, "recall@10": { "mean": 0.13866510538641685 }, "recall@100": { "mean": 0.23757220921155345 }, "recall@20": { "mean": 0.14451990632318498 }, "recall@5": { "mean": 0.13362997658079626 } }, "depth@50": { "hit@10": { "mean": 0.29180327868852457 }, "hit@5": { "ci_high": 0.3180327868852459, "ci_low": 0.21967213114754097, "mean": 0.26885245901639343 }, "judged@10": { "mean": 0.12754098360655738 }, "mrr@10": { "mean": 0.23511319281811083 }, "ndcg@10": { "ci_high": 0.20971222961028466, "ci_low": 0.13859832805919556, "mean": 0.17327323069484168 }, "ndcg_condensed@10": { "mean": 0.230950579014053 }, "recall@10": { "mean": 0.17389539422326308 }, "recall@100": { "mean": 0.23757220921155345 }, "recall@20": { "mean": 0.18568306010928962 }, "recall@5": { "mean": 0.15877439500390317 } }, "run_id": "quality-keyword-71k--querit-4b-20261010T140048Z-13edb8" }, "qwen3-0.6b": { "comparison": { "contamination": [ "MIRACLReranking" ], "contamination_known": true, "delta_best_minus_model_points": 0.7677551709576985, "delta_vs_first_stage_points": 13.72916186930352, "ndcg@10": 0.21157400364059564, "p_vs_best_holm": 0.3533646635336466, "p_vs_first_stage_holm": 0.0008999100089991002, "wording_best_vs_model": "on par", "wording_vs_first_stage": "clearly better" }, "created_utc": "2026-10-10T14:00:48+00:00", "depth@100": { "hit@10": { "mean": 0.3540983606557377 }, "hit@5": { "ci_high": 0.3836065573770492, "ci_low": 0.2786885245901639, "mean": 0.33114754098360655 }, "judged@10": { "mean": 0.15245901639344261 }, "mrr@10": { "mean": 0.28004033307311993 }, "ndcg@10": { "ci_high": 0.24927066607275247, "ci_low": 0.17481064938739987, "mean": 0.21157400364059564 }, "ndcg_condensed@10": { "mean": 0.2367359706431128 }, "recall@10": { "mean": 0.22225604996096804 }, "recall@100": { "mean": 0.23757220921155345 }, "recall@20": { "mean": 0.23426229508196716 }, "recall@5": { "mean": 0.199032006245121 } }, "depth@20": { "hit@10": { "mean": 0.25245901639344265 }, "hit@5": { "ci_high": 0.29180327868852457, "ci_low": 0.19672131147540983, "mean": 0.24262295081967214 }, "judged@10": { "mean": 0.1 }, "mrr@10": { "mean": 0.21468904501691388 }, "ndcg@10": { "ci_high": 0.17812701867460315, "ci_low": 0.11596023597707562, "mean": 0.14653768811764603 }, "ndcg_condensed@10": { "mean": 0.22188490540484399 }, "recall@10": { "mean": 0.14085089773614362 }, "recall@100": { "mean": 0.23757220921155345 }, "recall@20": { "mean": 0.14451990632318498 }, "recall@5": { "mean": 0.133551912568306 } }, "depth@50": { "hit@10": { "mean": 0.2983606557377049 }, "hit@5": { "ci_high": 0.3344262295081967, "ci_low": 0.2360655737704918, "mean": 0.28524590163934427 }, "judged@10": { "mean": 0.13245901639344265 }, "mrr@10": { "mean": 0.24639734582357528 }, "ndcg@10": { "ci_high": 0.2141539096579519, "ci_low": 0.1443127086966573, "mean": 0.17863489825943782 }, "ndcg_condensed@10": { "mean": 0.2302806839728451 }, "recall@10": { "mean": 0.17935987509758 }, "recall@100": { "mean": 0.23757220921155345 }, "recall@20": { "mean": 0.18412177985948477 }, "recall@5": { "mean": 0.16943793911007027 } }, "run_id": "quality-keyword-71k--qwen3-0.6b-20261010T140048Z-2f5584" }, "qwen3-4b": { "comparison": { "contamination": [ "MIRACLReranking" ], "contamination_known": true, "delta_vs_first_stage_points": 14.49691704026122, "ndcg@10": 0.21925155535017263, "p_vs_first_stage_holm": 0.0008999100089991002, "wording_vs_first_stage": "clearly better" }, "created_utc": "2026-10-10T14:00:48+00:00", "depth@100": { "hit@10": { "mean": 0.35081967213114756 }, "hit@5": { "ci_high": 0.3901639344262295, "ci_low": 0.2819672131147541, "mean": 0.3344262295081967 }, "judged@10": { "mean": 0.14885245901639346 }, "mrr@10": { "mean": 0.2826541764246682 }, "ndcg@10": { "ci_high": 0.2586610459958759, "ci_low": 0.18086466923470787, "mean": 0.21925155535017263 }, "ndcg_condensed@10": { "mean": 0.24553639463294044 }, "recall@10": { "mean": 0.22597189695550354 }, "recall@100": { "mean": 0.23757220921155345 }, "recall@20": { "mean": 0.23554254488680715 }, "recall@5": { "mean": 0.21034348165495703 } }, "depth@20": { "hit@10": { "mean": 0.25245901639344265 }, "hit@5": { "ci_high": 0.29180327868852457, "ci_low": 0.19672131147540983, "mean": 0.24262295081967214 }, "judged@10": { "mean": 0.09770491803278689 }, "mrr@10": { "mean": 0.21347775175644026 }, "ndcg@10": { "ci_high": 0.18159452510199617, "ci_low": 0.11746152851103583, "mean": 0.1487511289661505 }, "ndcg_condensed@10": { "mean": 0.22451231096206362 }, "recall@10": { "mean": 0.14249024199843868 }, "recall@100": { "mean": 0.23757220921155345 }, "recall@20": { "mean": 0.14451990632318498 }, "recall@5": { "mean": 0.1370257611241218 } }, "depth@50": { "hit@10": { "mean": 0.2983606557377049 }, "hit@5": { "ci_high": 0.3442622950819672, "ci_low": 0.23934426229508196, "mean": 0.29180327868852457 }, "judged@10": { "mean": 0.12721311475409836 }, "mrr@10": { "mean": 0.24188914910226386 }, "ndcg@10": { "ci_high": 0.21466708149819036, "ci_low": 0.14385984579358166, "mean": 0.17848399571639093 }, "ndcg_condensed@10": { "mean": 0.23387204206131706 }, "recall@10": { "mean": 0.17881342701014832 }, "recall@100": { "mean": 0.23757220921155345 }, "recall@20": { "mean": 0.18521467603434819 }, "recall@5": { "mean": 0.16943793911007027 } }, "run_id": "quality-keyword-71k--qwen3-4b-20261010T140048Z-5af098" }, "voyage-rerank-3": { "comparison": { "contamination": [], "contamination_known": false, "delta_best_minus_model_points": 0.4533453184885544, "delta_vs_first_stage_points": 14.043571721772663, "ndcg@10": 0.21471810216528708, "p_vs_best_holm": 0.7073292670732927, "p_vs_first_stage_holm": 0.0008999100089991002, "wording_best_vs_model": "on par", "wording_vs_first_stage": "clearly better" }, "created_utc": "2026-10-10T14:00:49+00:00", "depth@100": { "hit@10": { "mean": 0.3442622950819672 }, "hit@5": { "ci_high": 0.380327868852459, "ci_low": 0.2754098360655738, "mean": 0.32786885245901637 }, "judged@10": { "mean": 0.1422950819672131 }, "mrr@10": { "mean": 0.2782825917252147 }, "ndcg@10": { "ci_high": 0.25368979341347114, "ci_low": 0.1766081026701181, "mean": 0.21471810216528708 }, "ndcg_condensed@10": { "mean": 0.24672083508171044 }, "recall@10": { "mean": 0.21933645589383294 }, "recall@100": { "mean": 0.23757220921155345 }, "recall@20": { "mean": 0.22912568306010928 }, "recall@5": { "mean": 0.20348165495706477 } }, "depth@20": { "hit@10": { "mean": 0.25245901639344265 }, "hit@5": { "ci_high": 0.29180327868852457, "ci_low": 0.19672131147540983, "mean": 0.24262295081967214 }, "judged@10": { "mean": 0.09606557377049181 }, "mrr@10": { "mean": 0.21082357533177204 }, "ndcg@10": { "ci_high": 0.17983818480808522, "ci_low": 0.11658282971892497, "mean": 0.1475207995266199 }, "ndcg_condensed@10": { "mean": 0.2262228168274266 }, "recall@10": { "mean": 0.14350507416081187 }, "recall@100": { "mean": 0.23757220921155345 }, "recall@20": { "mean": 0.14451990632318498 }, "recall@5": { "mean": 0.13554254488680717 } }, "depth@50": { "hit@10": { "mean": 0.29180327868852457 }, "hit@5": { "ci_high": 0.3180327868852459, "ci_low": 0.21967213114754097, "mean": 0.26885245901639343 }, "judged@10": { "mean": 0.12262295081967213 }, "mrr@10": { "mean": 0.23753447827218316 }, "ndcg@10": { "ci_high": 0.20938414845246395, "ci_low": 0.14000525444451697, "mean": 0.17383937069398447 }, "ndcg_condensed@10": { "mean": 0.23409791165454785 }, "recall@10": { "mean": 0.1756752537080406 }, "recall@100": { "mean": 0.23757220921155345 }, "recall@20": { "mean": 0.18568306010928962 }, "recall@5": { "mean": 0.15858704137392662 } }, "run_id": "quality-keyword-71k--voyage-rerank-3-20261010T140049Z-e38cf5" }, "voyage-rerank-3-lite": { "comparison": { "contamination": [], "contamination_known": false, "delta_best_minus_model_points": 1.30792333106573, "delta_vs_first_stage_points": 13.18899370919549, "ndcg@10": 0.20617232203951533, "p_vs_best_holm": 0.1334866513348665, "p_vs_first_stage_holm": 0.0008999100089991002, "wording_best_vs_model": "on par", "wording_vs_first_stage": "clearly better" }, "created_utc": "2026-10-10T14:00:49+00:00", "depth@100": { "hit@10": { "mean": 0.35081967213114756 }, "hit@5": { "ci_high": 0.3639344262295082, "ci_low": 0.25901639344262295, "mean": 0.3114754098360656 }, "judged@10": { "mean": 0.14327868852459016 }, "mrr@10": { "mean": 0.2681537861046058 }, "ndcg@10": { "ci_high": 0.2439439677722593, "ci_low": 0.170084227303951, "mean": 0.20617232203951533 }, "ndcg_condensed@10": { "mean": 0.2422751001303 }, "recall@10": { "mean": 0.22188134270101484 }, "recall@100": { "mean": 0.23757220921155345 }, "recall@20": { "mean": 0.2297814207650273 }, "recall@5": { "mean": 0.1942232630757221 } }, "depth@20": { "hit@10": { "mean": 0.25245901639344265 }, "hit@5": { "ci_high": 0.29508196721311475, "ci_low": 0.19672131147540983, "mean": 0.2459016393442623 }, "judged@10": { "mean": 0.09606557377049181 }, "mrr@10": { "mean": 0.20508977361436379 }, "ndcg@10": { "ci_high": 0.17515653015317442, "ci_low": 0.11315110803421424, "mean": 0.14331223912120375 }, "ndcg_condensed@10": { "mean": 0.2234052014834114 }, "recall@10": { "mean": 0.14233411397345822 }, "recall@100": { "mean": 0.23757220921155345 }, "recall@20": { "mean": 0.14451990632318498 }, "recall@5": { "mean": 0.13702576112412176 } }, "depth@50": { "hit@10": { "mean": 0.29508196721311475 }, "hit@5": { "ci_high": 0.3180327868852459, "ci_low": 0.21967213114754097, "mean": 0.26885245901639343 }, "judged@10": { "mean": 0.12360655737704919 }, "mrr@10": { "mean": 0.23302628155087174 }, "ndcg@10": { "ci_high": 0.2030867682857751, "ci_low": 0.1354718691767485, "mean": 0.16883034364124921 }, "ndcg_condensed@10": { "mean": 0.23095379502871838 }, "recall@10": { "mean": 0.17458235753317722 }, "recall@100": { "mean": 0.23757220921155345 }, "recall@20": { "mean": 0.18459016393442623 }, "recall@5": { "mean": 0.15730679156908664 } }, "run_id": "quality-keyword-71k--voyage-rerank-3-lite-20261010T140049Z-b3ebcd" } }, "not_measured": {}, "run_id": "quality-summary-keyword-71k-20261012T080507Z-ab7e07", "winner": null }, "keyword-full": { "best": "qwen3-4b", "created_utc": "2026-10-12T08:05:08+00:00", "models": { "bge-v2-m3": { "comparison": { "contamination": [ "MIRACLReranking", "MIRACLRetrieval", "MIRACLRetrievalHardNegatives" ], "contamination_known": true, "delta_best_minus_model_points": 0.7959718270794308, "delta_vs_first_stage_points": 9.178887710433736, "ndcg@10": 0.15324959976082256, "p_vs_best_holm": 0.132986701329867, "p_vs_first_stage_holm": 0.0007999200079992001, "wording_best_vs_model": "on par", "wording_vs_first_stage": "clearly better" }, "created_utc": "2026-10-10T14:00:54+00:00", "depth@100": { "hit@10": { "mean": 0.2459016393442623 }, "hit@5": { "ci_high": 0.28524590163934427, "ci_low": 0.1901639344262295, "mean": 0.2360655737704918 }, "judged@10": { "mean": 0.10524590163934426 }, "mrr@10": { "mean": 0.20323315118397087 }, "ndcg@10": { "ci_high": 0.1876121372889568, "ci_low": 0.12067956733669885, "mean": 0.15324959976082256 }, "ndcg_condensed@10": { "mean": 0.16983045745168313 }, "recall@10": { "mean": 0.15714285714285714 }, "recall@100": { "mean": 0.1662217017954723 }, "recall@20": { "mean": 0.16294301327088212 }, "recall@5": { "mean": 0.14453551912568305 } }, "depth@20": { "hit@10": { "mean": 0.18032786885245902 }, "hit@5": { "ci_high": 0.22295081967213115, "ci_low": 0.13442622950819672, "mean": 0.17704918032786884 }, "judged@10": { "mean": 0.06557377049180328 }, "mrr@10": { "mean": 0.14986338797814205 }, "ndcg@10": { "ci_high": 0.13114649268286377, "ci_low": 0.07635708004719584, "mean": 0.10340429113425216 }, "ndcg_condensed@10": { "mean": 0.15365514259587465 }, "recall@10": { "mean": 0.10030444964871194 }, "recall@100": { "mean": 0.1662217017954723 }, "recall@20": { "mean": 0.10241217798594845 }, "recall@5": { "mean": 0.09765027322404371 } }, "depth@50": { "hit@10": { "mean": 0.21639344262295082 }, "hit@5": { "ci_high": 0.25245901639344265, "ci_low": 0.16393442622950818, "mean": 0.20655737704918034 }, "judged@10": { "mean": 0.08655737704918032 }, "mrr@10": { "mean": 0.1762607338017174 }, "ndcg@10": { "ci_high": 0.1590677230404351, "ci_low": 0.09798472460511233, "mean": 0.12769846888253736 }, "ndcg_condensed@10": { "mean": 0.16009584888012687 }, "recall@10": { "mean": 0.12743950039032007 }, "recall@100": { "mean": 0.1662217017954723 }, "recall@20": { "mean": 0.1322014051522248 }, "recall@5": { "mean": 0.11882903981264636 } }, "run_id": "quality-keyword-full--bge-v2-m3-20261010T140054Z-eae4a7" }, "gte-base": { "comparison": { "contamination": [], "contamination_known": false, "delta_best_minus_model_points": 1.4127863554521614, "delta_vs_first_stage_points": 8.562073182061006, "ndcg@10": 0.14708145447709525, "p_vs_best_holm": 0.047995200479952004, "p_vs_first_stage_holm": 0.0007999200079992001, "wording_best_vs_model": "measurably better", "wording_vs_first_stage": "clearly better" }, "created_utc": "2026-10-10T14:00:53+00:00", "depth@100": { "hit@10": { "mean": 0.24262295081967214 }, "hit@5": { "ci_high": 0.2721311475409836, "ci_low": 0.17704918032786884, "mean": 0.22295081967213115 }, "judged@10": { "mean": 0.1 }, "mrr@10": { "mean": 0.20153395784543326 }, "ndcg@10": { "ci_high": 0.1805240911286306, "ci_low": 0.11544359393597362, "mean": 0.14708145447709525 }, "ndcg_condensed@10": { "mean": 0.16577939928802604 }, "recall@10": { "mean": 0.14763466042154566 }, "recall@100": { "mean": 0.1662217017954723 }, "recall@20": { "mean": 0.15624512099921933 }, "recall@5": { "mean": 0.13473067915690867 } }, "depth@20": { "hit@10": { "mean": 0.18032786885245902 }, "hit@5": { "ci_high": 0.2098360655737705, "ci_low": 0.12786885245901639, "mean": 0.16721311475409836 }, "judged@10": { "mean": 0.06688524590163934 }, "mrr@10": { "mean": 0.14694379391100704 }, "ndcg@10": { "ci_high": 0.1284729899929193, "ci_low": 0.07489654927780182, "mean": 0.10112476926716563 }, "ndcg_condensed@10": { "mean": 0.15197193221705046 }, "recall@10": { "mean": 0.10092896174863387 }, "recall@100": { "mean": 0.1662217017954723 }, "recall@20": { "mean": 0.10241217798594845 }, "recall@5": { "mean": 0.08989071038251366 } }, "depth@50": { "hit@10": { "mean": 0.20655737704918034 }, "hit@5": { "ci_high": 0.24262295081967214, "ci_low": 0.1540983606557377, "mean": 0.19672131147540983 }, "judged@10": { "mean": 0.08557377049180329 }, "mrr@10": { "mean": 0.17743950039032005 }, "ndcg@10": { "ci_high": 0.154536258238579, "ci_low": 0.09520342568812486, "mean": 0.12395809742353789 }, "ndcg_condensed@10": { "mean": 0.1577460976402245 }, "recall@10": { "mean": 0.12104605776736924 }, "recall@100": { "mean": 0.1662217017954723 }, "recall@20": { "mean": 0.13110850897736145 }, "recall@5": { "mean": 0.11309133489461358 } }, "run_id": "quality-keyword-full--gte-base-20261010T140053Z-580b0f" }, "jina-v3.5": { "comparison": null, "created_utc": "2026-10-10T14:00:54+00:00", "depth@20": { "hit@10": { "mean": 0.18032786885245902 }, "hit@5": { "ci_high": 0.22295081967213115, "ci_low": 0.13442622950819672, "mean": 0.17704918032786884 }, "judged@10": { "mean": 0.06655737704918033 }, "mrr@10": { "mean": 0.15699453551912568 }, "ndcg@10": { "ci_high": 0.13585047440745623, "ci_low": 0.07928442451759551, "mean": 0.10695834023282147 }, "ndcg_condensed@10": { "mean": 0.15462917727046047 }, "recall@10": { "mean": 0.09921155347384854 }, "recall@100": { "mean": 0.1662217017954723 }, "recall@20": { "mean": 0.10241217798594845 }, "recall@5": { "mean": 0.09546448087431694 } }, "depth@50": { "hit@10": { "mean": 0.21639344262295082 }, "hit@5": { "ci_high": 0.25573770491803277, "ci_low": 0.16393442622950818, "mean": 0.2098360655737705 }, "judged@10": { "mean": 0.08262295081967214 }, "mrr@10": { "mean": 0.18858834244080147 }, "ndcg@10": { "ci_high": 0.16615712755776582, "ci_low": 0.10301505578552911, "mean": 0.1336880605119036 }, "ndcg_condensed@10": { "mean": 0.1624169357838719 }, "recall@10": { "mean": 0.12544886807181888 }, "recall@100": { "mean": 0.1662217017954723 }, "recall@20": { "mean": 0.1322014051522248 }, "recall@5": { "mean": 0.1205776736924278 } }, "run_id": "quality-keyword-full--jina-v3.5-20261010T140054Z-59ad2e" }, "mxbai-large-v2": { "comparison": { "contamination": [], "contamination_known": true, "delta_best_minus_model_points": 1.3201322306043957, "delta_vs_first_stage_points": 8.654727306908772, "ndcg@10": 0.1480079957255729, "p_vs_best_holm": 0.0027997200279972004, "p_vs_first_stage_holm": 0.0007999200079992001, "wording_best_vs_model": "measurably better", "wording_vs_first_stage": "clearly better" }, "created_utc": "2026-10-10T14:00:54+00:00", "depth@100": { "hit@10": { "mean": 0.24918032786885247 }, "hit@5": { "ci_high": 0.2819672131147541, "ci_low": 0.18688524590163935, "mean": 0.23278688524590163 }, "judged@10": { "mean": 0.09573770491803278 }, "mrr@10": { "mean": 0.19702966432474628 }, "ndcg@10": { "ci_high": 0.18140476168809122, "ci_low": 0.11627323536666193, "mean": 0.1480079957255729 }, "ndcg_condensed@10": { "mean": 0.16925034556148405 }, "recall@10": { "mean": 0.15583138173302105 }, "recall@100": { "mean": 0.1662217017954723 }, "recall@20": { "mean": 0.162256049960968 }, "recall@5": { "mean": 0.14073380171740826 } }, "depth@20": { "hit@10": { "mean": 0.18032786885245902 }, "hit@5": { "ci_high": 0.21311475409836064, "ci_low": 0.12786885245901639, "mean": 0.17049180327868851 }, "judged@10": { "mean": 0.06622950819672133 }, "mrr@10": { "mean": 0.14122040072859746 }, "ndcg@10": { "ci_high": 0.12724534712529592, "ci_low": 0.07372439637977753, "mean": 0.0999149408943261 }, "ndcg_condensed@10": { "mean": 0.1522846087782386 }, "recall@10": { "mean": 0.10022638563622169 }, "recall@100": { "mean": 0.1662217017954723 }, "recall@20": { "mean": 0.10241217798594845 }, "recall@5": { "mean": 0.09433255269320842 } }, "depth@50": { "hit@10": { "mean": 0.21639344262295082 }, "hit@5": { "ci_high": 0.25245901639344265, "ci_low": 0.16065573770491803, "mean": 0.20655737704918034 }, "judged@10": { "mean": 0.08295081967213114 }, "mrr@10": { "mean": 0.16469945355191257 }, "ndcg@10": { "ci_high": 0.15163608388116134, "ci_low": 0.09277004298998619, "mean": 0.12145672704910448 }, "ndcg_condensed@10": { "mean": 0.1586243496643319 }, "recall@10": { "mean": 0.12678376268540203 }, "recall@100": { "mean": 0.1662217017954723 }, "recall@20": { "mean": 0.13110850897736145 }, "recall@5": { "mean": 0.11797033567525368 } }, "run_id": "quality-keyword-full--mxbai-large-v2-20261010T140054Z-5ce6de" }, "querit-4b": { "comparison": { "contamination": [ "MIRACLReranking" ], "contamination_known": true, "delta_best_minus_model_points": 0.56837768068905, "delta_vs_first_stage_points": 9.406481856824117, "ndcg@10": 0.15552554122472637, "p_vs_best_holm": 0.3735626437356264, "p_vs_first_stage_holm": 0.0007999200079992001, "wording_best_vs_model": "on par", "wording_vs_first_stage": "clearly better" }, "created_utc": "2026-10-10T14:00:55+00:00", "depth@100": { "hit@10": { "mean": 0.24918032786885247 }, "hit@5": { "ci_high": 0.28852459016393445, "ci_low": 0.19344262295081968, "mean": 0.23934426229508196 }, "judged@10": { "mean": 0.10327868852459017 }, "mrr@10": { "mean": 0.20549180327868852 }, "ndcg@10": { "ci_high": 0.18988863862009214, "ci_low": 0.12259765387149046, "mean": 0.15552554122472637 }, "ndcg_condensed@10": { "mean": 0.1727162086603017 }, "recall@10": { "mean": 0.1591647150663544 }, "recall@100": { "mean": 0.1662217017954723 }, "recall@20": { "mean": 0.1651288056206089 }, "recall@5": { "mean": 0.14455893832943012 } }, "depth@20": { "hit@10": { "mean": 0.18032786885245902 }, "hit@5": { "ci_high": 0.21639344262295082, "ci_low": 0.13114754098360656, "mean": 0.1737704918032787 }, "judged@10": { "mean": 0.06622950819672131 }, "mrr@10": { "mean": 0.15106557377049182 }, "ndcg@10": { "ci_high": 0.1334002991893428, "ci_low": 0.07750803025464365, "mean": 0.10495360038423385 }, "ndcg_condensed@10": { "mean": 0.15524471914086818 }, "recall@10": { "mean": 0.09983606557377049 }, "recall@100": { "mean": 0.1662217017954723 }, "recall@20": { "mean": 0.10241217798594845 }, "recall@5": { "mean": 0.09671350507416081 } }, "depth@50": { "hit@10": { "mean": 0.21967213114754097 }, "hit@5": { "ci_high": 0.25245901639344265, "ci_low": 0.16065573770491803, "mean": 0.20655737704918034 }, "judged@10": { "mean": 0.0862295081967213 }, "mrr@10": { "mean": 0.17828259172521466 }, "ndcg@10": { "ci_high": 0.1629730529351945, "ci_low": 0.10053450406058406, "mean": 0.13096653968638636 }, "ndcg_condensed@10": { "mean": 0.1626340262208967 }, "recall@10": { "mean": 0.12962529274004683 }, "recall@100": { "mean": 0.1662217017954723 }, "recall@20": { "mean": 0.1322014051522248 }, "recall@5": { "mean": 0.12217798594847774 } }, "run_id": "quality-keyword-full--querit-4b-20261010T140055Z-7b302f" }, "qwen3-0.6b": { "comparison": { "contamination": [ "MIRACLReranking" ], "contamination_known": true, "delta_best_minus_model_points": 0.6611628838818912, "delta_vs_first_stage_points": 9.313696653631276, "ndcg@10": 0.15459768919279795, "p_vs_best_holm": 0.33956604339566043, "p_vs_first_stage_holm": 0.0007999200079992001, "wording_best_vs_model": "on par", "wording_vs_first_stage": "clearly better" }, "created_utc": "2026-10-10T14:00:54+00:00", "depth@100": { "hit@10": { "mean": 0.24918032786885247 }, "hit@5": { "ci_high": 0.28852459016393445, "ci_low": 0.19344262295081968, "mean": 0.23934426229508196 }, "judged@10": { "mean": 0.10754098360655737 }, "mrr@10": { "mean": 0.20978922716627635 }, "ndcg@10": { "ci_high": 0.18876323179423424, "ci_low": 0.12173553181898808, "mean": 0.15459768919279795 }, "ndcg_condensed@10": { "mean": 0.16949611689005795 }, "recall@10": { "mean": 0.15530835284933645 }, "recall@100": { "mean": 0.1662217017954723 }, "recall@20": { "mean": 0.16356752537080405 }, "recall@5": { "mean": 0.14483996877439498 } }, "depth@20": { "hit@10": { "mean": 0.18032786885245902 }, "hit@5": { "ci_high": 0.21639344262295082, "ci_low": 0.13114754098360656, "mean": 0.1737704918032787 }, "judged@10": { "mean": 0.06655737704918033 }, "mrr@10": { "mean": 0.14951209992193598 }, "ndcg@10": { "ci_high": 0.13122288275678337, "ci_low": 0.07610595421532484, "mean": 0.10291801101493535 }, "ndcg_condensed@10": { "mean": 0.15309488666667448 }, "recall@10": { "mean": 0.09928961748633879 }, "recall@100": { "mean": 0.1662217017954723 }, "recall@20": { "mean": 0.10241217798594845 }, "recall@5": { "mean": 0.09472287275565963 } }, "depth@50": { "hit@10": { "mean": 0.21639344262295082 }, "hit@5": { "ci_high": 0.25245901639344265, "ci_low": 0.16065573770491803, "mean": 0.20655737704918034 }, "judged@10": { "mean": 0.08852459016393442 }, "mrr@10": { "mean": 0.17987639864689045 }, "ndcg@10": { "ci_high": 0.15846821078093493, "ci_low": 0.09759381019878155, "mean": 0.12723659924663488 }, "ndcg_condensed@10": { "mean": 0.15789384973529244 }, "recall@10": { "mean": 0.1258782201405152 }, "recall@100": { "mean": 0.1662217017954723 }, "recall@20": { "mean": 0.12954722872755658 }, "recall@5": { "mean": 0.11846994535519126 } }, "run_id": "quality-keyword-full--qwen3-0.6b-20261010T140054Z-85e548" }, "qwen3-4b": { "comparison": { "contamination": [ "MIRACLReranking" ], "contamination_known": true, "delta_vs_first_stage_points": 9.974859537513167, "ndcg@10": 0.16120931803161687, "p_vs_first_stage_holm": 0.0007999200079992001, "wording_vs_first_stage": "clearly better" }, "created_utc": "2026-10-10T14:00:55+00:00", "depth@100": { "hit@10": { "mean": 0.24918032786885247 }, "hit@5": { "ci_high": 0.29508196721311475, "ci_low": 0.19991803278688533, "mean": 0.2459016393442623 }, "judged@10": { "mean": 0.10065573770491804 }, "mrr@10": { "mean": 0.21385636221701793 }, "ndcg@10": { "ci_high": 0.1971036885204342, "ci_low": 0.1272055640349873, "mean": 0.16120931803161687 }, "ndcg_condensed@10": { "mean": 0.1753596866728576 }, "recall@10": { "mean": 0.15760343481654954 }, "recall@100": { "mean": 0.1662217017954723 }, "recall@20": { "mean": 0.16466042154566743 }, "recall@5": { "mean": 0.15325526932084307 } }, "depth@20": { "hit@10": { "mean": 0.18032786885245902 }, "hit@5": { "ci_high": 0.21311475409836064, "ci_low": 0.12786885245901639, "mean": 0.17049180327868851 }, "judged@10": { "mean": 0.06622950819672133 }, "mrr@10": { "mean": 0.15571818891491024 }, "ndcg@10": { "ci_high": 0.13640789412236545, "ci_low": 0.07952166832867467, "mean": 0.10747000530098098 }, "ndcg_condensed@10": { "mean": 0.15569753423392294 }, "recall@10": { "mean": 0.10038251366120218 }, "recall@100": { "mean": 0.1662217017954723 }, "recall@20": { "mean": 0.10241217798594845 }, "recall@5": { "mean": 0.09491803278688524 } }, "depth@50": { "hit@10": { "mean": 0.21639344262295082 }, "hit@5": { "ci_high": 0.25573770491803277, "ci_low": 0.16393442622950818, "mean": 0.2098360655737705 }, "judged@10": { "mean": 0.08295081967213115 }, "mrr@10": { "mean": 0.18749024199843872 }, "ndcg@10": { "ci_high": 0.16592334017183824, "ci_low": 0.10259831114166858, "mean": 0.13324822925148228 }, "ndcg_condensed@10": { "mean": 0.16390896727904114 }, "recall@10": { "mean": 0.12642466822794693 }, "recall@100": { "mean": 0.1662217017954723 }, "recall@20": { "mean": 0.13173302107728338 }, "recall@5": { "mean": 0.12295081967213115 } }, "run_id": "quality-keyword-full--qwen3-4b-20261010T140055Z-c2ecbd" }, "voyage-rerank-3": { "comparison": { "contamination": [], "contamination_known": false, "delta_best_minus_model_points": 0.26278534493688055, "delta_vs_first_stage_points": 9.712074192576287, "ndcg@10": 0.15858146458224806, "p_vs_best_holm": 0.39416058394160586, "p_vs_first_stage_holm": 0.0007999200079992001, "wording_best_vs_model": "on par", "wording_vs_first_stage": "clearly better" }, "created_utc": "2026-10-10T14:00:55+00:00", "depth@100": { "hit@10": { "mean": 0.25573770491803277 }, "hit@5": { "ci_high": 0.28524590163934427, "ci_low": 0.1901639344262295, "mean": 0.2360655737704918 }, "judged@10": { "mean": 0.09868852459016395 }, "mrr@10": { "mean": 0.21355971896955503 }, "ndcg@10": { "ci_high": 0.19403500460552764, "ci_low": 0.12506885805394333, "mean": 0.15858146458224806 }, "ndcg_condensed@10": { "mean": 0.17629901622057914 }, "recall@10": { "mean": 0.1582903981264637 }, "recall@100": { "mean": 0.1662217017954723 }, "recall@20": { "mean": 0.16338017174082747 }, "recall@5": { "mean": 0.1456830601092896 } }, "depth@20": { "hit@10": { "mean": 0.18032786885245902 }, "hit@5": { "ci_high": 0.21311475409836064, "ci_low": 0.12786885245901639, "mean": 0.17049180327868851 }, "judged@10": { "mean": 0.06590163934426228 }, "mrr@10": { "mean": 0.15257611241217797 }, "ndcg@10": { "ci_high": 0.13502222084249912, "ci_low": 0.07845336714664575, "mean": 0.10611515883574905 }, "ndcg_condensed@10": { "mean": 0.15689886018388052 }, "recall@10": { "mean": 0.10139734582357532 }, "recall@100": { "mean": 0.1662217017954723 }, "recall@20": { "mean": 0.10241217798594845 }, "recall@5": { "mean": 0.09433255269320842 } }, "depth@50": { "hit@10": { "mean": 0.21639344262295082 }, "hit@5": { "ci_high": 0.25573770491803277, "ci_low": 0.16393442622950818, "mean": 0.2098360655737705 }, "judged@10": { "mean": 0.08262295081967214 }, "mrr@10": { "mean": 0.1823809523809524 }, "ndcg@10": { "ci_high": 0.1622064454625992, "ci_low": 0.10037347499708084, "mean": 0.13049073189962007 }, "ndcg_condensed@10": { "mean": 0.16681024070941672 }, "recall@10": { "mean": 0.12733021077283374 }, "recall@100": { "mean": 0.1662217017954723 }, "recall@20": { "mean": 0.1322014051522248 }, "recall@5": { "mean": 0.12199063231850117 } }, "run_id": "quality-keyword-full--voyage-rerank-3-20261010T140055Z-3a6828" }, "voyage-rerank-3-lite": { "comparison": { "contamination": [], "contamination_known": false, "delta_best_minus_model_points": 0.8356265832463738, "delta_vs_first_stage_points": 9.139232954266793, "ndcg@10": 0.15285305219915313, "p_vs_best_holm": 0.16798320167983202, "p_vs_first_stage_holm": 0.0007999200079992001, "wording_best_vs_model": "on par", "wording_vs_first_stage": "clearly better" }, "created_utc": "2026-10-10T14:00:55+00:00", "depth@100": { "hit@10": { "mean": 0.25573770491803277 }, "hit@5": { "ci_high": 0.2786885245901639, "ci_low": 0.18360655737704917, "mean": 0.22950819672131148 }, "judged@10": { "mean": 0.0980327868852459 }, "mrr@10": { "mean": 0.20547619047619045 }, "ndcg@10": { "ci_high": 0.1870632009574082, "ci_low": 0.12015883591189834, "mean": 0.15285305219915313 }, "ndcg_condensed@10": { "mean": 0.17311217929281097 }, "recall@10": { "mean": 0.15730679156908664 }, "recall@100": { "mean": 0.1662217017954723 }, "recall@20": { "mean": 0.16272443403590944 }, "recall@5": { "mean": 0.13882903981264635 } }, "depth@20": { "hit@10": { "mean": 0.18032786885245902 }, "hit@5": { "ci_high": 0.21639344262295082, "ci_low": 0.13114754098360656, "mean": 0.1737704918032787 }, "judged@10": { "mean": 0.06622950819672133 }, "mrr@10": { "mean": 0.146615925058548 }, "ndcg@10": { "ci_high": 0.12984925015962645, "ci_low": 0.0753069854807992, "mean": 0.10206586700519445 }, "ndcg_condensed@10": { "mean": 0.15411511429629096 }, "recall@10": { "mean": 0.10022638563622169 }, "recall@100": { "mean": 0.1662217017954723 }, "recall@20": { "mean": 0.10241217798594845 }, "recall@5": { "mean": 0.09581576893052302 } }, "depth@50": { "hit@10": { "mean": 0.21639344262295082 }, "hit@5": { "ci_high": 0.24918032786885247, "ci_low": 0.15737704918032788, "mean": 0.20327868852459016 }, "judged@10": { "mean": 0.08163934426229508 }, "mrr@10": { "mean": 0.17981264637002342 }, "ndcg@10": { "ci_high": 0.1582900550030593, "ci_low": 0.09749057816107252, "mean": 0.12711566871317556 }, "ndcg_condensed@10": { "mean": 0.16350046890238082 }, "recall@10": { "mean": 0.12569086651053865 }, "recall@100": { "mean": 0.1662217017954723 }, "recall@20": { "mean": 0.1322014051522248 }, "recall@5": { "mean": 0.11562841530054645 } }, "run_id": "quality-keyword-full--voyage-rerank-3-lite-20261010T140055Z-ad9a59" } }, "not_measured": { "jina-v3.5": { "depth@100": "vendor listwise block exceeds 16 GB VRAM for at least 13 of 304 queries (prompts >= 45.2k tokens OOM; 35.5k fit)" } }, "run_id": "quality-summary-keyword-full-20261012T080508Z-368030", "winner": null }, "vector-71k": { "best": "qwen3-4b", "created_utc": "2026-10-12T08:05:07+00:00", "models": { "bge-v2-m3": { "comparison": { "contamination": [ "MIRACLReranking", "MIRACLRetrieval", "MIRACLRetrievalHardNegatives" ], "contamination_known": true, "delta_best_minus_model_points": 4.4539439515354555, "delta_vs_first_stage_points": 4.749198979332736, "ndcg@10": 0.6378067818750585, "p_vs_best_holm": 0.0011998800119988001, "p_vs_first_stage_holm": 0.0035996400359964006, "wording_best_vs_model": "clearly better", "wording_vs_first_stage": "clearly better" }, "created_utc": "2026-10-10T14:00:49+00:00", "depth@100": { "hit@10": { "mean": 0.9049180327868852 }, "hit@5": { "ci_high": 0.8918032786885246, "ci_low": 0.8131147540983606, "mean": 0.8524590163934426 }, "judged@10": { "mean": 0.37901639344262295 }, "mrr@10": { "mean": 0.6919971376528753 }, "ndcg@10": { "ci_high": 0.67411429739069, "ci_low": 0.6001374171462157, "mean": 0.6378067818750585 }, "ndcg_condensed@10": { "mean": 0.8351385818380394 }, "recall@10": { "mean": 0.7463882383554514 }, "recall@100": { "mean": 0.9652354931043456 }, "recall@20": { "mean": 0.8469372885766329 }, "recall@5": { "mean": 0.632305490502212 } }, "depth@20": { "hit@10": { "mean": 0.8819672131147541 }, "hit@5": { "ci_high": 0.8918032786885246, "ci_low": 0.8131147540983606, "mean": 0.8524590163934426 }, "judged@10": { "mean": 0.3885245901639344 }, "mrr@10": { "mean": 0.6885688264376788 }, "ndcg@10": { "ci_high": 0.6682886627716289, "ci_low": 0.5924248833338733, "mean": 0.6308555537779529 }, "ndcg_condensed@10": { "mean": 0.8327037231634649 }, "recall@10": { "mean": 0.7325956284153006 }, "recall@100": { "mean": 0.9652354931043456 }, "recall@20": { "mean": 0.8045771532656778 }, "recall@5": { "mean": 0.6315716887848035 } }, "depth@50": { "hit@10": { "mean": 0.9016393442622951 }, "hit@5": { "ci_high": 0.8918032786885246, "ci_low": 0.8131147540983606, "mean": 0.8524590163934426 }, "judged@10": { "mean": 0.38688524590163936 }, "mrr@10": { "mean": 0.6933320322664585 }, "ndcg@10": { "ci_high": 0.6733505576169614, "ci_low": 0.5991343373599449, "mean": 0.6368482763401818 }, "ndcg_condensed@10": { "mean": 0.8337023821043765 }, "recall@10": { "mean": 0.7434608378870674 }, "recall@100": { "mean": 0.9652354931043456 }, "recall@20": { "mean": 0.8474837366640646 }, "recall@5": { "mean": 0.6305022118136873 } }, "run_id": "quality-vector-71k--bge-v2-m3-20261010T140049Z-9d7aab" }, "gte-base": { "comparison": { "contamination": [], "contamination_known": false, "delta_best_minus_model_points": 11.487950584923755, "delta_vs_first_stage_points": -2.284807654055565, "ndcg@10": 0.5674667155411754, "p_vs_best_holm": 0.0007999200079992001, "p_vs_first_stage_holm": 0.12488751124887511, "wording_best_vs_model": "clearly better", "wording_vs_first_stage": "on par" }, "created_utc": "2026-10-10T14:00:49+00:00", "depth@100": { "hit@10": { "mean": 0.8721311475409836 }, "hit@5": { "ci_high": 0.8295081967213115, "ci_low": 0.7377049180327869, "mean": 0.7836065573770492 }, "judged@10": { "mean": 0.36426229508196717 }, "mrr@10": { "mean": 0.6166185271922978 }, "ndcg@10": { "ci_high": 0.6047538282770902, "ci_low": 0.5297907144336069, "mean": 0.5674667155411754 }, "ndcg_condensed@10": { "mean": 0.80009864631187 }, "recall@10": { "mean": 0.6919346864428831 }, "recall@100": { "mean": 0.9652354931043456 }, "recall@20": { "mean": 0.801975019516003 }, "recall@5": { "mean": 0.552311995836586 } }, "depth@20": { "hit@10": { "mean": 0.8721311475409836 }, "hit@5": { "ci_high": 0.839344262295082, "ci_low": 0.7475409836065574, "mean": 0.7934426229508197 }, "judged@10": { "mean": 0.3842622950819672 }, "mrr@10": { "mean": 0.6253642987249544 }, "ndcg@10": { "ci_high": 0.618961395416209, "ci_low": 0.5439891239799626, "mean": 0.5814561727701212 }, "ndcg_condensed@10": { "mean": 0.8056012226383644 }, "recall@10": { "mean": 0.7105412438199323 }, "recall@100": { "mean": 0.9652354931043456 }, "recall@20": { "mean": 0.8045771532656778 }, "recall@5": { "mean": 0.5735415040333073 } }, "depth@50": { "hit@10": { "mean": 0.8754098360655738 }, "hit@5": { "ci_high": 0.8360655737704918, "ci_low": 0.7442622950819672, "mean": 0.7901639344262295 }, "judged@10": { "mean": 0.3744262295081967 }, "mrr@10": { "mean": 0.6205503512880562 }, "ndcg@10": { "ci_high": 0.6110628913281227, "ci_low": 0.5366775762458359, "mean": 0.5740975352535735 }, "ndcg_condensed@10": { "mean": 0.8026997614652696 }, "recall@10": { "mean": 0.7024811345303148 }, "recall@100": { "mean": 0.9652354931043456 }, "recall@20": { "mean": 0.8181342701014832 }, "recall@5": { "mean": 0.5666289357272964 } }, "run_id": "quality-vector-71k--gte-base-20261010T140049Z-d0c904" }, "jina-v3.5": { "comparison": { "contamination": [], "contamination_known": false, "delta_best_minus_model_points": 4.087316917778594, "delta_vs_first_stage_points": 5.115826013089597, "ndcg@10": 0.6414730522126271, "p_vs_best_holm": 0.0029997000299970002, "p_vs_first_stage_holm": 0.0008999100089991002, "wording_best_vs_model": "clearly better", "wording_vs_first_stage": "clearly better" }, "created_utc": "2026-10-10T14:00:50+00:00", "depth@100": { "hit@10": { "mean": 0.9081967213114754 }, "hit@5": { "ci_high": 0.8918032786885246, "ci_low": 0.8131147540983606, "mean": 0.8524590163934426 }, "judged@10": { "mean": 0.3678688524590164 }, "mrr@10": { "mean": 0.69747983346344 }, "ndcg@10": { "ci_high": 0.6771751394545925, "ci_low": 0.6055945512387628, "mean": 0.6414730522126271 }, "ndcg_condensed@10": { "mean": 0.8448905406981201 }, "recall@10": { "mean": 0.755647931303669 }, "recall@100": { "mean": 0.9652354931043456 }, "recall@20": { "mean": 0.8448373666406453 }, "recall@5": { "mean": 0.6440658339838669 } }, "depth@20": { "hit@10": { "mean": 0.8885245901639345 }, "hit@5": { "ci_high": 0.8819672131147541, "ci_low": 0.8, "mean": 0.8426229508196721 }, "judged@10": { "mean": 0.3888524590163934 }, "mrr@10": { "mean": 0.6946383034087952 }, "ndcg@10": { "ci_high": 0.676165336944166, "ci_low": 0.601984524425781, "mean": 0.6394433771761813 }, "ndcg_condensed@10": { "mean": 0.8443026209146947 }, "recall@10": { "mean": 0.7439227166276345 }, "recall@100": { "mean": 0.9652354931043456 }, "recall@20": { "mean": 0.8045771532656778 }, "recall@5": { "mean": 0.6462516263335935 } }, "depth@50": { "hit@10": { "mean": 0.9180327868852459 }, "hit@5": { "ci_high": 0.8885245901639345, "ci_low": 0.8098360655737705, "mean": 0.8491803278688524 }, "judged@10": { "mean": 0.38 }, "mrr@10": { "mean": 0.6970686963309914 }, "ndcg@10": { "ci_high": 0.6821538136501765, "ci_low": 0.6115662192593385, "mean": 0.6471846699929054 }, "ndcg_condensed@10": { "mean": 0.8431071238028902 }, "recall@10": { "mean": 0.7668813427010148 }, "recall@100": { "mean": 0.9652354931043456 }, "recall@20": { "mean": 0.8541582097319803 }, "recall@5": { "mean": 0.6453773093937029 } }, "run_id": "quality-vector-71k--jina-v3.5-20261010T140050Z-1856e8" }, "mxbai-large-v2": { "comparison": { "contamination": [], "contamination_known": true, "delta_best_minus_model_points": 14.259358268771594, "delta_vs_first_stage_points": -5.056215337903403, "ndcg@10": 0.5397526387026971, "p_vs_best_holm": 0.0007999200079992001, "p_vs_first_stage_holm": 0.0035996400359964006, "wording_best_vs_model": "clearly better", "wording_vs_first_stage": "clearly worse" }, "created_utc": "2026-10-10T14:00:50+00:00", "depth@100": { "hit@10": { "mean": 0.839344262295082 }, "hit@5": { "ci_high": 0.8098360655737705, "ci_low": 0.7147540983606557, "mean": 0.7639344262295082 }, "judged@10": { "mean": 0.3075409836065574 }, "mrr@10": { "mean": 0.59655997918293 }, "ndcg@10": { "ci_high": 0.577871969447336, "ci_low": 0.5021764504847352, "mean": 0.5397526387026971 }, "ndcg_condensed@10": { "mean": 0.8240041341229625 }, "recall@10": { "mean": 0.6619463960447567 }, "recall@100": { "mean": 0.9652354931043456 }, "recall@20": { "mean": 0.7999154306531356 }, "recall@5": { "mean": 0.532352328909706 } }, "depth@20": { "hit@10": { "mean": 0.8754098360655738 }, "hit@5": { "ci_high": 0.819672131147541, "ci_low": 0.7245901639344262, "mean": 0.7737704918032787 }, "judged@10": { "mean": 0.36 }, "mrr@10": { "mean": 0.6164754098360655 }, "ndcg@10": { "ci_high": 0.609059393604139, "ci_low": 0.53569023457762, "mean": 0.5721945829034601 }, "ndcg_condensed@10": { "mean": 0.819982416599322 }, "recall@10": { "mean": 0.7120673952641166 }, "recall@100": { "mean": 0.9652354931043456 }, "recall@20": { "mean": 0.8045771532656778 }, "recall@5": { "mean": 0.5552016653655999 } }, "depth@50": { "hit@10": { "mean": 0.8491803278688524 }, "hit@5": { "ci_high": 0.8131147540983606, "ci_low": 0.7180327868852459, "mean": 0.7672131147540984 }, "judged@10": { "mean": 0.3265573770491803 }, "mrr@10": { "mean": 0.6028181108508978 }, "ndcg@10": { "ci_high": 0.5881968658460042, "ci_low": 0.5131816814520416, "mean": 0.5503631621834044 }, "ndcg_condensed@10": { "mean": 0.8232645824547447 }, "recall@10": { "mean": 0.6782110330470986 }, "recall@100": { "mean": 0.9652354931043456 }, "recall@20": { "mean": 0.8145420244600573 }, "recall@5": { "mean": 0.5406583398386677 } }, "run_id": "quality-vector-71k--mxbai-large-v2-20261010T140050Z-c649bf" }, "querit-4b": { "comparison": { "contamination": [ "MIRACLReranking" ], "contamination_known": true, "delta_best_minus_model_points": 4.907023706335901, "delta_vs_first_stage_points": 4.296119224532291, "ndcg@10": 0.633275984327054, "p_vs_best_holm": 0.0011998800119988001, "p_vs_first_stage_holm": 0.0035996400359964006, "wording_best_vs_model": "clearly better", "wording_vs_first_stage": "clearly better" }, "created_utc": "2026-10-10T14:00:50+00:00", "depth@100": { "hit@10": { "mean": 0.8786885245901639 }, "hit@5": { "ci_high": 0.8688524590163934, "ci_low": 0.7868852459016393, "mean": 0.8295081967213115 }, "judged@10": { "mean": 0.3518032786885246 }, "mrr@10": { "mean": 0.6878584439240177 }, "ndcg@10": { "ci_high": 0.6707712454339817, "ci_low": 0.5958101551696311, "mean": 0.633275984327054 }, "ndcg_condensed@10": { "mean": 0.8625024593639037 }, "recall@10": { "mean": 0.7356791569086651 }, "recall@100": { "mean": 0.9652354931043456 }, "recall@20": { "mean": 0.8284985688264377 }, "recall@5": { "mean": 0.6261930783242259 } }, "depth@20": { "hit@10": { "mean": 0.8819672131147541 }, "hit@5": { "ci_high": 0.8786885245901639, "ci_low": 0.7967213114754098, "mean": 0.839344262295082 }, "judged@10": { "mean": 0.3701639344262295 }, "mrr@10": { "mean": 0.6934361176164456 }, "ndcg@10": { "ci_high": 0.674537619118934, "ci_low": 0.5988182624859166, "mean": 0.6370932209333862 }, "ndcg_condensed@10": { "mean": 0.8578057704077697 }, "recall@10": { "mean": 0.7348568826437679 }, "recall@100": { "mean": 0.9652354931043456 }, "recall@20": { "mean": 0.8045771532656778 }, "recall@5": { "mean": 0.6340931563882384 } }, "depth@50": { "hit@10": { "mean": 0.8786885245901639 }, "hit@5": { "ci_high": 0.8721311475409836, "ci_low": 0.7868852459016393, "mean": 0.8295081967213115 }, "judged@10": { "mean": 0.3554098360655738 }, "mrr@10": { "mean": 0.6881680978402289 }, "ndcg@10": { "ci_high": 0.6693086465439604, "ci_low": 0.5938707492863832, "mean": 0.6318191865678772 }, "ndcg_condensed@10": { "mean": 0.860479560982111 }, "recall@10": { "mean": 0.7315027322404372 }, "recall@100": { "mean": 0.9652354931043456 }, "recall@20": { "mean": 0.8259302628155087 }, "recall@5": { "mean": 0.6271220400728597 } }, "run_id": "quality-vector-71k--querit-4b-20261010T140050Z-3b27ed" }, "qwen3-0.6b": { "comparison": { "contamination": [ "MIRACLReranking" ], "contamination_known": true, "delta_best_minus_model_points": 4.449820961778695, "delta_vs_first_stage_points": 4.753321969089496, "ndcg@10": 0.6378480117726261, "p_vs_best_holm": 0.0029997000299970002, "p_vs_first_stage_holm": 0.005099490050994901, "wording_best_vs_model": "clearly better", "wording_vs_first_stage": "clearly better" }, "created_utc": "2026-10-10T14:00:50+00:00", "depth@100": { "hit@10": { "mean": 0.9344262295081968 }, "hit@5": { "ci_high": 0.9081967213114754, "ci_low": 0.8327868852459016, "mean": 0.8721311475409836 }, "judged@10": { "mean": 0.38491803278688524 }, "mrr@10": { "mean": 0.6783008066614624 }, "ndcg@10": { "ci_high": 0.6716597586936816, "ci_low": 0.6037861846135041, "mean": 0.6378480117726261 }, "ndcg_condensed@10": { "mean": 0.8380814480733674 }, "recall@10": { "mean": 0.7771922976841009 }, "recall@100": { "mean": 0.9652354931043456 }, "recall@20": { "mean": 0.8863674212854542 }, "recall@5": { "mean": 0.6599557637262555 } }, "depth@20": { "hit@10": { "mean": 0.9016393442622951 }, "hit@5": { "ci_high": 0.8819672131147541, "ci_low": 0.8, "mean": 0.8426229508196721 }, "judged@10": { "mean": 0.39901639344262296 }, "mrr@10": { "mean": 0.6772404371584699 }, "ndcg@10": { "ci_high": 0.6681747634107194, "ci_low": 0.5965125199383119, "mean": 0.6328005557054308 }, "ndcg_condensed@10": { "mean": 0.8306955058581681 }, "recall@10": { "mean": 0.755647931303669 }, "recall@100": { "mean": 0.9652354931043456 }, "recall@20": { "mean": 0.8045771532656778 }, "recall@5": { "mean": 0.649471766848816 } }, "depth@50": { "hit@10": { "mean": 0.9278688524590164 }, "hit@5": { "ci_high": 0.898360655737705, "ci_low": 0.8229508196721311, "mean": 0.8622950819672132 }, "judged@10": { "mean": 0.39311475409836066 }, "mrr@10": { "mean": 0.6749986989331251 }, "ndcg@10": { "ci_high": 0.6729771872254016, "ci_low": 0.6044359231660911, "mean": 0.638788489556897 }, "ndcg_condensed@10": { "mean": 0.833543808126558 }, "recall@10": { "mean": 0.7805074160811866 }, "recall@100": { "mean": 0.9652354931043456 }, "recall@20": { "mean": 0.8755477491543064 }, "recall@5": { "mean": 0.6563492063492063 } }, "run_id": "quality-vector-71k--qwen3-0.6b-20261010T140050Z-3b2735" }, "qwen3-4b": { "comparison": { "contamination": [ "MIRACLReranking" ], "contamination_known": true, "delta_vs_first_stage_points": 9.203142930868191, "ndcg@10": 0.682346221390413, "p_vs_first_stage_holm": 0.0008999100089991002, "wording_vs_first_stage": "clearly better" }, "created_utc": "2026-10-10T14:00:50+00:00", "depth@100": { "hit@10": { "mean": 0.9377049180327869 }, "hit@5": { "ci_high": 0.9180327868852459, "ci_low": 0.8459016393442623, "mean": 0.8819672131147541 }, "judged@10": { "mean": 0.36950819672131147 }, "mrr@10": { "mean": 0.7326424668227947 }, "ndcg@10": { "ci_high": 0.7164572473158747, "ci_low": 0.6477023930108017, "mean": 0.682346221390413 }, "ndcg_condensed@10": { "mean": 0.8749943502501049 }, "recall@10": { "mean": 0.804367681498829 }, "recall@100": { "mean": 0.9652354931043456 }, "recall@20": { "mean": 0.891376528753578 }, "recall@5": { "mean": 0.6840202966432474 } }, "depth@20": { "hit@10": { "mean": 0.9081967213114754 }, "hit@5": { "ci_high": 0.8885245901639345, "ci_low": 0.8098360655737705, "mean": 0.8491803278688524 }, "judged@10": { "mean": 0.3816393442622951 }, "mrr@10": { "mean": 0.720536039552433 }, "ndcg@10": { "ci_high": 0.7028292442990848, "ci_low": 0.6289827346183486, "mean": 0.6661395536260493 }, "ndcg_condensed@10": { "mean": 0.8577181499165303 }, "recall@10": { "mean": 0.7672482435597189 }, "recall@100": { "mean": 0.9652354931043456 }, "recall@20": { "mean": 0.8045771532656778 }, "recall@5": { "mean": 0.6673302107728337 } }, "depth@50": { "hit@10": { "mean": 0.9278688524590164 }, "hit@5": { "ci_high": 0.9049180327868852, "ci_low": 0.8295081967213115, "mean": 0.8688524590163934 }, "judged@10": { "mean": 0.3770491803278688 }, "mrr@10": { "mean": 0.73220010408535 }, "ndcg@10": { "ci_high": 0.7173139290042012, "ci_low": 0.6467005887740274, "mean": 0.6826100234134929 }, "ndcg_condensed@10": { "mean": 0.8706027878392347 }, "recall@10": { "mean": 0.801174863387978 }, "recall@100": { "mean": 0.9652354931043456 }, "recall@20": { "mean": 0.8776333593546709 }, "recall@5": { "mean": 0.679047619047619 } }, "run_id": "quality-vector-71k--qwen3-4b-20261010T140050Z-1dfdc4" }, "voyage-rerank-3": { "comparison": { "contamination": [], "contamination_known": false, "delta_best_minus_model_points": 2.3914284963181376, "delta_vs_first_stage_points": 6.811714434550053, "ndcg@10": 0.6584319364272316, "p_vs_best_holm": 0.0347965203479652, "p_vs_first_stage_holm": 0.0008999100089991002, "wording_best_vs_model": "clearly better", "wording_vs_first_stage": "clearly better" }, "created_utc": "2026-10-10T14:00:51+00:00", "depth@100": { "hit@10": { "mean": 0.9081967213114754 }, "hit@5": { "ci_high": 0.8918032786885246, "ci_low": 0.8131147540983606, "mean": 0.8524590163934426 }, "judged@10": { "mean": 0.34950819672131145 }, "mrr@10": { "mean": 0.7232435597189696 }, "ndcg@10": { "ci_high": 0.6938436837188919, "ci_low": 0.6221351876224293, "mean": 0.6584319364272316 }, "ndcg_condensed@10": { "mean": 0.8781302295814465 }, "recall@10": { "mean": 0.7648946135831381 }, "recall@100": { "mean": 0.9652354931043456 }, "recall@20": { "mean": 0.8640072859744992 }, "recall@5": { "mean": 0.6480783242258652 } }, "depth@20": { "hit@10": { "mean": 0.9016393442622951 }, "hit@5": { "ci_high": 0.8885245901639345, "ci_low": 0.8065573770491803, "mean": 0.8491803278688524 }, "judged@10": { "mean": 0.37836065573770494 }, "mrr@10": { "mean": 0.722832422586521 }, "ndcg@10": { "ci_high": 0.6977824468639741, "ci_low": 0.6255043342364389, "mean": 0.6620686673058775 }, "ndcg_condensed@10": { "mean": 0.8669394025853439 }, "recall@10": { "mean": 0.7624290918553213 }, "recall@100": { "mean": 0.9652354931043456 }, "recall@20": { "mean": 0.8045771532656778 }, "recall@5": { "mean": 0.655104085349987 } }, "depth@50": { "hit@10": { "mean": 0.9114754098360656 }, "hit@5": { "ci_high": 0.8918032786885246, "ci_low": 0.8131147540983606, "mean": 0.8524590163934426 }, "judged@10": { "mean": 0.3586885245901639 }, "mrr@10": { "mean": 0.7256661462399168 }, "ndcg@10": { "ci_high": 0.6989355823855826, "ci_low": 0.6276843905150683, "mean": 0.6637719602525459 }, "ndcg_condensed@10": { "mean": 0.8764807067594783 }, "recall@10": { "mean": 0.7736377829820453 }, "recall@100": { "mean": 0.9652354931043456 }, "recall@20": { "mean": 0.8523367161072078 }, "recall@5": { "mean": 0.6491712204007285 } }, "run_id": "quality-vector-71k--voyage-rerank-3-20261010T140051Z-e1024d" }, "voyage-rerank-3-lite": { "comparison": { "contamination": [], "contamination_known": false, "delta_best_minus_model_points": 5.513357208126002, "delta_vs_first_stage_points": 3.6897857227421893, "ndcg@10": 0.627212649309153, "p_vs_best_holm": 0.0011998800119988001, "p_vs_first_stage_holm": 0.015998400159984, "wording_best_vs_model": "clearly better", "wording_vs_first_stage": "clearly better" }, "created_utc": "2026-10-10T14:00:51+00:00", "depth@100": { "hit@10": { "mean": 0.9016393442622951 }, "hit@5": { "ci_high": 0.8786885245901639, "ci_low": 0.7967213114754098, "mean": 0.839344262295082 }, "judged@10": { "mean": 0.3416393442622951 }, "mrr@10": { "mean": 0.6968774395003903 }, "ndcg@10": { "ci_high": 0.6630589452789711, "ci_low": 0.5912355110004343, "mean": 0.627212649309153 }, "ndcg_condensed@10": { "mean": 0.8591661655502605 }, "recall@10": { "mean": 0.7345901639344262 }, "recall@100": { "mean": 0.9652354931043456 }, "recall@20": { "mean": 0.8507832422586522 }, "recall@5": { "mean": 0.6118293000260213 } }, "depth@20": { "hit@10": { "mean": 0.8950819672131147 }, "hit@5": { "ci_high": 0.8754098360655738, "ci_low": 0.7934426229508197, "mean": 0.8360655737704918 }, "judged@10": { "mean": 0.37868852459016394 }, "mrr@10": { "mean": 0.7052498048399688 }, "ndcg@10": { "ci_high": 0.6786889970778162, "ci_low": 0.6067107324730893, "mean": 0.6427508753769122 }, "ndcg_condensed@10": { "mean": 0.8517304357650044 }, "recall@10": { "mean": 0.7512373145979703 }, "recall@100": { "mean": 0.9652354931043456 }, "recall@20": { "mean": 0.8045771532656778 }, "recall@5": { "mean": 0.624479573250065 } }, "depth@50": { "hit@10": { "mean": 0.9147540983606557 }, "hit@5": { "ci_high": 0.8885245901639345, "ci_low": 0.8098360655737705, "mean": 0.8491803278688524 }, "judged@10": { "mean": 0.3557377049180328 }, "mrr@10": { "mean": 0.7016770752016653 }, "ndcg@10": { "ci_high": 0.6747291376547622, "ci_low": 0.6039642948647348, "mean": 0.6393966532230647 }, "ndcg_condensed@10": { "mean": 0.8571626712776917 }, "recall@10": { "mean": 0.756120218579235 }, "recall@100": { "mean": 0.9652354931043456 }, "recall@20": { "mean": 0.8485662243039293 }, "recall@5": { "mean": 0.6168566224303929 } }, "run_id": "quality-vector-71k--voyage-rerank-3-lite-20261010T140051Z-23b020" } }, "not_measured": {}, "run_id": "quality-summary-vector-71k-20261012T080507Z-e8fbcc", "winner": "qwen3-4b" } }, "repo_commit": "e715d27a1bf73df51f6cbddc99e936b293488590", "repo_dirty": false, "schema_version": 1 }