Spaces:
Runtime error
Runtime error
| { | |
| "generated_at": "2026-06-30T01:39:45.501910+00:00", | |
| "date": "2026-06-29", | |
| "benchmarks": [ | |
| "dataset-neonai-qa", | |
| "dataset-nucleotidings-qa", | |
| "dataset-logistics-qa", | |
| "dataset-landuse-wa-qa", | |
| "dataset-shakespeare-qa", | |
| "dataset-futurewise-qa", | |
| "dataset-belebele-qa" | |
| ], | |
| "candidates": { | |
| "retrieval-only::bm25-only::none": { | |
| "id": "retrieval-only::bm25-only::none", | |
| "model": "retrieval-only", | |
| "model_adapter": "brainforge", | |
| "model_repo": "retrieval-only", | |
| "retriever": "bm25-only", | |
| "retriever_adapter": "search_pipeline", | |
| "reranker": "none", | |
| "reranker_model": null, | |
| "is_neon": false, | |
| "is_retrieval_only": true | |
| }, | |
| "retrieval-only::bm25-only::minilm": { | |
| "id": "retrieval-only::bm25-only::minilm", | |
| "model": "retrieval-only", | |
| "model_adapter": "brainforge", | |
| "model_repo": "retrieval-only", | |
| "retriever": "bm25-only", | |
| "retriever_adapter": "search_pipeline", | |
| "reranker": "minilm", | |
| "reranker_model": "cross-encoder/ms-marco-MiniLM-L-6-v2", | |
| "is_neon": false, | |
| "is_retrieval_only": true | |
| }, | |
| "retrieval-only::bm25-only::mxbai": { | |
| "id": "retrieval-only::bm25-only::mxbai", | |
| "model": "retrieval-only", | |
| "model_adapter": "brainforge", | |
| "model_repo": "retrieval-only", | |
| "retriever": "bm25-only", | |
| "retriever_adapter": "search_pipeline", | |
| "reranker": "mxbai", | |
| "reranker_model": "mixedbread-ai/mxbai-rerank-base-v1", | |
| "is_neon": false, | |
| "is_retrieval_only": true | |
| }, | |
| "retrieval-only::embedding-bge-m3::none": { | |
| "id": "retrieval-only::embedding-bge-m3::none", | |
| "model": "retrieval-only", | |
| "model_adapter": "brainforge", | |
| "model_repo": "retrieval-only", | |
| "retriever": "embedding-bge-m3", | |
| "retriever_adapter": "search_pipeline", | |
| "reranker": "none", | |
| "reranker_model": null, | |
| "is_neon": false, | |
| "is_retrieval_only": true | |
| }, | |
| "retrieval-only::embedding-bge-m3::minilm": { | |
| "id": "retrieval-only::embedding-bge-m3::minilm", | |
| "model": "retrieval-only", | |
| "model_adapter": "brainforge", | |
| "model_repo": "retrieval-only", | |
| "retriever": "embedding-bge-m3", | |
| "retriever_adapter": "search_pipeline", | |
| "reranker": "minilm", | |
| "reranker_model": "cross-encoder/ms-marco-MiniLM-L-6-v2", | |
| "is_neon": false, | |
| "is_retrieval_only": true | |
| }, | |
| "retrieval-only::embedding-bge-m3::mxbai": { | |
| "id": "retrieval-only::embedding-bge-m3::mxbai", | |
| "model": "retrieval-only", | |
| "model_adapter": "brainforge", | |
| "model_repo": "retrieval-only", | |
| "retriever": "embedding-bge-m3", | |
| "retriever_adapter": "search_pipeline", | |
| "reranker": "mxbai", | |
| "reranker_model": "mixedbread-ai/mxbai-rerank-base-v1", | |
| "is_neon": false, | |
| "is_retrieval_only": true | |
| }, | |
| "retrieval-only::embedding-bge-large::none": { | |
| "id": "retrieval-only::embedding-bge-large::none", | |
| "model": "retrieval-only", | |
| "model_adapter": "brainforge", | |
| "model_repo": "retrieval-only", | |
| "retriever": "embedding-bge-large", | |
| "retriever_adapter": "search_pipeline", | |
| "reranker": "none", | |
| "reranker_model": null, | |
| "is_neon": false, | |
| "is_retrieval_only": true | |
| }, | |
| "retrieval-only::embedding-bge-large::minilm": { | |
| "id": "retrieval-only::embedding-bge-large::minilm", | |
| "model": "retrieval-only", | |
| "model_adapter": "brainforge", | |
| "model_repo": "retrieval-only", | |
| "retriever": "embedding-bge-large", | |
| "retriever_adapter": "search_pipeline", | |
| "reranker": "minilm", | |
| "reranker_model": "cross-encoder/ms-marco-MiniLM-L-6-v2", | |
| "is_neon": false, | |
| "is_retrieval_only": true | |
| }, | |
| "retrieval-only::embedding-bge-large::mxbai": { | |
| "id": "retrieval-only::embedding-bge-large::mxbai", | |
| "model": "retrieval-only", | |
| "model_adapter": "brainforge", | |
| "model_repo": "retrieval-only", | |
| "retriever": "embedding-bge-large", | |
| "retriever_adapter": "search_pipeline", | |
| "reranker": "mxbai", | |
| "reranker_model": "mixedbread-ai/mxbai-rerank-base-v1", | |
| "is_neon": false, | |
| "is_retrieval_only": true | |
| }, | |
| "retrieval-only::hybrid::none": { | |
| "id": "retrieval-only::hybrid::none", | |
| "model": "retrieval-only", | |
| "model_adapter": "brainforge", | |
| "model_repo": "retrieval-only", | |
| "retriever": "hybrid", | |
| "retriever_adapter": "search_pipeline", | |
| "reranker": "none", | |
| "reranker_model": null, | |
| "is_neon": false, | |
| "is_retrieval_only": true | |
| }, | |
| "retrieval-only::hybrid::minilm": { | |
| "id": "retrieval-only::hybrid::minilm", | |
| "model": "retrieval-only", | |
| "model_adapter": "brainforge", | |
| "model_repo": "retrieval-only", | |
| "retriever": "hybrid", | |
| "retriever_adapter": "search_pipeline", | |
| "reranker": "minilm", | |
| "reranker_model": "cross-encoder/ms-marco-MiniLM-L-6-v2", | |
| "is_neon": false, | |
| "is_retrieval_only": true | |
| }, | |
| "retrieval-only::hybrid::mxbai": { | |
| "id": "retrieval-only::hybrid::mxbai", | |
| "model": "retrieval-only", | |
| "model_adapter": "brainforge", | |
| "model_repo": "retrieval-only", | |
| "retriever": "hybrid", | |
| "retriever_adapter": "search_pipeline", | |
| "reranker": "mxbai", | |
| "reranker_model": "mixedbread-ai/mxbai-rerank-base-v1", | |
| "is_neon": false, | |
| "is_retrieval_only": true | |
| }, | |
| "retrieval-only::hybrid-bge-large::none": { | |
| "id": "retrieval-only::hybrid-bge-large::none", | |
| "model": "retrieval-only", | |
| "model_adapter": "brainforge", | |
| "model_repo": "retrieval-only", | |
| "retriever": "hybrid-bge-large", | |
| "retriever_adapter": "search_pipeline", | |
| "reranker": "none", | |
| "reranker_model": null, | |
| "is_neon": false, | |
| "is_retrieval_only": true | |
| }, | |
| "retrieval-only::hybrid-bge-large::minilm": { | |
| "id": "retrieval-only::hybrid-bge-large::minilm", | |
| "model": "retrieval-only", | |
| "model_adapter": "brainforge", | |
| "model_repo": "retrieval-only", | |
| "retriever": "hybrid-bge-large", | |
| "retriever_adapter": "search_pipeline", | |
| "reranker": "minilm", | |
| "reranker_model": "cross-encoder/ms-marco-MiniLM-L-6-v2", | |
| "is_neon": false, | |
| "is_retrieval_only": true | |
| }, | |
| "retrieval-only::hybrid-bge-large::mxbai": { | |
| "id": "retrieval-only::hybrid-bge-large::mxbai", | |
| "model": "retrieval-only", | |
| "model_adapter": "brainforge", | |
| "model_repo": "retrieval-only", | |
| "retriever": "hybrid-bge-large", | |
| "retriever_adapter": "search_pipeline", | |
| "reranker": "mxbai", | |
| "reranker_model": "mixedbread-ai/mxbai-rerank-base-v1", | |
| "is_neon": false, | |
| "is_retrieval_only": true | |
| }, | |
| "retrieval-only::splade-only::none": { | |
| "id": "retrieval-only::splade-only::none", | |
| "model": "retrieval-only", | |
| "model_adapter": "brainforge", | |
| "model_repo": "retrieval-only", | |
| "retriever": "splade-only", | |
| "retriever_adapter": "search_pipeline", | |
| "reranker": "none", | |
| "reranker_model": null, | |
| "is_neon": false, | |
| "is_retrieval_only": true | |
| }, | |
| "retrieval-only::splade-only::minilm": { | |
| "id": "retrieval-only::splade-only::minilm", | |
| "model": "retrieval-only", | |
| "model_adapter": "brainforge", | |
| "model_repo": "retrieval-only", | |
| "retriever": "splade-only", | |
| "retriever_adapter": "search_pipeline", | |
| "reranker": "minilm", | |
| "reranker_model": "cross-encoder/ms-marco-MiniLM-L-6-v2", | |
| "is_neon": false, | |
| "is_retrieval_only": true | |
| }, | |
| "retrieval-only::splade-only::mxbai": { | |
| "id": "retrieval-only::splade-only::mxbai", | |
| "model": "retrieval-only", | |
| "model_adapter": "brainforge", | |
| "model_repo": "retrieval-only", | |
| "retriever": "splade-only", | |
| "retriever_adapter": "search_pipeline", | |
| "reranker": "mxbai", | |
| "reranker_model": "mixedbread-ai/mxbai-rerank-base-v1", | |
| "is_neon": false, | |
| "is_retrieval_only": true | |
| }, | |
| "retrieval-only::hybrid-triple::none": { | |
| "id": "retrieval-only::hybrid-triple::none", | |
| "model": "retrieval-only", | |
| "model_adapter": "brainforge", | |
| "model_repo": "retrieval-only", | |
| "retriever": "hybrid-triple", | |
| "retriever_adapter": "search_pipeline", | |
| "reranker": "none", | |
| "reranker_model": null, | |
| "is_neon": false, | |
| "is_retrieval_only": true | |
| }, | |
| "retrieval-only::hybrid-triple::minilm": { | |
| "id": "retrieval-only::hybrid-triple::minilm", | |
| "model": "retrieval-only", | |
| "model_adapter": "brainforge", | |
| "model_repo": "retrieval-only", | |
| "retriever": "hybrid-triple", | |
| "retriever_adapter": "search_pipeline", | |
| "reranker": "minilm", | |
| "reranker_model": "cross-encoder/ms-marco-MiniLM-L-6-v2", | |
| "is_neon": false, | |
| "is_retrieval_only": true | |
| }, | |
| "retrieval-only::hybrid-triple::mxbai": { | |
| "id": "retrieval-only::hybrid-triple::mxbai", | |
| "model": "retrieval-only", | |
| "model_adapter": "brainforge", | |
| "model_repo": "retrieval-only", | |
| "retriever": "hybrid-triple", | |
| "retriever_adapter": "search_pipeline", | |
| "reranker": "mxbai", | |
| "reranker_model": "mixedbread-ai/mxbai-rerank-base-v1", | |
| "is_neon": false, | |
| "is_retrieval_only": true | |
| }, | |
| "retrieval-only::hybrid-web::none": { | |
| "id": "retrieval-only::hybrid-web::none", | |
| "model": "retrieval-only", | |
| "model_adapter": "brainforge", | |
| "model_repo": "retrieval-only", | |
| "retriever": "hybrid-web", | |
| "retriever_adapter": "search_pipeline", | |
| "reranker": "none", | |
| "reranker_model": null, | |
| "is_neon": false, | |
| "is_retrieval_only": true | |
| }, | |
| "retrieval-only::hybrid-web::minilm": { | |
| "id": "retrieval-only::hybrid-web::minilm", | |
| "model": "retrieval-only", | |
| "model_adapter": "brainforge", | |
| "model_repo": "retrieval-only", | |
| "retriever": "hybrid-web", | |
| "retriever_adapter": "search_pipeline", | |
| "reranker": "minilm", | |
| "reranker_model": "cross-encoder/ms-marco-MiniLM-L-6-v2", | |
| "is_neon": false, | |
| "is_retrieval_only": true | |
| }, | |
| "retrieval-only::hybrid-web::mxbai": { | |
| "id": "retrieval-only::hybrid-web::mxbai", | |
| "model": "retrieval-only", | |
| "model_adapter": "brainforge", | |
| "model_repo": "retrieval-only", | |
| "retriever": "hybrid-web", | |
| "retriever_adapter": "search_pipeline", | |
| "reranker": "mxbai", | |
| "reranker_model": "mixedbread-ai/mxbai-rerank-base-v1", | |
| "is_neon": false, | |
| "is_retrieval_only": true | |
| }, | |
| "devstral-123b::hybrid::none": { | |
| "id": "devstral-123b::hybrid::none", | |
| "model": "devstral-123b", | |
| "model_adapter": "brainforge", | |
| "model_repo": "devstral-2512:123b", | |
| "retriever": "hybrid", | |
| "retriever_adapter": "search_pipeline", | |
| "reranker": "none", | |
| "reranker_model": null, | |
| "is_neon": true, | |
| "is_retrieval_only": false | |
| }, | |
| "ministral-14b-reasoning::hybrid::none": { | |
| "id": "ministral-14b-reasoning::hybrid::none", | |
| "model": "ministral-14b-reasoning", | |
| "model_adapter": "brainforge", | |
| "model_repo": "mistralai/Ministral-3-14B-Reasoning-2512", | |
| "retriever": "hybrid", | |
| "retriever_adapter": "search_pipeline", | |
| "reranker": "none", | |
| "reranker_model": null, | |
| "is_neon": true, | |
| "is_retrieval_only": false | |
| }, | |
| "magistral-small::hybrid::none": { | |
| "id": "magistral-small::hybrid::none", | |
| "model": "magistral-small", | |
| "model_adapter": "brainforge", | |
| "model_repo": "mistralai/Magistral-Small-2509", | |
| "retriever": "hybrid", | |
| "retriever_adapter": "search_pipeline", | |
| "reranker": "none", | |
| "reranker_model": null, | |
| "is_neon": true, | |
| "is_retrieval_only": false | |
| }, | |
| "devstral-small-24b::hybrid::none": { | |
| "id": "devstral-small-24b::hybrid::none", | |
| "model": "devstral-small-24b", | |
| "model_adapter": "brainforge", | |
| "model_repo": "mistralai/Devstral-Small-2-24B-Instruct-2512", | |
| "retriever": "hybrid", | |
| "retriever_adapter": "search_pipeline", | |
| "reranker": "none", | |
| "reranker_model": null, | |
| "is_neon": true, | |
| "is_retrieval_only": false | |
| }, | |
| "mistral-small-3.2-24b::hybrid::none": { | |
| "id": "mistral-small-3.2-24b::hybrid::none", | |
| "model": "mistral-small-3.2-24b", | |
| "model_adapter": "brainforge", | |
| "model_repo": "mistralai/Mistral-Small-3.2-24B-Instruct-2506", | |
| "retriever": "hybrid", | |
| "retriever_adapter": "search_pipeline", | |
| "reranker": "none", | |
| "reranker_model": null, | |
| "is_neon": true, | |
| "is_retrieval_only": false | |
| }, | |
| "granite-4-30b::hybrid::none": { | |
| "id": "granite-4-30b::hybrid::none", | |
| "model": "granite-4-30b", | |
| "model_adapter": "brainforge", | |
| "model_repo": "ibm-granite/granite-4.1-30b", | |
| "retriever": "hybrid", | |
| "retriever_adapter": "search_pipeline", | |
| "reranker": "none", | |
| "reranker_model": null, | |
| "is_neon": true, | |
| "is_retrieval_only": false | |
| } | |
| }, | |
| "retrievers": { | |
| "bm25-only": { | |
| "name": "bm25-only", | |
| "components": [ | |
| { | |
| "key": "bm25", | |
| "label": "BM25" | |
| } | |
| ], | |
| "summary": "BM25" | |
| }, | |
| "embedding-bge-m3": { | |
| "name": "embedding-bge-m3", | |
| "components": [ | |
| { | |
| "key": "emb", | |
| "label": "Embedding", | |
| "detail": "bge-m3" | |
| } | |
| ], | |
| "summary": "Embedding (bge-m3)" | |
| }, | |
| "embedding-bge-large": { | |
| "name": "embedding-bge-large", | |
| "components": [ | |
| { | |
| "key": "emb", | |
| "label": "Embedding", | |
| "detail": "bge-large" | |
| } | |
| ], | |
| "summary": "Embedding (bge-large)" | |
| }, | |
| "hybrid": { | |
| "name": "hybrid", | |
| "components": [ | |
| { | |
| "key": "bm25", | |
| "label": "BM25" | |
| }, | |
| { | |
| "key": "emb", | |
| "label": "Embedding", | |
| "detail": "bge-m3" | |
| } | |
| ], | |
| "summary": "BM25 + Embedding (bge-m3)" | |
| }, | |
| "hybrid-bge-large": { | |
| "name": "hybrid-bge-large", | |
| "components": [ | |
| { | |
| "key": "bm25", | |
| "label": "BM25" | |
| }, | |
| { | |
| "key": "emb", | |
| "label": "Embedding", | |
| "detail": "bge-large" | |
| } | |
| ], | |
| "summary": "BM25 + Embedding (bge-large)" | |
| }, | |
| "splade-only": { | |
| "name": "splade-only", | |
| "components": [ | |
| { | |
| "key": "splade", | |
| "label": "SPLADE" | |
| } | |
| ], | |
| "summary": "SPLADE" | |
| }, | |
| "hybrid-triple": { | |
| "name": "hybrid-triple", | |
| "components": [ | |
| { | |
| "key": "bm25", | |
| "label": "BM25" | |
| }, | |
| { | |
| "key": "emb", | |
| "label": "Embedding", | |
| "detail": "bge-m3" | |
| }, | |
| { | |
| "key": "splade", | |
| "label": "SPLADE" | |
| } | |
| ], | |
| "summary": "BM25 + Embedding (bge-m3) + SPLADE" | |
| }, | |
| "hybrid-web": { | |
| "name": "hybrid-web", | |
| "components": [ | |
| { | |
| "key": "bm25", | |
| "label": "BM25" | |
| }, | |
| { | |
| "key": "emb", | |
| "label": "Embedding", | |
| "detail": "bge-m3" | |
| }, | |
| { | |
| "key": "web", | |
| "label": "Web" | |
| } | |
| ], | |
| "summary": "BM25 + Embedding (bge-m3) + Web" | |
| } | |
| }, | |
| "rerankers": { | |
| "none": { | |
| "name": "none", | |
| "model": null | |
| }, | |
| "minilm": { | |
| "name": "minilm", | |
| "model": "cross-encoder/ms-marco-MiniLM-L-6-v2" | |
| }, | |
| "mxbai": { | |
| "name": "mxbai", | |
| "model": "mixedbread-ai/mxbai-rerank-base-v1" | |
| } | |
| }, | |
| "aggregates": [ | |
| { | |
| "candidate_id": "retrieval-only::bm25-only::none", | |
| "benchmark": "dataset-neonai-qa", | |
| "model": "retrieval-only", | |
| "retriever": "bm25-only", | |
| "reranker": "none", | |
| "is_neon": false, | |
| "error": null, | |
| "mrr": 0.4084285714285716, | |
| "hit_at_1": 0.33, | |
| "hit_at_5": 0.52, | |
| "ndcg_at_10": 0.46015234807813926, | |
| "p50_retrieval_latency_ms": 0.19860174506902695, | |
| "p95_retrieval_latency_ms": 0.28101541101932526, | |
| "avg_retrieval_latency_ms": 0.21203815005719662, | |
| "index_build_seconds": 1.3586820373311639, | |
| "est_cost_per_query_usd": 0.0, | |
| "generation_acc": null, | |
| "p50_generation_latency_ms": null, | |
| "avg_generation_latency_ms": null, | |
| "generation_tokens_per_sec": null, | |
| "est_generation_cost_per_query_usd": null, | |
| "paraphrase_drop": 0.06159920634920657, | |
| "shift_drop": 0.13370634920634938, | |
| "n_queries": 100 | |
| }, | |
| { | |
| "candidate_id": "retrieval-only::bm25-only::minilm", | |
| "benchmark": "dataset-neonai-qa", | |
| "model": "retrieval-only", | |
| "retriever": "bm25-only", | |
| "reranker": "minilm", | |
| "is_neon": false, | |
| "error": null, | |
| "mrr": 0.48669444444444454, | |
| "hit_at_1": 0.38, | |
| "hit_at_5": 0.61, | |
| "ndcg_at_10": 0.5465171846177451, | |
| "p50_retrieval_latency_ms": 23.795085959136486, | |
| "p95_retrieval_latency_ms": 27.18852460384369, | |
| "avg_retrieval_latency_ms": 53.217622097581625, | |
| "index_build_seconds": 0.002659284509718418, | |
| "est_cost_per_query_usd": 0.0, | |
| "generation_acc": null, | |
| "p50_generation_latency_ms": null, | |
| "avg_generation_latency_ms": null, | |
| "generation_tokens_per_sec": null, | |
| "est_generation_cost_per_query_usd": null, | |
| "paraphrase_drop": 0.03348809523809515, | |
| "shift_drop": 0.26363888888888903, | |
| "n_queries": 100 | |
| }, | |
| { | |
| "candidate_id": "retrieval-only::bm25-only::mxbai", | |
| "benchmark": "dataset-neonai-qa", | |
| "model": "retrieval-only", | |
| "retriever": "bm25-only", | |
| "reranker": "mxbai", | |
| "is_neon": false, | |
| "error": null, | |
| "mrr": 0.38216269841269834, | |
| "hit_at_1": 0.27, | |
| "hit_at_5": 0.57, | |
| "ndcg_at_10": 0.45257484841179746, | |
| "p50_retrieval_latency_ms": 184.44390688091516, | |
| "p95_retrieval_latency_ms": 188.86269722133875, | |
| "avg_retrieval_latency_ms": 206.04804329574108, | |
| "index_build_seconds": 0.003439362160861492, | |
| "est_cost_per_query_usd": 0.0, | |
| "generation_acc": null, | |
| "p50_generation_latency_ms": null, | |
| "avg_generation_latency_ms": null, | |
| "generation_tokens_per_sec": null, | |
| "est_generation_cost_per_query_usd": null, | |
| "paraphrase_drop": 0.06377777777777771, | |
| "shift_drop": 0.17174603174603165, | |
| "n_queries": 100 | |
| }, | |
| { | |
| "candidate_id": "retrieval-only::embedding-bge-m3::none", | |
| "benchmark": "dataset-neonai-qa", | |
| "model": "retrieval-only", | |
| "retriever": "embedding-bge-m3", | |
| "reranker": "none", | |
| "is_neon": false, | |
| "error": null, | |
| "mrr": 0.4358452380952382, | |
| "hit_at_1": 0.3, | |
| "hit_at_5": 0.63, | |
| "ndcg_at_10": 0.49556998931673163, | |
| "p50_retrieval_latency_ms": 5.7725729420781136, | |
| "p95_retrieval_latency_ms": 6.527797318994999, | |
| "avg_retrieval_latency_ms": 70.81005768850446, | |
| "index_build_seconds": 0.0027428651228547096, | |
| "est_cost_per_query_usd": 0.0, | |
| "generation_acc": null, | |
| "p50_generation_latency_ms": null, | |
| "avg_generation_latency_ms": null, | |
| "generation_tokens_per_sec": null, | |
| "est_generation_cost_per_query_usd": null, | |
| "paraphrase_drop": 0.1001428571428572, | |
| "shift_drop": 0.08578571428571435, | |
| "n_queries": 100 | |
| }, | |
| { | |
| "candidate_id": "retrieval-only::embedding-bge-m3::minilm", | |
| "benchmark": "dataset-neonai-qa", | |
| "model": "retrieval-only", | |
| "retriever": "embedding-bge-m3", | |
| "reranker": "minilm", | |
| "is_neon": false, | |
| "error": null, | |
| "mrr": 0.4400634920634922, | |
| "hit_at_1": 0.35, | |
| "hit_at_5": 0.57, | |
| "ndcg_at_10": 0.49663346937895064, | |
| "p50_retrieval_latency_ms": 30.563555657863617, | |
| "p95_retrieval_latency_ms": 36.099014803767204, | |
| "avg_retrieval_latency_ms": 55.99849511869252, | |
| "index_build_seconds": 0.003172708675265312, | |
| "est_cost_per_query_usd": 0.0, | |
| "generation_acc": null, | |
| "p50_generation_latency_ms": null, | |
| "avg_generation_latency_ms": null, | |
| "generation_tokens_per_sec": null, | |
| "est_generation_cost_per_query_usd": null, | |
| "paraphrase_drop": 0.033496031746031774, | |
| "shift_drop": 0.2262539682539684, | |
| "n_queries": 100 | |
| }, | |
| { | |
| "candidate_id": "retrieval-only::embedding-bge-m3::mxbai", | |
| "benchmark": "dataset-neonai-qa", | |
| "model": "retrieval-only", | |
| "retriever": "embedding-bge-m3", | |
| "reranker": "mxbai", | |
| "is_neon": false, | |
| "error": null, | |
| "mrr": 0.4017301587301587, | |
| "hit_at_1": 0.31, | |
| "hit_at_5": 0.52, | |
| "ndcg_at_10": 0.46009413645764136, | |
| "p50_retrieval_latency_ms": 201.40339527279139, | |
| "p95_retrieval_latency_ms": 204.8659548163414, | |
| "avg_retrieval_latency_ms": 231.20932274498045, | |
| "index_build_seconds": 0.0026545068249106407, | |
| "est_cost_per_query_usd": 0.0, | |
| "generation_acc": null, | |
| "p50_generation_latency_ms": null, | |
| "avg_generation_latency_ms": null, | |
| "generation_tokens_per_sec": null, | |
| "est_generation_cost_per_query_usd": null, | |
| "paraphrase_drop": 0.10631746031746037, | |
| "shift_drop": 0.1821468253968254, | |
| "n_queries": 100 | |
| }, | |
| { | |
| "candidate_id": "retrieval-only::embedding-bge-large::none", | |
| "benchmark": "dataset-neonai-qa", | |
| "model": "retrieval-only", | |
| "retriever": "embedding-bge-large", | |
| "reranker": "none", | |
| "is_neon": false, | |
| "error": null, | |
| "mrr": 0.4435873015873017, | |
| "hit_at_1": 0.36, | |
| "hit_at_5": 0.54, | |
| "ndcg_at_10": 0.49865877957298116, | |
| "p50_retrieval_latency_ms": 5.794810131192207, | |
| "p95_retrieval_latency_ms": 6.4484914764761925, | |
| "avg_retrieval_latency_ms": 111.33181081153452, | |
| "index_build_seconds": 0.0026216572150588036, | |
| "est_cost_per_query_usd": 0.0, | |
| "generation_acc": null, | |
| "p50_generation_latency_ms": null, | |
| "avg_generation_latency_ms": null, | |
| "generation_tokens_per_sec": null, | |
| "est_generation_cost_per_query_usd": null, | |
| "paraphrase_drop": 0.08009126984126985, | |
| "shift_drop": 0.16916269841269854, | |
| "n_queries": 100 | |
| }, | |
| { | |
| "candidate_id": "retrieval-only::embedding-bge-large::minilm", | |
| "benchmark": "dataset-neonai-qa", | |
| "model": "retrieval-only", | |
| "retriever": "embedding-bge-large", | |
| "reranker": "minilm", | |
| "is_neon": false, | |
| "error": null, | |
| "mrr": 0.44914285714285734, | |
| "hit_at_1": 0.34, | |
| "hit_at_5": 0.59, | |
| "ndcg_at_10": 0.5199084083920833, | |
| "p50_retrieval_latency_ms": 30.867163091897964, | |
| "p95_retrieval_latency_ms": 34.51138734817505, | |
| "avg_retrieval_latency_ms": 55.47214945778251, | |
| "index_build_seconds": 0.002636570483446121, | |
| "est_cost_per_query_usd": 0.0, | |
| "generation_acc": null, | |
| "p50_generation_latency_ms": null, | |
| "avg_generation_latency_ms": null, | |
| "generation_tokens_per_sec": null, | |
| "est_generation_cost_per_query_usd": null, | |
| "paraphrase_drop": 0.01790873015873029, | |
| "shift_drop": 0.2918809523809526, | |
| "n_queries": 100 | |
| }, | |
| { | |
| "candidate_id": "retrieval-only::embedding-bge-large::mxbai", | |
| "benchmark": "dataset-neonai-qa", | |
| "model": "retrieval-only", | |
| "retriever": "embedding-bge-large", | |
| "reranker": "mxbai", | |
| "is_neon": false, | |
| "error": null, | |
| "mrr": 0.4083293650793652, | |
| "hit_at_1": 0.32, | |
| "hit_at_5": 0.57, | |
| "ndcg_at_10": 0.46550131063059724, | |
| "p50_retrieval_latency_ms": 203.4943252801895, | |
| "p95_retrieval_latency_ms": 206.6653138026595, | |
| "avg_retrieval_latency_ms": 220.18031349405646, | |
| "index_build_seconds": 0.00274475384503603, | |
| "est_cost_per_query_usd": 0.0, | |
| "generation_acc": null, | |
| "p50_generation_latency_ms": null, | |
| "avg_generation_latency_ms": null, | |
| "generation_tokens_per_sec": null, | |
| "est_generation_cost_per_query_usd": null, | |
| "paraphrase_drop": 0.10934920634920647, | |
| "shift_drop": 0.15874603174603186, | |
| "n_queries": 100 | |
| }, | |
| { | |
| "candidate_id": "retrieval-only::hybrid::none", | |
| "benchmark": "dataset-neonai-qa", | |
| "model": "retrieval-only", | |
| "retriever": "hybrid", | |
| "reranker": "none", | |
| "is_neon": false, | |
| "error": null, | |
| "mrr": 0.5988452380952382, | |
| "hit_at_1": 0.54, | |
| "hit_at_5": 0.68, | |
| "ndcg_at_10": 0.6323249003639557, | |
| "p50_retrieval_latency_ms": 6.271526217460632, | |
| "p95_retrieval_latency_ms": 6.964513100683689, | |
| "avg_retrieval_latency_ms": 6.477495562285185, | |
| "index_build_seconds": 0.032615289092063904, | |
| "est_cost_per_query_usd": 0.0, | |
| "generation_acc": null, | |
| "p50_generation_latency_ms": null, | |
| "avg_generation_latency_ms": null, | |
| "generation_tokens_per_sec": null, | |
| "est_generation_cost_per_query_usd": null, | |
| "paraphrase_drop": 0.07990079365079361, | |
| "shift_drop": 0.3071190476190476, | |
| "n_queries": 100 | |
| }, | |
| { | |
| "candidate_id": "retrieval-only::hybrid::minilm", | |
| "benchmark": "dataset-neonai-qa", | |
| "model": "retrieval-only", | |
| "retriever": "hybrid", | |
| "reranker": "minilm", | |
| "is_neon": false, | |
| "error": null, | |
| "mrr": 0.4738650793650794, | |
| "hit_at_1": 0.36, | |
| "hit_at_5": 0.59, | |
| "ndcg_at_10": 0.5388619648224681, | |
| "p50_retrieval_latency_ms": 33.86049438267946, | |
| "p95_retrieval_latency_ms": 36.04037035256624, | |
| "avg_retrieval_latency_ms": 58.50432200357318, | |
| "index_build_seconds": 0.0028517497703433037, | |
| "est_cost_per_query_usd": 0.0, | |
| "generation_acc": null, | |
| "p50_generation_latency_ms": null, | |
| "avg_generation_latency_ms": null, | |
| "generation_tokens_per_sec": null, | |
| "est_generation_cost_per_query_usd": null, | |
| "paraphrase_drop": 0.01648412698412688, | |
| "shift_drop": 0.2568809523809524, | |
| "n_queries": 100 | |
| }, | |
| { | |
| "candidate_id": "retrieval-only::hybrid::mxbai", | |
| "benchmark": "dataset-neonai-qa", | |
| "model": "retrieval-only", | |
| "retriever": "hybrid", | |
| "reranker": "mxbai", | |
| "is_neon": false, | |
| "error": null, | |
| "mrr": 0.42051190476190475, | |
| "hit_at_1": 0.32, | |
| "hit_at_5": 0.54, | |
| "ndcg_at_10": 0.48588144180113924, | |
| "p50_retrieval_latency_ms": 207.603694871068, | |
| "p95_retrieval_latency_ms": 210.07613744586706, | |
| "avg_retrieval_latency_ms": 224.12402764894068, | |
| "index_build_seconds": 0.030656594783067703, | |
| "est_cost_per_query_usd": 0.0, | |
| "generation_acc": null, | |
| "p50_generation_latency_ms": null, | |
| "avg_generation_latency_ms": null, | |
| "generation_tokens_per_sec": null, | |
| "est_generation_cost_per_query_usd": null, | |
| "paraphrase_drop": 0.1067063492063492, | |
| "shift_drop": 0.16828968253968257, | |
| "n_queries": 100 | |
| }, | |
| { | |
| "candidate_id": "retrieval-only::hybrid-bge-large::none", | |
| "benchmark": "dataset-neonai-qa", | |
| "model": "retrieval-only", | |
| "retriever": "hybrid-bge-large", | |
| "reranker": "none", | |
| "is_neon": false, | |
| "error": null, | |
| "mrr": 0.5777896825396825, | |
| "hit_at_1": 0.52, | |
| "hit_at_5": 0.65, | |
| "ndcg_at_10": 0.6136078722712829, | |
| "p50_retrieval_latency_ms": 6.311035715043545, | |
| "p95_retrieval_latency_ms": 7.2184475138783455, | |
| "avg_retrieval_latency_ms": 6.499620405957103, | |
| "index_build_seconds": 0.0035516079515218735, | |
| "est_cost_per_query_usd": 0.0, | |
| "generation_acc": null, | |
| "p50_generation_latency_ms": null, | |
| "avg_generation_latency_ms": null, | |
| "generation_tokens_per_sec": null, | |
| "est_generation_cost_per_query_usd": null, | |
| "paraphrase_drop": 0.07380555555555546, | |
| "shift_drop": 0.2280674603174604, | |
| "n_queries": 100 | |
| }, | |
| { | |
| "candidate_id": "retrieval-only::hybrid-bge-large::minilm", | |
| "benchmark": "dataset-neonai-qa", | |
| "model": "retrieval-only", | |
| "retriever": "hybrid-bge-large", | |
| "reranker": "minilm", | |
| "is_neon": false, | |
| "error": null, | |
| "mrr": 0.48867063492063506, | |
| "hit_at_1": 0.36, | |
| "hit_at_5": 0.65, | |
| "ndcg_at_10": 0.5624087810600242, | |
| "p50_retrieval_latency_ms": 32.56149683147669, | |
| "p95_retrieval_latency_ms": 35.16612760722637, | |
| "avg_retrieval_latency_ms": 56.71209678053856, | |
| "index_build_seconds": 0.0033905887976288795, | |
| "est_cost_per_query_usd": 0.0, | |
| "generation_acc": null, | |
| "p50_generation_latency_ms": null, | |
| "avg_generation_latency_ms": null, | |
| "generation_tokens_per_sec": null, | |
| "est_generation_cost_per_query_usd": null, | |
| "paraphrase_drop": 0.03553571428571445, | |
| "shift_drop": 0.29494047619047636, | |
| "n_queries": 100 | |
| }, | |
| { | |
| "candidate_id": "retrieval-only::hybrid-bge-large::mxbai", | |
| "benchmark": "dataset-neonai-qa", | |
| "model": "retrieval-only", | |
| "retriever": "hybrid-bge-large", | |
| "reranker": "mxbai", | |
| "is_neon": false, | |
| "error": null, | |
| "mrr": 0.41178174603174617, | |
| "hit_at_1": 0.3, | |
| "hit_at_5": 0.53, | |
| "ndcg_at_10": 0.48585014131635285, | |
| "p50_retrieval_latency_ms": 207.76377152651548, | |
| "p95_retrieval_latency_ms": 210.5171848088503, | |
| "avg_retrieval_latency_ms": 224.80354332365096, | |
| "index_build_seconds": 0.003694332204759121, | |
| "est_cost_per_query_usd": 0.0, | |
| "generation_acc": null, | |
| "p50_generation_latency_ms": null, | |
| "avg_generation_latency_ms": null, | |
| "generation_tokens_per_sec": null, | |
| "est_generation_cost_per_query_usd": null, | |
| "paraphrase_drop": 0.10304761904761917, | |
| "shift_drop": 0.16664285714285731, | |
| "n_queries": 100 | |
| }, | |
| { | |
| "candidate_id": "retrieval-only::splade-only::none", | |
| "benchmark": "dataset-neonai-qa", | |
| "model": "retrieval-only", | |
| "retriever": "splade-only", | |
| "reranker": "none", | |
| "is_neon": false, | |
| "error": null, | |
| "mrr": 0.4040317460317462, | |
| "hit_at_1": 0.33, | |
| "hit_at_5": 0.47, | |
| "ndcg_at_10": 0.45812179966388017, | |
| "p50_retrieval_latency_ms": 1.7055394127964973, | |
| "p95_retrieval_latency_ms": 1.9240109249949455, | |
| "avg_retrieval_latency_ms": 43.49541657604277, | |
| "index_build_seconds": 0.0022735456004738808, | |
| "est_cost_per_query_usd": 0.0, | |
| "generation_acc": null, | |
| "p50_generation_latency_ms": null, | |
| "avg_generation_latency_ms": null, | |
| "generation_tokens_per_sec": null, | |
| "est_generation_cost_per_query_usd": null, | |
| "paraphrase_drop": 0.001369047619047714, | |
| "shift_drop": 0.2857976190476192, | |
| "n_queries": 100 | |
| }, | |
| { | |
| "candidate_id": "retrieval-only::splade-only::minilm", | |
| "benchmark": "dataset-neonai-qa", | |
| "model": "retrieval-only", | |
| "retriever": "splade-only", | |
| "reranker": "minilm", | |
| "is_neon": false, | |
| "error": null, | |
| "mrr": 0.45292460317460337, | |
| "hit_at_1": 0.36, | |
| "hit_at_5": 0.58, | |
| "ndcg_at_10": 0.5158098248243791, | |
| "p50_retrieval_latency_ms": 27.02977228909731, | |
| "p95_retrieval_latency_ms": 29.83796689659357, | |
| "avg_retrieval_latency_ms": 70.1149411778897, | |
| "index_build_seconds": 0.0018904181197285652, | |
| "est_cost_per_query_usd": 0.0, | |
| "generation_acc": null, | |
| "p50_generation_latency_ms": null, | |
| "avg_generation_latency_ms": null, | |
| "generation_tokens_per_sec": null, | |
| "est_generation_cost_per_query_usd": null, | |
| "paraphrase_drop": 0.01390476190476192, | |
| "shift_drop": 0.33814285714285736, | |
| "n_queries": 100 | |
| }, | |
| { | |
| "candidate_id": "retrieval-only::splade-only::mxbai", | |
| "benchmark": "dataset-neonai-qa", | |
| "model": "retrieval-only", | |
| "retriever": "splade-only", | |
| "reranker": "mxbai", | |
| "is_neon": false, | |
| "error": null, | |
| "mrr": 0.37042857142857144, | |
| "hit_at_1": 0.29, | |
| "hit_at_5": 0.48, | |
| "ndcg_at_10": 0.41791732493775036, | |
| "p50_retrieval_latency_ms": 202.1782398223877, | |
| "p95_retrieval_latency_ms": 205.3959835320711, | |
| "avg_retrieval_latency_ms": 218.58474116772413, | |
| "index_build_seconds": 0.0017274552956223488, | |
| "est_cost_per_query_usd": 0.0, | |
| "generation_acc": null, | |
| "p50_generation_latency_ms": null, | |
| "avg_generation_latency_ms": null, | |
| "generation_tokens_per_sec": null, | |
| "est_generation_cost_per_query_usd": null, | |
| "paraphrase_drop": 0.0752936507936508, | |
| "shift_drop": 0.1287619047619048, | |
| "n_queries": 100 | |
| }, | |
| { | |
| "candidate_id": "retrieval-only::hybrid-triple::none", | |
| "benchmark": "dataset-neonai-qa", | |
| "model": "retrieval-only", | |
| "retriever": "hybrid-triple", | |
| "reranker": "none", | |
| "is_neon": false, | |
| "error": null, | |
| "mrr": 0.5843452380952382, | |
| "hit_at_1": 0.52, | |
| "hit_at_5": 0.66, | |
| "ndcg_at_10": 0.6189039868158873, | |
| "p50_retrieval_latency_ms": 7.933019660413265, | |
| "p95_retrieval_latency_ms": 8.787033148109913, | |
| "avg_retrieval_latency_ms": 8.227292886003852, | |
| "index_build_seconds": 0.003947743214666843, | |
| "est_cost_per_query_usd": 0.0, | |
| "generation_acc": null, | |
| "p50_generation_latency_ms": null, | |
| "avg_generation_latency_ms": null, | |
| "generation_tokens_per_sec": null, | |
| "est_generation_cost_per_query_usd": null, | |
| "paraphrase_drop": 0.08083333333333331, | |
| "shift_drop": 0.27720238095238103, | |
| "n_queries": 100 | |
| }, | |
| { | |
| "candidate_id": "retrieval-only::hybrid-triple::minilm", | |
| "benchmark": "dataset-neonai-qa", | |
| "model": "retrieval-only", | |
| "retriever": "hybrid-triple", | |
| "reranker": "minilm", | |
| "is_neon": false, | |
| "error": null, | |
| "mrr": 0.4700000000000001, | |
| "hit_at_1": 0.36, | |
| "hit_at_5": 0.63, | |
| "ndcg_at_10": 0.5404693071256521, | |
| "p50_retrieval_latency_ms": 35.64351052045822, | |
| "p95_retrieval_latency_ms": 40.313600562512875, | |
| "avg_retrieval_latency_ms": 59.9335350561887, | |
| "index_build_seconds": 0.0038605360314249992, | |
| "est_cost_per_query_usd": 0.0, | |
| "generation_acc": null, | |
| "p50_generation_latency_ms": null, | |
| "avg_generation_latency_ms": null, | |
| "generation_tokens_per_sec": null, | |
| "est_generation_cost_per_query_usd": null, | |
| "paraphrase_drop": 0.010769841269841318, | |
| "shift_drop": 0.3127777777777779, | |
| "n_queries": 100 | |
| }, | |
| { | |
| "candidate_id": "retrieval-only::hybrid-triple::mxbai", | |
| "benchmark": "dataset-neonai-qa", | |
| "model": "retrieval-only", | |
| "retriever": "hybrid-triple", | |
| "reranker": "mxbai", | |
| "is_neon": false, | |
| "error": null, | |
| "mrr": 0.41017460317460336, | |
| "hit_at_1": 0.31, | |
| "hit_at_5": 0.52, | |
| "ndcg_at_10": 0.47544300858120564, | |
| "p50_retrieval_latency_ms": 208.5275650024414, | |
| "p95_retrieval_latency_ms": 211.5223752334714, | |
| "avg_retrieval_latency_ms": 224.93235109373927, | |
| "index_build_seconds": 0.0035500377416610718, | |
| "est_cost_per_query_usd": 0.0, | |
| "generation_acc": null, | |
| "p50_generation_latency_ms": null, | |
| "avg_generation_latency_ms": null, | |
| "generation_tokens_per_sec": null, | |
| "est_generation_cost_per_query_usd": null, | |
| "paraphrase_drop": 0.09506349206349224, | |
| "shift_drop": 0.17017460317460342, | |
| "n_queries": 100 | |
| }, | |
| { | |
| "candidate_id": "retrieval-only::hybrid-web::none", | |
| "benchmark": "dataset-neonai-qa", | |
| "model": "retrieval-only", | |
| "retriever": "hybrid-web", | |
| "reranker": "none", | |
| "is_neon": false, | |
| "error": null, | |
| "mrr": 0.5988452380952382, | |
| "hit_at_1": 0.54, | |
| "hit_at_5": 0.68, | |
| "ndcg_at_10": 0.6323249003639557, | |
| "p50_retrieval_latency_ms": 82.29487668722868, | |
| "p95_retrieval_latency_ms": 5015.41096996516, | |
| "avg_retrieval_latency_ms": 1194.4180446770042, | |
| "index_build_seconds": 2.039119898341596, | |
| "est_cost_per_query_usd": 0.0, | |
| "generation_acc": null, | |
| "p50_generation_latency_ms": null, | |
| "avg_generation_latency_ms": null, | |
| "generation_tokens_per_sec": null, | |
| "est_generation_cost_per_query_usd": null, | |
| "paraphrase_drop": 0.07990079365079361, | |
| "shift_drop": 0.3071190476190476, | |
| "n_queries": 100 | |
| }, | |
| { | |
| "candidate_id": "retrieval-only::hybrid-web::minilm", | |
| "benchmark": "dataset-neonai-qa", | |
| "model": "retrieval-only", | |
| "retriever": "hybrid-web", | |
| "reranker": "minilm", | |
| "is_neon": false, | |
| "error": null, | |
| "mrr": 0.4738650793650794, | |
| "hit_at_1": 0.36, | |
| "hit_at_5": 0.59, | |
| "ndcg_at_10": 0.5388619648224681, | |
| "p50_retrieval_latency_ms": 5067.3279808834195, | |
| "p95_retrieval_latency_ms": 5079.049815423787, | |
| "avg_retrieval_latency_ms": 5086.534645138308, | |
| "index_build_seconds": 1.7707714615389705, | |
| "est_cost_per_query_usd": 0.0, | |
| "generation_acc": null, | |
| "p50_generation_latency_ms": null, | |
| "avg_generation_latency_ms": null, | |
| "generation_tokens_per_sec": null, | |
| "est_generation_cost_per_query_usd": null, | |
| "paraphrase_drop": 0.01648412698412688, | |
| "shift_drop": 0.2568809523809524, | |
| "n_queries": 100 | |
| }, | |
| { | |
| "candidate_id": "retrieval-only::hybrid-web::mxbai", | |
| "benchmark": "dataset-neonai-qa", | |
| "model": "retrieval-only", | |
| "retriever": "hybrid-web", | |
| "reranker": "mxbai", | |
| "is_neon": false, | |
| "error": null, | |
| "mrr": 0.42051190476190475, | |
| "hit_at_1": 0.32, | |
| "hit_at_5": 0.54, | |
| "ndcg_at_10": 0.48588144180113924, | |
| "p50_retrieval_latency_ms": 5199.731380678713, | |
| "p95_retrieval_latency_ms": 5228.679247200489, | |
| "avg_retrieval_latency_ms": 5222.45318280533, | |
| "index_build_seconds": 1.761662513948977, | |
| "est_cost_per_query_usd": 0.0, | |
| "generation_acc": null, | |
| "p50_generation_latency_ms": null, | |
| "avg_generation_latency_ms": null, | |
| "generation_tokens_per_sec": null, | |
| "est_generation_cost_per_query_usd": null, | |
| "paraphrase_drop": 0.1067063492063492, | |
| "shift_drop": 0.16828968253968257, | |
| "n_queries": 100 | |
| }, | |
| { | |
| "candidate_id": "retrieval-only::bm25-only::none", | |
| "benchmark": "dataset-nucleotidings-qa", | |
| "model": "retrieval-only", | |
| "retriever": "bm25-only", | |
| "reranker": "none", | |
| "is_neon": false, | |
| "error": null, | |
| "mrr": 0.03176190476190476, | |
| "hit_at_1": 0.0, | |
| "hit_at_5": 0.05, | |
| "ndcg_at_10": 0.056450340414280334, | |
| "p50_retrieval_latency_ms": 0.28319843113422394, | |
| "p95_retrieval_latency_ms": 0.7622698321938515, | |
| "avg_retrieval_latency_ms": 0.3548811189830303, | |
| "index_build_seconds": 0.020172995515167713, | |
| "est_cost_per_query_usd": 0.0, | |
| "generation_acc": null, | |
| "p50_generation_latency_ms": null, | |
| "avg_generation_latency_ms": null, | |
| "generation_tokens_per_sec": null, | |
| "est_generation_cost_per_query_usd": null, | |
| "paraphrase_drop": 0.010376984126984123, | |
| "shift_drop": 0.0016031746031746029, | |
| "n_queries": 100 | |
| }, | |
| { | |
| "candidate_id": "retrieval-only::bm25-only::minilm", | |
| "benchmark": "dataset-nucleotidings-qa", | |
| "model": "retrieval-only", | |
| "retriever": "bm25-only", | |
| "reranker": "minilm", | |
| "is_neon": false, | |
| "error": null, | |
| "mrr": 0.05194444444444444, | |
| "hit_at_1": 0.02, | |
| "hit_at_5": 0.1, | |
| "ndcg_at_10": 0.06818196008810176, | |
| "p50_retrieval_latency_ms": 21.595277823507786, | |
| "p95_retrieval_latency_ms": 22.373918443918228, | |
| "avg_retrieval_latency_ms": 47.56713105365634, | |
| "index_build_seconds": 0.018008974380791187, | |
| "est_cost_per_query_usd": 0.0, | |
| "generation_acc": null, | |
| "p50_generation_latency_ms": null, | |
| "avg_generation_latency_ms": null, | |
| "generation_tokens_per_sec": null, | |
| "est_generation_cost_per_query_usd": null, | |
| "paraphrase_drop": 0.009765873015873013, | |
| "shift_drop": 0.05194444444444444, | |
| "n_queries": 100 | |
| }, | |
| { | |
| "candidate_id": "retrieval-only::bm25-only::mxbai", | |
| "benchmark": "dataset-nucleotidings-qa", | |
| "model": "retrieval-only", | |
| "retriever": "bm25-only", | |
| "reranker": "mxbai", | |
| "is_neon": false, | |
| "error": null, | |
| "mrr": 0.03586904761904762, | |
| "hit_at_1": 0.02, | |
| "hit_at_5": 0.06, | |
| "ndcg_at_10": 0.05063670800167056, | |
| "p50_retrieval_latency_ms": 195.6018004566431, | |
| "p95_retrieval_latency_ms": 199.5778325945139, | |
| "avg_retrieval_latency_ms": 211.85935158282518, | |
| "index_build_seconds": 0.020156115293502808, | |
| "est_cost_per_query_usd": 0.0, | |
| "generation_acc": null, | |
| "p50_generation_latency_ms": null, | |
| "avg_generation_latency_ms": null, | |
| "generation_tokens_per_sec": null, | |
| "est_generation_cost_per_query_usd": null, | |
| "paraphrase_drop": -0.01257539682539683, | |
| "shift_drop": 0.03586904761904762, | |
| "n_queries": 100 | |
| }, | |
| { | |
| "candidate_id": "retrieval-only::embedding-bge-m3::none", | |
| "benchmark": "dataset-nucleotidings-qa", | |
| "model": "retrieval-only", | |
| "retriever": "embedding-bge-m3", | |
| "reranker": "none", | |
| "is_neon": false, | |
| "error": null, | |
| "mrr": 0.03541666666666667, | |
| "hit_at_1": 0.01, | |
| "hit_at_5": 0.06, | |
| "ndcg_at_10": 0.061047911233524085, | |
| "p50_retrieval_latency_ms": 5.855381488800049, | |
| "p95_retrieval_latency_ms": 6.634416989982128, | |
| "avg_retrieval_latency_ms": 6.102179624140263, | |
| "index_build_seconds": 0.04834281001240015, | |
| "est_cost_per_query_usd": 0.0, | |
| "generation_acc": null, | |
| "p50_generation_latency_ms": null, | |
| "avg_generation_latency_ms": null, | |
| "generation_tokens_per_sec": null, | |
| "est_generation_cost_per_query_usd": null, | |
| "paraphrase_drop": 0.007460317460317462, | |
| "shift_drop": 0.03541666666666667, | |
| "n_queries": 100 | |
| }, | |
| { | |
| "candidate_id": "retrieval-only::embedding-bge-m3::minilm", | |
| "benchmark": "dataset-nucleotidings-qa", | |
| "model": "retrieval-only", | |
| "retriever": "embedding-bge-m3", | |
| "reranker": "minilm", | |
| "is_neon": false, | |
| "error": null, | |
| "mrr": 0.05586111111111112, | |
| "hit_at_1": 0.01, | |
| "hit_at_5": 0.13, | |
| "ndcg_at_10": 0.08305216356875389, | |
| "p50_retrieval_latency_ms": 28.12165580689907, | |
| "p95_retrieval_latency_ms": 29.309977777302265, | |
| "avg_retrieval_latency_ms": 52.21044613048434, | |
| "index_build_seconds": 0.019170415587723255, | |
| "est_cost_per_query_usd": 0.0, | |
| "generation_acc": null, | |
| "p50_generation_latency_ms": null, | |
| "avg_generation_latency_ms": null, | |
| "generation_tokens_per_sec": null, | |
| "est_generation_cost_per_query_usd": null, | |
| "paraphrase_drop": 0.010932539682539683, | |
| "shift_drop": 0.05586111111111112, | |
| "n_queries": 100 | |
| }, | |
| { | |
| "candidate_id": "retrieval-only::embedding-bge-m3::mxbai", | |
| "benchmark": "dataset-nucleotidings-qa", | |
| "model": "retrieval-only", | |
| "retriever": "embedding-bge-m3", | |
| "reranker": "mxbai", | |
| "is_neon": false, | |
| "error": null, | |
| "mrr": 0.050857142857142865, | |
| "hit_at_1": 0.02, | |
| "hit_at_5": 0.07, | |
| "ndcg_at_10": 0.0759087886743895, | |
| "p50_retrieval_latency_ms": 206.3822029158473, | |
| "p95_retrieval_latency_ms": 210.27577761560678, | |
| "avg_retrieval_latency_ms": 222.049624286592, | |
| "index_build_seconds": 0.05048837885260582, | |
| "est_cost_per_query_usd": 0.0, | |
| "generation_acc": null, | |
| "p50_generation_latency_ms": null, | |
| "avg_generation_latency_ms": null, | |
| "generation_tokens_per_sec": null, | |
| "est_generation_cost_per_query_usd": null, | |
| "paraphrase_drop": 0.0085, | |
| "shift_drop": 0.050857142857142865, | |
| "n_queries": 100 | |
| }, | |
| { | |
| "candidate_id": "retrieval-only::embedding-bge-large::none", | |
| "benchmark": "dataset-nucleotidings-qa", | |
| "model": "retrieval-only", | |
| "retriever": "embedding-bge-large", | |
| "reranker": "none", | |
| "is_neon": false, | |
| "error": null, | |
| "mrr": 0.030083333333333333, | |
| "hit_at_1": 0.0, | |
| "hit_at_5": 0.07, | |
| "ndcg_at_10": 0.04671055516233946, | |
| "p50_retrieval_latency_ms": 5.988357588648796, | |
| "p95_retrieval_latency_ms": 6.613876670598984, | |
| "avg_retrieval_latency_ms": 6.17221912369132, | |
| "index_build_seconds": 0.01803918555378914, | |
| "est_cost_per_query_usd": 0.0, | |
| "generation_acc": null, | |
| "p50_generation_latency_ms": null, | |
| "avg_generation_latency_ms": null, | |
| "generation_tokens_per_sec": null, | |
| "est_generation_cost_per_query_usd": null, | |
| "paraphrase_drop": 0.003416666666666665, | |
| "shift_drop": 0.030083333333333333, | |
| "n_queries": 100 | |
| }, | |
| { | |
| "candidate_id": "retrieval-only::embedding-bge-large::minilm", | |
| "benchmark": "dataset-nucleotidings-qa", | |
| "model": "retrieval-only", | |
| "retriever": "embedding-bge-large", | |
| "reranker": "minilm", | |
| "is_neon": false, | |
| "error": null, | |
| "mrr": 0.043777777777777784, | |
| "hit_at_1": 0.01, | |
| "hit_at_5": 0.1, | |
| "ndcg_at_10": 0.07039101522343577, | |
| "p50_retrieval_latency_ms": 27.764969505369663, | |
| "p95_retrieval_latency_ms": 29.288841411471367, | |
| "avg_retrieval_latency_ms": 52.290981682017446, | |
| "index_build_seconds": 0.019241033121943474, | |
| "est_cost_per_query_usd": 0.0, | |
| "generation_acc": null, | |
| "p50_generation_latency_ms": null, | |
| "avg_generation_latency_ms": null, | |
| "generation_tokens_per_sec": null, | |
| "est_generation_cost_per_query_usd": null, | |
| "paraphrase_drop": 0.007388888888888889, | |
| "shift_drop": 0.043777777777777784, | |
| "n_queries": 100 | |
| }, | |
| { | |
| "candidate_id": "retrieval-only::embedding-bge-large::mxbai", | |
| "benchmark": "dataset-nucleotidings-qa", | |
| "model": "retrieval-only", | |
| "retriever": "embedding-bge-large", | |
| "reranker": "mxbai", | |
| "is_neon": false, | |
| "error": null, | |
| "mrr": 0.046880952380952384, | |
| "hit_at_1": 0.02, | |
| "hit_at_5": 0.06, | |
| "ndcg_at_10": 0.06816413425401462, | |
| "p50_retrieval_latency_ms": 206.22909720987082, | |
| "p95_retrieval_latency_ms": 210.59383265674114, | |
| "avg_retrieval_latency_ms": 224.03612443245947, | |
| "index_build_seconds": 0.0204993961378932, | |
| "est_cost_per_query_usd": 0.0, | |
| "generation_acc": null, | |
| "p50_generation_latency_ms": null, | |
| "avg_generation_latency_ms": null, | |
| "generation_tokens_per_sec": null, | |
| "est_generation_cost_per_query_usd": null, | |
| "paraphrase_drop": 0.0012619047619047696, | |
| "shift_drop": 0.046880952380952384, | |
| "n_queries": 100 | |
| }, | |
| { | |
| "candidate_id": "retrieval-only::hybrid::none", | |
| "benchmark": "dataset-nucleotidings-qa", | |
| "model": "retrieval-only", | |
| "retriever": "hybrid", | |
| "reranker": "none", | |
| "is_neon": false, | |
| "error": null, | |
| "mrr": 0.048261904761904756, | |
| "hit_at_1": 0.01, | |
| "hit_at_5": 0.1, | |
| "ndcg_at_10": 0.06569076601728688, | |
| "p50_retrieval_latency_ms": 6.575613282620907, | |
| "p95_retrieval_latency_ms": 7.236745208501816, | |
| "avg_retrieval_latency_ms": 6.708775740116835, | |
| "index_build_seconds": 0.0318792499601841, | |
| "est_cost_per_query_usd": 0.0, | |
| "generation_acc": null, | |
| "p50_generation_latency_ms": null, | |
| "avg_generation_latency_ms": null, | |
| "generation_tokens_per_sec": null, | |
| "est_generation_cost_per_query_usd": null, | |
| "paraphrase_drop": 0.015111111111111103, | |
| "shift_drop": 0.048261904761904756, | |
| "n_queries": 100 | |
| }, | |
| { | |
| "candidate_id": "retrieval-only::hybrid::minilm", | |
| "benchmark": "dataset-nucleotidings-qa", | |
| "model": "retrieval-only", | |
| "retriever": "hybrid", | |
| "reranker": "minilm", | |
| "is_neon": false, | |
| "error": null, | |
| "mrr": 0.06034523809523809, | |
| "hit_at_1": 0.02, | |
| "hit_at_5": 0.12, | |
| "ndcg_at_10": 0.08610983041193102, | |
| "p50_retrieval_latency_ms": 29.029096476733685, | |
| "p95_retrieval_latency_ms": 30.2345072850585, | |
| "avg_retrieval_latency_ms": 69.65694262646139, | |
| "index_build_seconds": 0.02931369189172983, | |
| "est_cost_per_query_usd": 0.0, | |
| "generation_acc": null, | |
| "p50_generation_latency_ms": null, | |
| "avg_generation_latency_ms": null, | |
| "generation_tokens_per_sec": null, | |
| "est_generation_cost_per_query_usd": null, | |
| "paraphrase_drop": 0.018138888888888885, | |
| "shift_drop": 0.06034523809523809, | |
| "n_queries": 100 | |
| }, | |
| { | |
| "candidate_id": "retrieval-only::hybrid::mxbai", | |
| "benchmark": "dataset-nucleotidings-qa", | |
| "model": "retrieval-only", | |
| "retriever": "hybrid", | |
| "reranker": "mxbai", | |
| "is_neon": false, | |
| "error": null, | |
| "mrr": 0.05573015873015873, | |
| "hit_at_1": 0.03, | |
| "hit_at_5": 0.09, | |
| "ndcg_at_10": 0.07737790184364975, | |
| "p50_retrieval_latency_ms": 206.8657660856843, | |
| "p95_retrieval_latency_ms": 209.80359707027674, | |
| "avg_retrieval_latency_ms": 223.3355513587594, | |
| "index_build_seconds": 0.0300194276496768, | |
| "est_cost_per_query_usd": 0.0, | |
| "generation_acc": null, | |
| "p50_generation_latency_ms": null, | |
| "avg_generation_latency_ms": null, | |
| "generation_tokens_per_sec": null, | |
| "est_generation_cost_per_query_usd": null, | |
| "paraphrase_drop": 0.014634920634920633, | |
| "shift_drop": 0.05573015873015873, | |
| "n_queries": 100 | |
| }, | |
| { | |
| "candidate_id": "retrieval-only::hybrid-bge-large::none", | |
| "benchmark": "dataset-nucleotidings-qa", | |
| "model": "retrieval-only", | |
| "retriever": "hybrid-bge-large", | |
| "reranker": "none", | |
| "is_neon": false, | |
| "error": null, | |
| "mrr": 0.02687301587301587, | |
| "hit_at_1": 0.0, | |
| "hit_at_5": 0.07, | |
| "ndcg_at_10": 0.048346886529284386, | |
| "p50_retrieval_latency_ms": 6.49019330739975, | |
| "p95_retrieval_latency_ms": 7.493586279451847, | |
| "avg_retrieval_latency_ms": 6.714150300249457, | |
| "index_build_seconds": 0.030312607064843178, | |
| "est_cost_per_query_usd": 0.0, | |
| "generation_acc": null, | |
| "p50_generation_latency_ms": null, | |
| "avg_generation_latency_ms": null, | |
| "generation_tokens_per_sec": null, | |
| "est_generation_cost_per_query_usd": null, | |
| "paraphrase_drop": 0.005111111111111108, | |
| "shift_drop": 0.02687301587301587, | |
| "n_queries": 100 | |
| }, | |
| { | |
| "candidate_id": "retrieval-only::hybrid-bge-large::minilm", | |
| "benchmark": "dataset-nucleotidings-qa", | |
| "model": "retrieval-only", | |
| "retriever": "hybrid-bge-large", | |
| "reranker": "minilm", | |
| "is_neon": false, | |
| "error": null, | |
| "mrr": 0.06802380952380953, | |
| "hit_at_1": 0.01, | |
| "hit_at_5": 0.15, | |
| "ndcg_at_10": 0.10615988438420187, | |
| "p50_retrieval_latency_ms": 29.03958596289158, | |
| "p95_retrieval_latency_ms": 30.06952255964279, | |
| "avg_retrieval_latency_ms": 54.366486547514796, | |
| "index_build_seconds": 0.030157318338751793, | |
| "est_cost_per_query_usd": 0.0, | |
| "generation_acc": null, | |
| "p50_generation_latency_ms": null, | |
| "avg_generation_latency_ms": null, | |
| "generation_tokens_per_sec": null, | |
| "est_generation_cost_per_query_usd": null, | |
| "paraphrase_drop": 0.03101190476190476, | |
| "shift_drop": 0.029758503401360548, | |
| "n_queries": 100 | |
| }, | |
| { | |
| "candidate_id": "retrieval-only::hybrid-bge-large::mxbai", | |
| "benchmark": "dataset-nucleotidings-qa", | |
| "model": "retrieval-only", | |
| "retriever": "hybrid-bge-large", | |
| "reranker": "mxbai", | |
| "is_neon": false, | |
| "error": null, | |
| "mrr": 0.05845238095238095, | |
| "hit_at_1": 0.02, | |
| "hit_at_5": 0.1, | |
| "ndcg_at_10": 0.08229580387362238, | |
| "p50_retrieval_latency_ms": 206.1399845406413, | |
| "p95_retrieval_latency_ms": 209.65103805065155, | |
| "avg_retrieval_latency_ms": 223.03275553509593, | |
| "index_build_seconds": 0.030014587566256523, | |
| "est_cost_per_query_usd": 0.0, | |
| "generation_acc": null, | |
| "p50_generation_latency_ms": null, | |
| "avg_generation_latency_ms": null, | |
| "generation_tokens_per_sec": null, | |
| "est_generation_cost_per_query_usd": null, | |
| "paraphrase_drop": 0.01740476190476191, | |
| "shift_drop": 0.01763605442176871, | |
| "n_queries": 100 | |
| }, | |
| { | |
| "candidate_id": "retrieval-only::splade-only::none", | |
| "benchmark": "dataset-nucleotidings-qa", | |
| "model": "retrieval-only", | |
| "retriever": "splade-only", | |
| "reranker": "none", | |
| "is_neon": false, | |
| "error": null, | |
| "mrr": 0.04010714285714286, | |
| "hit_at_1": 0.0, | |
| "hit_at_5": 0.11, | |
| "ndcg_at_10": 0.0639683234201718, | |
| "p50_retrieval_latency_ms": 3.832290880382061, | |
| "p95_retrieval_latency_ms": 4.106180742383003, | |
| "avg_retrieval_latency_ms": 3.891169335693121, | |
| "index_build_seconds": 0.01208497118204832, | |
| "est_cost_per_query_usd": 0.0, | |
| "generation_acc": null, | |
| "p50_generation_latency_ms": null, | |
| "avg_generation_latency_ms": null, | |
| "generation_tokens_per_sec": null, | |
| "est_generation_cost_per_query_usd": null, | |
| "paraphrase_drop": 0.004273809523809534, | |
| "shift_drop": 0.04010714285714286, | |
| "n_queries": 100 | |
| }, | |
| { | |
| "candidate_id": "retrieval-only::splade-only::minilm", | |
| "benchmark": "dataset-nucleotidings-qa", | |
| "model": "retrieval-only", | |
| "retriever": "splade-only", | |
| "reranker": "minilm", | |
| "is_neon": false, | |
| "error": null, | |
| "mrr": 0.05749603174603173, | |
| "hit_at_1": 0.02, | |
| "hit_at_5": 0.1, | |
| "ndcg_at_10": 0.0837599478144725, | |
| "p50_retrieval_latency_ms": 26.7010610550642, | |
| "p95_retrieval_latency_ms": 27.352082543075085, | |
| "avg_retrieval_latency_ms": 53.00288221798837, | |
| "index_build_seconds": 0.010814650915563107, | |
| "est_cost_per_query_usd": 0.0, | |
| "generation_acc": null, | |
| "p50_generation_latency_ms": null, | |
| "avg_generation_latency_ms": null, | |
| "generation_tokens_per_sec": null, | |
| "est_generation_cost_per_query_usd": null, | |
| "paraphrase_drop": 0.019996031746031734, | |
| "shift_drop": 0.05749603174603173, | |
| "n_queries": 100 | |
| }, | |
| { | |
| "candidate_id": "retrieval-only::splade-only::mxbai", | |
| "benchmark": "dataset-nucleotidings-qa", | |
| "model": "retrieval-only", | |
| "retriever": "splade-only", | |
| "reranker": "mxbai", | |
| "is_neon": false, | |
| "error": null, | |
| "mrr": 0.05083333333333334, | |
| "hit_at_1": 0.02, | |
| "hit_at_5": 0.1, | |
| "ndcg_at_10": 0.07383673926531788, | |
| "p50_retrieval_latency_ms": 203.553250990808, | |
| "p95_retrieval_latency_ms": 207.03642163425684, | |
| "avg_retrieval_latency_ms": 221.47191389463842, | |
| "index_build_seconds": 0.03780477773398161, | |
| "est_cost_per_query_usd": 0.0, | |
| "generation_acc": null, | |
| "p50_generation_latency_ms": null, | |
| "avg_generation_latency_ms": null, | |
| "generation_tokens_per_sec": null, | |
| "est_generation_cost_per_query_usd": null, | |
| "paraphrase_drop": 0.014710317460317465, | |
| "shift_drop": 0.05083333333333334, | |
| "n_queries": 100 | |
| }, | |
| { | |
| "candidate_id": "retrieval-only::hybrid-triple::none", | |
| "benchmark": "dataset-nucleotidings-qa", | |
| "model": "retrieval-only", | |
| "retriever": "hybrid-triple", | |
| "reranker": "none", | |
| "is_neon": false, | |
| "error": null, | |
| "mrr": 0.057595238095238095, | |
| "hit_at_1": 0.02, | |
| "hit_at_5": 0.1, | |
| "ndcg_at_10": 0.07731042095533185, | |
| "p50_retrieval_latency_ms": 11.1324368044734, | |
| "p95_retrieval_latency_ms": 12.249251827597618, | |
| "avg_retrieval_latency_ms": 11.381908534094691, | |
| "index_build_seconds": 0.03214253671467304, | |
| "est_cost_per_query_usd": 0.0, | |
| "generation_acc": null, | |
| "p50_generation_latency_ms": null, | |
| "avg_generation_latency_ms": null, | |
| "generation_tokens_per_sec": null, | |
| "est_generation_cost_per_query_usd": null, | |
| "paraphrase_drop": 0.020178571428571428, | |
| "shift_drop": 0.057595238095238095, | |
| "n_queries": 100 | |
| }, | |
| { | |
| "candidate_id": "retrieval-only::hybrid-triple::minilm", | |
| "benchmark": "dataset-nucleotidings-qa", | |
| "model": "retrieval-only", | |
| "retriever": "hybrid-triple", | |
| "reranker": "minilm", | |
| "is_neon": false, | |
| "error": null, | |
| "mrr": 0.0634126984126984, | |
| "hit_at_1": 0.02, | |
| "hit_at_5": 0.11, | |
| "ndcg_at_10": 0.09714608128796602, | |
| "p50_retrieval_latency_ms": 33.852593973279, | |
| "p95_retrieval_latency_ms": 34.9635211750865, | |
| "avg_retrieval_latency_ms": 58.79330011084676, | |
| "index_build_seconds": 0.03294236399233341, | |
| "est_cost_per_query_usd": 0.0, | |
| "generation_acc": null, | |
| "p50_generation_latency_ms": null, | |
| "avg_generation_latency_ms": null, | |
| "generation_tokens_per_sec": null, | |
| "est_generation_cost_per_query_usd": null, | |
| "paraphrase_drop": 0.020718253968253955, | |
| "shift_drop": 0.0634126984126984, | |
| "n_queries": 100 | |
| }, | |
| { | |
| "candidate_id": "retrieval-only::hybrid-triple::mxbai", | |
| "benchmark": "dataset-nucleotidings-qa", | |
| "model": "retrieval-only", | |
| "retriever": "hybrid-triple", | |
| "reranker": "mxbai", | |
| "is_neon": false, | |
| "error": null, | |
| "mrr": 0.05472222222222222, | |
| "hit_at_1": 0.03, | |
| "hit_at_5": 0.08, | |
| "ndcg_at_10": 0.07404850529840498, | |
| "p50_retrieval_latency_ms": 207.9247822985053, | |
| "p95_retrieval_latency_ms": 210.9046708792448, | |
| "avg_retrieval_latency_ms": 225.0969840027392, | |
| "index_build_seconds": 0.03245750721544027, | |
| "est_cost_per_query_usd": 0.0, | |
| "generation_acc": null, | |
| "p50_generation_latency_ms": null, | |
| "avg_generation_latency_ms": null, | |
| "generation_tokens_per_sec": null, | |
| "est_generation_cost_per_query_usd": null, | |
| "paraphrase_drop": 0.00942063492063492, | |
| "shift_drop": 0.05472222222222222, | |
| "n_queries": 100 | |
| }, | |
| { | |
| "candidate_id": "retrieval-only::hybrid-web::none", | |
| "benchmark": "dataset-nucleotidings-qa", | |
| "model": "retrieval-only", | |
| "retriever": "hybrid-web", | |
| "reranker": "none", | |
| "is_neon": false, | |
| "error": null, | |
| "mrr": 0.048261904761904756, | |
| "hit_at_1": 0.01, | |
| "hit_at_5": 0.1, | |
| "ndcg_at_10": 0.06569076601728688, | |
| "p50_retrieval_latency_ms": 5015.710390172899, | |
| "p95_retrieval_latency_ms": 5045.650668442249, | |
| "avg_retrieval_latency_ms": 5023.558369465172, | |
| "index_build_seconds": 29.299102989025414, | |
| "est_cost_per_query_usd": 0.0, | |
| "generation_acc": null, | |
| "p50_generation_latency_ms": null, | |
| "avg_generation_latency_ms": null, | |
| "generation_tokens_per_sec": null, | |
| "est_generation_cost_per_query_usd": null, | |
| "paraphrase_drop": 0.015111111111111103, | |
| "shift_drop": 0.048261904761904756, | |
| "n_queries": 100 | |
| }, | |
| { | |
| "candidate_id": "retrieval-only::hybrid-web::minilm", | |
| "benchmark": "dataset-nucleotidings-qa", | |
| "model": "retrieval-only", | |
| "retriever": "hybrid-web", | |
| "reranker": "minilm", | |
| "is_neon": false, | |
| "error": null, | |
| "mrr": 0.06034523809523809, | |
| "hit_at_1": 0.02, | |
| "hit_at_5": 0.12, | |
| "ndcg_at_10": 0.08610983041193102, | |
| "p50_retrieval_latency_ms": 5062.78524082154, | |
| "p95_retrieval_latency_ms": 5068.56173556298, | |
| "avg_retrieval_latency_ms": 5080.987708000466, | |
| "index_build_seconds": 26.2965495781973, | |
| "est_cost_per_query_usd": 0.0, | |
| "generation_acc": null, | |
| "p50_generation_latency_ms": null, | |
| "avg_generation_latency_ms": null, | |
| "generation_tokens_per_sec": null, | |
| "est_generation_cost_per_query_usd": null, | |
| "paraphrase_drop": 0.018138888888888885, | |
| "shift_drop": 0.06034523809523809, | |
| "n_queries": 100 | |
| }, | |
| { | |
| "candidate_id": "retrieval-only::hybrid-web::mxbai", | |
| "benchmark": "dataset-nucleotidings-qa", | |
| "model": "retrieval-only", | |
| "retriever": "hybrid-web", | |
| "reranker": "mxbai", | |
| "is_neon": false, | |
| "error": null, | |
| "mrr": 0.05573015873015873, | |
| "hit_at_1": 0.03, | |
| "hit_at_5": 0.09, | |
| "ndcg_at_10": 0.07737790184364975, | |
| "p50_retrieval_latency_ms": 5195.400764234364, | |
| "p95_retrieval_latency_ms": 5222.220067866147, | |
| "avg_retrieval_latency_ms": 5219.289752468467, | |
| "index_build_seconds": 26.202580992132425, | |
| "est_cost_per_query_usd": 0.0, | |
| "generation_acc": null, | |
| "p50_generation_latency_ms": null, | |
| "avg_generation_latency_ms": null, | |
| "generation_tokens_per_sec": null, | |
| "est_generation_cost_per_query_usd": null, | |
| "paraphrase_drop": 0.014634920634920633, | |
| "shift_drop": 0.05573015873015873, | |
| "n_queries": 100 | |
| }, | |
| { | |
| "candidate_id": "devstral-123b::hybrid::none", | |
| "benchmark": "dataset-neonai-qa", | |
| "model": "devstral-123b", | |
| "retriever": "hybrid", | |
| "reranker": "none", | |
| "is_neon": true, | |
| "error": null, | |
| "mrr": 0.5988452380952382, | |
| "hit_at_1": 0.54, | |
| "hit_at_5": 0.68, | |
| "ndcg_at_10": 0.6323249003639557, | |
| "p50_retrieval_latency_ms": 7.858959957957268, | |
| "p95_retrieval_latency_ms": 10.774773545563221, | |
| "avg_retrieval_latency_ms": 8.430057065561414, | |
| "index_build_seconds": 0.002399919554591179, | |
| "est_cost_per_query_usd": 0.0011859399999999997, | |
| "generation_acc": 0.92, | |
| "p50_generation_latency_ms": 640.4259856790304, | |
| "avg_generation_latency_ms": 795.348126348108, | |
| "generation_tokens_per_sec": 712.2525639619537, | |
| "est_generation_cost_per_query_usd": 0.0011859399999999997, | |
| "paraphrase_drop": 0.07990079365079361, | |
| "shift_drop": 0.3071190476190476, | |
| "n_queries": 100 | |
| }, | |
| { | |
| "candidate_id": "devstral-123b::hybrid::none", | |
| "benchmark": "dataset-nucleotidings-qa", | |
| "model": "devstral-123b", | |
| "retriever": "hybrid", | |
| "reranker": "none", | |
| "is_neon": true, | |
| "error": null, | |
| "mrr": 0.048261904761904756, | |
| "hit_at_1": 0.01, | |
| "hit_at_5": 0.1, | |
| "ndcg_at_10": 0.06569076601728688, | |
| "p50_retrieval_latency_ms": 8.428102359175682, | |
| "p95_retrieval_latency_ms": 9.656098671257496, | |
| "avg_retrieval_latency_ms": 8.969784323126078, | |
| "index_build_seconds": 0.022853504866361618, | |
| "est_cost_per_query_usd": 0.0013255999999999993, | |
| "generation_acc": 0.66, | |
| "p50_generation_latency_ms": 703.2549865543842, | |
| "avg_generation_latency_ms": 882.6579355821013, | |
| "generation_tokens_per_sec": 719.36760451188, | |
| "est_generation_cost_per_query_usd": 0.0013255999999999993, | |
| "paraphrase_drop": 0.015111111111111103, | |
| "shift_drop": 0.048261904761904756, | |
| "n_queries": 100 | |
| }, | |
| { | |
| "candidate_id": "devstral-123b::hybrid::none", | |
| "benchmark": "dataset-logistics-qa", | |
| "model": "devstral-123b", | |
| "retriever": "hybrid", | |
| "reranker": "none", | |
| "is_neon": true, | |
| "error": null, | |
| "mrr": 1.0, | |
| "hit_at_1": 1.0, | |
| "hit_at_5": 1.0, | |
| "ndcg_at_10": 1.0, | |
| "p50_retrieval_latency_ms": 7.7611226588487625, | |
| "p95_retrieval_latency_ms": 8.55210516601801, | |
| "avg_retrieval_latency_ms": 7.866447418928146, | |
| "index_build_seconds": 0.0368603216484189, | |
| "est_cost_per_query_usd": 0.001025, | |
| "generation_acc": 0.71, | |
| "p50_generation_latency_ms": 617.4995861947536, | |
| "avg_generation_latency_ms": 736.3612976484001, | |
| "generation_tokens_per_sec": 676.4109967916651, | |
| "est_generation_cost_per_query_usd": 0.001025, | |
| "paraphrase_drop": 0.0, | |
| "shift_drop": 0.0, | |
| "n_queries": 100 | |
| }, | |
| { | |
| "candidate_id": "devstral-123b::hybrid::none", | |
| "benchmark": "dataset-landuse-wa-qa", | |
| "model": "devstral-123b", | |
| "retriever": "hybrid", | |
| "reranker": "none", | |
| "is_neon": true, | |
| "error": null, | |
| "mrr": 1.0, | |
| "hit_at_1": 1.0, | |
| "hit_at_5": 1.0, | |
| "ndcg_at_10": 1.0, | |
| "p50_retrieval_latency_ms": 7.857654243707657, | |
| "p95_retrieval_latency_ms": 8.58728401362896, | |
| "avg_retrieval_latency_ms": 8.052368555217981, | |
| "index_build_seconds": 0.007834444753825665, | |
| "est_cost_per_query_usd": 0.0012769, | |
| "generation_acc": 0.89, | |
| "p50_generation_latency_ms": 794.1247960552573, | |
| "avg_generation_latency_ms": 883.8255668338388, | |
| "generation_tokens_per_sec": 705.0630370070047, | |
| "est_generation_cost_per_query_usd": 0.0012769, | |
| "paraphrase_drop": 0.0, | |
| "shift_drop": 1.0, | |
| "n_queries": 100 | |
| }, | |
| { | |
| "candidate_id": "devstral-123b::hybrid::none", | |
| "benchmark": "dataset-shakespeare-qa", | |
| "model": "devstral-123b", | |
| "retriever": "hybrid", | |
| "reranker": "none", | |
| "is_neon": true, | |
| "error": null, | |
| "mrr": 1.0, | |
| "hit_at_1": 1.0, | |
| "hit_at_5": 1.0, | |
| "ndcg_at_10": 1.0, | |
| "p50_retrieval_latency_ms": 8.404421620070934, | |
| "p95_retrieval_latency_ms": 8.764853700995445, | |
| "avg_retrieval_latency_ms": 8.54923996143043, | |
| "index_build_seconds": 0.05274257529526949, | |
| "est_cost_per_query_usd": 0.0005438600000000001, | |
| "generation_acc": 0.71, | |
| "p50_generation_latency_ms": 461.6543473675847, | |
| "avg_generation_latency_ms": 579.4168068934232, | |
| "generation_tokens_per_sec": 556.6365243323613, | |
| "est_generation_cost_per_query_usd": 0.0005438600000000001, | |
| "paraphrase_drop": 0.0, | |
| "shift_drop": 0.0, | |
| "n_queries": 100 | |
| }, | |
| { | |
| "candidate_id": "devstral-123b::hybrid::none", | |
| "benchmark": "dataset-futurewise-qa", | |
| "model": "devstral-123b", | |
| "retriever": "hybrid", | |
| "reranker": "none", | |
| "is_neon": true, | |
| "error": null, | |
| "mrr": 0.09780952380952382, | |
| "hit_at_1": 0.03, | |
| "hit_at_5": 0.19, | |
| "ndcg_at_10": 0.13218233672282742, | |
| "p50_retrieval_latency_ms": 8.942721411585808, | |
| "p95_retrieval_latency_ms": 9.492317214608192, | |
| "avg_retrieval_latency_ms": 8.958483096212149, | |
| "index_build_seconds": 0.07995483838021755, | |
| "est_cost_per_query_usd": 0.0005834799999999999, | |
| "generation_acc": 0.77, | |
| "p50_generation_latency_ms": 445.1367845758796, | |
| "avg_generation_latency_ms": 535.1428753789514, | |
| "generation_tokens_per_sec": 584.1880310153064, | |
| "est_generation_cost_per_query_usd": 0.0005834799999999999, | |
| "paraphrase_drop": 0.03988095238095239, | |
| "shift_drop": 0.04168707482993198, | |
| "n_queries": 100 | |
| }, | |
| { | |
| "candidate_id": "devstral-123b::hybrid::none", | |
| "benchmark": "dataset-belebele-qa", | |
| "model": "devstral-123b", | |
| "retriever": "hybrid", | |
| "reranker": "none", | |
| "is_neon": true, | |
| "error": null, | |
| "mrr": 0.9678571428571429, | |
| "hit_at_1": 0.95, | |
| "hit_at_5": 0.9928571428571429, | |
| "ndcg_at_10": 0.974179086645947, | |
| "p50_retrieval_latency_ms": 8.227682672441006, | |
| "p95_retrieval_latency_ms": 8.662175387144089, | |
| "avg_retrieval_latency_ms": 8.255848175446902, | |
| "index_build_seconds": 0.031107299029827118, | |
| "est_cost_per_query_usd": 0.000441642857142857, | |
| "generation_acc": 0.9, | |
| "p50_generation_latency_ms": 418.36012806743383, | |
| "avg_generation_latency_ms": 428.0198134681476, | |
| "generation_tokens_per_sec": 503.8791067857494, | |
| "est_generation_cost_per_query_usd": 0.000441642857142857, | |
| "paraphrase_drop": 0.06440476190476185, | |
| "shift_drop": 0.0035714285714285587, | |
| "n_queries": 140 | |
| }, | |
| { | |
| "candidate_id": "ministral-14b-reasoning::hybrid::none", | |
| "benchmark": "dataset-neonai-qa", | |
| "model": "ministral-14b-reasoning", | |
| "retriever": "hybrid", | |
| "reranker": "none", | |
| "is_neon": true, | |
| "error": null, | |
| "mrr": 0.5988452380952382, | |
| "hit_at_1": 0.54, | |
| "hit_at_5": 0.68, | |
| "ndcg_at_10": 0.6323249003639557, | |
| "p50_retrieval_latency_ms": 7.1721151471138, | |
| "p95_retrieval_latency_ms": 7.782273925840855, | |
| "avg_retrieval_latency_ms": 7.279454693198204, | |
| "index_build_seconds": 0.03248952701687813, | |
| "est_cost_per_query_usd": 0.0, | |
| "generation_acc": null, | |
| "p50_generation_latency_ms": null, | |
| "avg_generation_latency_ms": null, | |
| "generation_tokens_per_sec": null, | |
| "est_generation_cost_per_query_usd": null, | |
| "paraphrase_drop": 0.07990079365079361, | |
| "shift_drop": 0.3071190476190476, | |
| "n_queries": 100 | |
| }, | |
| { | |
| "candidate_id": "ministral-14b-reasoning::hybrid::none", | |
| "benchmark": "dataset-nucleotidings-qa", | |
| "model": "ministral-14b-reasoning", | |
| "retriever": "hybrid", | |
| "reranker": "none", | |
| "is_neon": true, | |
| "error": null, | |
| "mrr": 0.048261904761904756, | |
| "hit_at_1": 0.01, | |
| "hit_at_5": 0.1, | |
| "ndcg_at_10": 0.06569076601728688, | |
| "p50_retrieval_latency_ms": 7.616497576236725, | |
| "p95_retrieval_latency_ms": 8.379131555557251, | |
| "avg_retrieval_latency_ms": 7.737708697095513, | |
| "index_build_seconds": 0.032947403378784657, | |
| "est_cost_per_query_usd": 0.0, | |
| "generation_acc": null, | |
| "p50_generation_latency_ms": null, | |
| "avg_generation_latency_ms": null, | |
| "generation_tokens_per_sec": null, | |
| "est_generation_cost_per_query_usd": null, | |
| "paraphrase_drop": 0.015111111111111103, | |
| "shift_drop": 0.048261904761904756, | |
| "n_queries": 100 | |
| }, | |
| { | |
| "candidate_id": "ministral-14b-reasoning::hybrid::none", | |
| "benchmark": "dataset-logistics-qa", | |
| "model": "ministral-14b-reasoning", | |
| "retriever": "hybrid", | |
| "reranker": "none", | |
| "is_neon": true, | |
| "error": null, | |
| "mrr": 1.0, | |
| "hit_at_1": 1.0, | |
| "hit_at_5": 1.0, | |
| "ndcg_at_10": 1.0, | |
| "p50_retrieval_latency_ms": 6.865703500807285, | |
| "p95_retrieval_latency_ms": 7.6750898733735085, | |
| "avg_retrieval_latency_ms": 7.112867152318358, | |
| "index_build_seconds": 0.009198973886668682, | |
| "est_cost_per_query_usd": 0.0, | |
| "generation_acc": null, | |
| "p50_generation_latency_ms": null, | |
| "avg_generation_latency_ms": null, | |
| "generation_tokens_per_sec": null, | |
| "est_generation_cost_per_query_usd": null, | |
| "paraphrase_drop": 0.0, | |
| "shift_drop": 0.0, | |
| "n_queries": 100 | |
| }, | |
| { | |
| "candidate_id": "ministral-14b-reasoning::hybrid::none", | |
| "benchmark": "dataset-landuse-wa-qa", | |
| "model": "ministral-14b-reasoning", | |
| "retriever": "hybrid", | |
| "reranker": "none", | |
| "is_neon": true, | |
| "error": null, | |
| "mrr": 1.0, | |
| "hit_at_1": 1.0, | |
| "hit_at_5": 1.0, | |
| "ndcg_at_10": 1.0, | |
| "p50_retrieval_latency_ms": 7.199288345873356, | |
| "p95_retrieval_latency_ms": 7.892884314060211, | |
| "avg_retrieval_latency_ms": 7.272108998149633, | |
| "index_build_seconds": 0.007608828134834766, | |
| "est_cost_per_query_usd": 0.0, | |
| "generation_acc": null, | |
| "p50_generation_latency_ms": null, | |
| "avg_generation_latency_ms": null, | |
| "generation_tokens_per_sec": null, | |
| "est_generation_cost_per_query_usd": null, | |
| "paraphrase_drop": 0.0, | |
| "shift_drop": 1.0, | |
| "n_queries": 100 | |
| }, | |
| { | |
| "candidate_id": "ministral-14b-reasoning::hybrid::none", | |
| "benchmark": "dataset-shakespeare-qa", | |
| "model": "ministral-14b-reasoning", | |
| "retriever": "hybrid", | |
| "reranker": "none", | |
| "is_neon": true, | |
| "error": null, | |
| "mrr": 1.0, | |
| "hit_at_1": 1.0, | |
| "hit_at_5": 1.0, | |
| "ndcg_at_10": 1.0, | |
| "p50_retrieval_latency_ms": 7.541215978562832, | |
| "p95_retrieval_latency_ms": 7.804302498698235, | |
| "avg_retrieval_latency_ms": 7.6186764147132635, | |
| "index_build_seconds": 0.05469726677983999, | |
| "est_cost_per_query_usd": 0.0, | |
| "generation_acc": null, | |
| "p50_generation_latency_ms": null, | |
| "avg_generation_latency_ms": null, | |
| "generation_tokens_per_sec": null, | |
| "est_generation_cost_per_query_usd": null, | |
| "paraphrase_drop": 0.0, | |
| "shift_drop": 0.0, | |
| "n_queries": 100 | |
| }, | |
| { | |
| "candidate_id": "ministral-14b-reasoning::hybrid::none", | |
| "benchmark": "dataset-futurewise-qa", | |
| "model": "ministral-14b-reasoning", | |
| "retriever": "hybrid", | |
| "reranker": "none", | |
| "is_neon": true, | |
| "error": null, | |
| "mrr": 0.09780952380952382, | |
| "hit_at_1": 0.03, | |
| "hit_at_5": 0.19, | |
| "ndcg_at_10": 0.13218233672282742, | |
| "p50_retrieval_latency_ms": 7.985537871718407, | |
| "p95_retrieval_latency_ms": 8.448821492493153, | |
| "avg_retrieval_latency_ms": 8.036215221509337, | |
| "index_build_seconds": 0.08024525735527277, | |
| "est_cost_per_query_usd": 0.0, | |
| "generation_acc": null, | |
| "p50_generation_latency_ms": null, | |
| "avg_generation_latency_ms": null, | |
| "generation_tokens_per_sec": null, | |
| "est_generation_cost_per_query_usd": null, | |
| "paraphrase_drop": 0.03988095238095239, | |
| "shift_drop": 0.04168707482993198, | |
| "n_queries": 100 | |
| }, | |
| { | |
| "candidate_id": "ministral-14b-reasoning::hybrid::none", | |
| "benchmark": "dataset-belebele-qa", | |
| "model": "ministral-14b-reasoning", | |
| "retriever": "hybrid", | |
| "reranker": "none", | |
| "is_neon": true, | |
| "error": null, | |
| "mrr": 0.9678571428571429, | |
| "hit_at_1": 0.95, | |
| "hit_at_5": 0.9928571428571429, | |
| "ndcg_at_10": 0.974179086645947, | |
| "p50_retrieval_latency_ms": 7.509047165513039, | |
| "p95_retrieval_latency_ms": 7.911396212875843, | |
| "avg_retrieval_latency_ms": 7.4360380082258155, | |
| "index_build_seconds": 0.0037398524582386017, | |
| "est_cost_per_query_usd": 0.0, | |
| "generation_acc": null, | |
| "p50_generation_latency_ms": null, | |
| "avg_generation_latency_ms": null, | |
| "generation_tokens_per_sec": null, | |
| "est_generation_cost_per_query_usd": null, | |
| "paraphrase_drop": 0.06440476190476185, | |
| "shift_drop": 0.0035714285714285587, | |
| "n_queries": 140 | |
| }, | |
| { | |
| "candidate_id": "magistral-small::hybrid::none", | |
| "benchmark": "dataset-neonai-qa", | |
| "model": "magistral-small", | |
| "retriever": "hybrid", | |
| "reranker": "none", | |
| "is_neon": true, | |
| "error": null, | |
| "mrr": 0.5988452380952382, | |
| "hit_at_1": 0.54, | |
| "hit_at_5": 0.68, | |
| "ndcg_at_10": 0.6323249003639557, | |
| "p50_retrieval_latency_ms": 6.954291835427284, | |
| "p95_retrieval_latency_ms": 7.816776633262634, | |
| "avg_retrieval_latency_ms": 7.214257745072246, | |
| "index_build_seconds": 0.004052711650729179, | |
| "est_cost_per_query_usd": 0.0, | |
| "generation_acc": null, | |
| "p50_generation_latency_ms": null, | |
| "avg_generation_latency_ms": null, | |
| "generation_tokens_per_sec": null, | |
| "est_generation_cost_per_query_usd": null, | |
| "paraphrase_drop": 0.07990079365079361, | |
| "shift_drop": 0.3071190476190476, | |
| "n_queries": 100 | |
| }, | |
| { | |
| "candidate_id": "magistral-small::hybrid::none", | |
| "benchmark": "dataset-nucleotidings-qa", | |
| "model": "magistral-small", | |
| "retriever": "hybrid", | |
| "reranker": "none", | |
| "is_neon": true, | |
| "error": null, | |
| "mrr": 0.048261904761904756, | |
| "hit_at_1": 0.01, | |
| "hit_at_5": 0.1, | |
| "ndcg_at_10": 0.06569076601728688, | |
| "p50_retrieval_latency_ms": 7.779561914503574, | |
| "p95_retrieval_latency_ms": 8.675494231283665, | |
| "avg_retrieval_latency_ms": 7.911766804754734, | |
| "index_build_seconds": 0.0325782485306263, | |
| "est_cost_per_query_usd": 0.0, | |
| "generation_acc": null, | |
| "p50_generation_latency_ms": null, | |
| "avg_generation_latency_ms": null, | |
| "generation_tokens_per_sec": null, | |
| "est_generation_cost_per_query_usd": null, | |
| "paraphrase_drop": 0.015111111111111103, | |
| "shift_drop": 0.048261904761904756, | |
| "n_queries": 100 | |
| }, | |
| { | |
| "candidate_id": "magistral-small::hybrid::none", | |
| "benchmark": "dataset-logistics-qa", | |
| "model": "magistral-small", | |
| "retriever": "hybrid", | |
| "reranker": "none", | |
| "is_neon": true, | |
| "error": null, | |
| "mrr": 1.0, | |
| "hit_at_1": 1.0, | |
| "hit_at_5": 1.0, | |
| "ndcg_at_10": 1.0, | |
| "p50_retrieval_latency_ms": 6.913617253303528, | |
| "p95_retrieval_latency_ms": 7.607226260006428, | |
| "avg_retrieval_latency_ms": 7.0380013063549995, | |
| "index_build_seconds": 0.009966623038053513, | |
| "est_cost_per_query_usd": 0.0, | |
| "generation_acc": null, | |
| "p50_generation_latency_ms": null, | |
| "avg_generation_latency_ms": null, | |
| "generation_tokens_per_sec": null, | |
| "est_generation_cost_per_query_usd": null, | |
| "paraphrase_drop": 0.0, | |
| "shift_drop": 0.0, | |
| "n_queries": 100 | |
| }, | |
| { | |
| "candidate_id": "magistral-small::hybrid::none", | |
| "benchmark": "dataset-landuse-wa-qa", | |
| "model": "magistral-small", | |
| "retriever": "hybrid", | |
| "reranker": "none", | |
| "is_neon": true, | |
| "error": null, | |
| "mrr": 1.0, | |
| "hit_at_1": 1.0, | |
| "hit_at_5": 1.0, | |
| "ndcg_at_10": 1.0, | |
| "p50_retrieval_latency_ms": 7.033617235720158, | |
| "p95_retrieval_latency_ms": 7.600724697113037, | |
| "avg_retrieval_latency_ms": 7.124365046620369, | |
| "index_build_seconds": 0.007180897518992424, | |
| "est_cost_per_query_usd": 0.0, | |
| "generation_acc": null, | |
| "p50_generation_latency_ms": null, | |
| "avg_generation_latency_ms": null, | |
| "generation_tokens_per_sec": null, | |
| "est_generation_cost_per_query_usd": null, | |
| "paraphrase_drop": 0.0, | |
| "shift_drop": 1.0, | |
| "n_queries": 100 | |
| }, | |
| { | |
| "candidate_id": "magistral-small::hybrid::none", | |
| "benchmark": "dataset-shakespeare-qa", | |
| "model": "magistral-small", | |
| "retriever": "hybrid", | |
| "reranker": "none", | |
| "is_neon": true, | |
| "error": null, | |
| "mrr": 1.0, | |
| "hit_at_1": 1.0, | |
| "hit_at_5": 1.0, | |
| "ndcg_at_10": 1.0, | |
| "p50_retrieval_latency_ms": 7.53379613161087, | |
| "p95_retrieval_latency_ms": 7.8609399497509, | |
| "avg_retrieval_latency_ms": 7.591790938749909, | |
| "index_build_seconds": 0.05281056836247444, | |
| "est_cost_per_query_usd": 0.0, | |
| "generation_acc": null, | |
| "p50_generation_latency_ms": null, | |
| "avg_generation_latency_ms": null, | |
| "generation_tokens_per_sec": null, | |
| "est_generation_cost_per_query_usd": null, | |
| "paraphrase_drop": 0.0, | |
| "shift_drop": 0.0, | |
| "n_queries": 100 | |
| }, | |
| { | |
| "candidate_id": "magistral-small::hybrid::none", | |
| "benchmark": "dataset-futurewise-qa", | |
| "model": "magistral-small", | |
| "retriever": "hybrid", | |
| "reranker": "none", | |
| "is_neon": true, | |
| "error": null, | |
| "mrr": 0.09780952380952382, | |
| "hit_at_1": 0.03, | |
| "hit_at_5": 0.19, | |
| "ndcg_at_10": 0.13218233672282742, | |
| "p50_retrieval_latency_ms": 8.058656938374043, | |
| "p95_retrieval_latency_ms": 8.640003390610218, | |
| "avg_retrieval_latency_ms": 8.11074884608388, | |
| "index_build_seconds": 0.08653946872800589, | |
| "est_cost_per_query_usd": 0.0, | |
| "generation_acc": null, | |
| "p50_generation_latency_ms": null, | |
| "avg_generation_latency_ms": null, | |
| "generation_tokens_per_sec": null, | |
| "est_generation_cost_per_query_usd": null, | |
| "paraphrase_drop": 0.03988095238095239, | |
| "shift_drop": 0.04168707482993198, | |
| "n_queries": 100 | |
| }, | |
| { | |
| "candidate_id": "magistral-small::hybrid::none", | |
| "benchmark": "dataset-belebele-qa", | |
| "model": "magistral-small", | |
| "retriever": "hybrid", | |
| "reranker": "none", | |
| "is_neon": true, | |
| "error": null, | |
| "mrr": 0.9678571428571429, | |
| "hit_at_1": 0.95, | |
| "hit_at_5": 0.9928571428571429, | |
| "ndcg_at_10": 0.974179086645947, | |
| "p50_retrieval_latency_ms": 7.476835511624813, | |
| "p95_retrieval_latency_ms": 8.002723567187786, | |
| "avg_retrieval_latency_ms": 7.445732723655445, | |
| "index_build_seconds": 0.0043265325948596, | |
| "est_cost_per_query_usd": 0.0, | |
| "generation_acc": null, | |
| "p50_generation_latency_ms": null, | |
| "avg_generation_latency_ms": null, | |
| "generation_tokens_per_sec": null, | |
| "est_generation_cost_per_query_usd": null, | |
| "paraphrase_drop": 0.06440476190476185, | |
| "shift_drop": 0.0035714285714285587, | |
| "n_queries": 140 | |
| }, | |
| { | |
| "candidate_id": "devstral-small-24b::hybrid::none", | |
| "benchmark": "dataset-neonai-qa", | |
| "model": "devstral-small-24b", | |
| "retriever": "hybrid", | |
| "reranker": "none", | |
| "is_neon": true, | |
| "error": null, | |
| "mrr": 0.5988452380952382, | |
| "hit_at_1": 0.54, | |
| "hit_at_5": 0.68, | |
| "ndcg_at_10": 0.6323249003639557, | |
| "p50_retrieval_latency_ms": 6.986231543123722, | |
| "p95_retrieval_latency_ms": 7.795398123562336, | |
| "avg_retrieval_latency_ms": 7.228139033541083, | |
| "index_build_seconds": 0.003448634408414364, | |
| "est_cost_per_query_usd": 0.0, | |
| "generation_acc": null, | |
| "p50_generation_latency_ms": null, | |
| "avg_generation_latency_ms": null, | |
| "generation_tokens_per_sec": null, | |
| "est_generation_cost_per_query_usd": null, | |
| "paraphrase_drop": 0.07990079365079361, | |
| "shift_drop": 0.3071190476190476, | |
| "n_queries": 100 | |
| }, | |
| { | |
| "candidate_id": "devstral-small-24b::hybrid::none", | |
| "benchmark": "dataset-nucleotidings-qa", | |
| "model": "devstral-small-24b", | |
| "retriever": "hybrid", | |
| "reranker": "none", | |
| "is_neon": true, | |
| "error": null, | |
| "mrr": 0.048261904761904756, | |
| "hit_at_1": 0.01, | |
| "hit_at_5": 0.1, | |
| "ndcg_at_10": 0.06569076601728688, | |
| "p50_retrieval_latency_ms": 7.648967206478119, | |
| "p95_retrieval_latency_ms": 8.57258029282093, | |
| "avg_retrieval_latency_ms": 7.856305995956063, | |
| "index_build_seconds": 0.03500657342374325, | |
| "est_cost_per_query_usd": 0.0, | |
| "generation_acc": null, | |
| "p50_generation_latency_ms": null, | |
| "avg_generation_latency_ms": null, | |
| "generation_tokens_per_sec": null, | |
| "est_generation_cost_per_query_usd": null, | |
| "paraphrase_drop": 0.015111111111111103, | |
| "shift_drop": 0.048261904761904756, | |
| "n_queries": 100 | |
| }, | |
| { | |
| "candidate_id": "devstral-small-24b::hybrid::none", | |
| "benchmark": "dataset-logistics-qa", | |
| "model": "devstral-small-24b", | |
| "retriever": "hybrid", | |
| "reranker": "none", | |
| "is_neon": true, | |
| "error": null, | |
| "mrr": 1.0, | |
| "hit_at_1": 1.0, | |
| "hit_at_5": 1.0, | |
| "ndcg_at_10": 1.0, | |
| "p50_retrieval_latency_ms": 7.002032361924648, | |
| "p95_retrieval_latency_ms": 7.7148256823420525, | |
| "avg_retrieval_latency_ms": 7.0997389033436775, | |
| "index_build_seconds": 0.01002301275730133, | |
| "est_cost_per_query_usd": 0.0, | |
| "generation_acc": null, | |
| "p50_generation_latency_ms": null, | |
| "avg_generation_latency_ms": null, | |
| "generation_tokens_per_sec": null, | |
| "est_generation_cost_per_query_usd": null, | |
| "paraphrase_drop": 0.0, | |
| "shift_drop": 0.0, | |
| "n_queries": 100 | |
| }, | |
| { | |
| "candidate_id": "devstral-small-24b::hybrid::none", | |
| "benchmark": "dataset-landuse-wa-qa", | |
| "model": "devstral-small-24b", | |
| "retriever": "hybrid", | |
| "reranker": "none", | |
| "is_neon": true, | |
| "error": null, | |
| "mrr": 1.0, | |
| "hit_at_1": 1.0, | |
| "hit_at_5": 1.0, | |
| "ndcg_at_10": 1.0, | |
| "p50_retrieval_latency_ms": 6.944989785552025, | |
| "p95_retrieval_latency_ms": 7.596067152917385, | |
| "avg_retrieval_latency_ms": 7.056590598076582, | |
| "index_build_seconds": 0.007299384102225304, | |
| "est_cost_per_query_usd": 0.0, | |
| "generation_acc": null, | |
| "p50_generation_latency_ms": null, | |
| "avg_generation_latency_ms": null, | |
| "generation_tokens_per_sec": null, | |
| "est_generation_cost_per_query_usd": null, | |
| "paraphrase_drop": 0.0, | |
| "shift_drop": 1.0, | |
| "n_queries": 100 | |
| }, | |
| { | |
| "candidate_id": "devstral-small-24b::hybrid::none", | |
| "benchmark": "dataset-shakespeare-qa", | |
| "model": "devstral-small-24b", | |
| "retriever": "hybrid", | |
| "reranker": "none", | |
| "is_neon": true, | |
| "error": null, | |
| "mrr": 1.0, | |
| "hit_at_1": 1.0, | |
| "hit_at_5": 1.0, | |
| "ndcg_at_10": 1.0, | |
| "p50_retrieval_latency_ms": 7.386316545307636, | |
| "p95_retrieval_latency_ms": 7.697527296841145, | |
| "avg_retrieval_latency_ms": 7.466522827744484, | |
| "index_build_seconds": 0.054405439645051956, | |
| "est_cost_per_query_usd": 0.0, | |
| "generation_acc": null, | |
| "p50_generation_latency_ms": null, | |
| "avg_generation_latency_ms": null, | |
| "generation_tokens_per_sec": null, | |
| "est_generation_cost_per_query_usd": null, | |
| "paraphrase_drop": 0.0, | |
| "shift_drop": 0.0, | |
| "n_queries": 100 | |
| }, | |
| { | |
| "candidate_id": "devstral-small-24b::hybrid::none", | |
| "benchmark": "dataset-futurewise-qa", | |
| "model": "devstral-small-24b", | |
| "retriever": "hybrid", | |
| "reranker": "none", | |
| "is_neon": true, | |
| "error": null, | |
| "mrr": 0.09780952380952382, | |
| "hit_at_1": 0.03, | |
| "hit_at_5": 0.19, | |
| "ndcg_at_10": 0.13218233672282742, | |
| "p50_retrieval_latency_ms": 7.762560620903969, | |
| "p95_retrieval_latency_ms": 8.184201084077358, | |
| "avg_retrieval_latency_ms": 7.790521765127778, | |
| "index_build_seconds": 0.08099719695746899, | |
| "est_cost_per_query_usd": 0.0, | |
| "generation_acc": null, | |
| "p50_generation_latency_ms": null, | |
| "avg_generation_latency_ms": null, | |
| "generation_tokens_per_sec": null, | |
| "est_generation_cost_per_query_usd": null, | |
| "paraphrase_drop": 0.03988095238095239, | |
| "shift_drop": 0.04168707482993198, | |
| "n_queries": 100 | |
| }, | |
| { | |
| "candidate_id": "devstral-small-24b::hybrid::none", | |
| "benchmark": "dataset-belebele-qa", | |
| "model": "devstral-small-24b", | |
| "retriever": "hybrid", | |
| "reranker": "none", | |
| "is_neon": true, | |
| "error": null, | |
| "mrr": 0.9678571428571429, | |
| "hit_at_1": 0.95, | |
| "hit_at_5": 0.9928571428571429, | |
| "ndcg_at_10": 0.974179086645947, | |
| "p50_retrieval_latency_ms": 7.34444335103035, | |
| "p95_retrieval_latency_ms": 7.850646041333675, | |
| "avg_retrieval_latency_ms": 7.294278585218957, | |
| "index_build_seconds": 0.0038021234795451164, | |
| "est_cost_per_query_usd": 0.0, | |
| "generation_acc": null, | |
| "p50_generation_latency_ms": null, | |
| "avg_generation_latency_ms": null, | |
| "generation_tokens_per_sec": null, | |
| "est_generation_cost_per_query_usd": null, | |
| "paraphrase_drop": 0.06440476190476185, | |
| "shift_drop": 0.0035714285714285587, | |
| "n_queries": 140 | |
| }, | |
| { | |
| "candidate_id": "mistral-small-3.2-24b::hybrid::none", | |
| "benchmark": "dataset-neonai-qa", | |
| "model": "mistral-small-3.2-24b", | |
| "retriever": "hybrid", | |
| "reranker": "none", | |
| "is_neon": true, | |
| "error": null, | |
| "mrr": 0.5988452380952382, | |
| "hit_at_1": 0.54, | |
| "hit_at_5": 0.68, | |
| "ndcg_at_10": 0.6323249003639557, | |
| "p50_retrieval_latency_ms": 7.042726501822472, | |
| "p95_retrieval_latency_ms": 7.8193424269557, | |
| "avg_retrieval_latency_ms": 7.272537685930729, | |
| "index_build_seconds": 0.004021286033093929, | |
| "est_cost_per_query_usd": 0.0, | |
| "generation_acc": null, | |
| "p50_generation_latency_ms": null, | |
| "avg_generation_latency_ms": null, | |
| "generation_tokens_per_sec": null, | |
| "est_generation_cost_per_query_usd": null, | |
| "paraphrase_drop": 0.07990079365079361, | |
| "shift_drop": 0.3071190476190476, | |
| "n_queries": 100 | |
| }, | |
| { | |
| "candidate_id": "mistral-small-3.2-24b::hybrid::none", | |
| "benchmark": "dataset-nucleotidings-qa", | |
| "model": "mistral-small-3.2-24b", | |
| "retriever": "hybrid", | |
| "reranker": "none", | |
| "is_neon": true, | |
| "error": null, | |
| "mrr": 0.048261904761904756, | |
| "hit_at_1": 0.01, | |
| "hit_at_5": 0.1, | |
| "ndcg_at_10": 0.06569076601728688, | |
| "p50_retrieval_latency_ms": 7.691754028201103, | |
| "p95_retrieval_latency_ms": 8.569280616939068, | |
| "avg_retrieval_latency_ms": 7.879103012382984, | |
| "index_build_seconds": 0.033108511939644814, | |
| "est_cost_per_query_usd": 0.0, | |
| "generation_acc": null, | |
| "p50_generation_latency_ms": null, | |
| "avg_generation_latency_ms": null, | |
| "generation_tokens_per_sec": null, | |
| "est_generation_cost_per_query_usd": null, | |
| "paraphrase_drop": 0.015111111111111103, | |
| "shift_drop": 0.048261904761904756, | |
| "n_queries": 100 | |
| }, | |
| { | |
| "candidate_id": "mistral-small-3.2-24b::hybrid::none", | |
| "benchmark": "dataset-logistics-qa", | |
| "model": "mistral-small-3.2-24b", | |
| "retriever": "hybrid", | |
| "reranker": "none", | |
| "is_neon": true, | |
| "error": null, | |
| "mrr": 1.0, | |
| "hit_at_1": 1.0, | |
| "hit_at_5": 1.0, | |
| "ndcg_at_10": 1.0, | |
| "p50_retrieval_latency_ms": 6.975045427680016, | |
| "p95_retrieval_latency_ms": 7.641519419848919, | |
| "avg_retrieval_latency_ms": 7.097518444061279, | |
| "index_build_seconds": 0.010510171763598919, | |
| "est_cost_per_query_usd": 0.0, | |
| "generation_acc": null, | |
| "p50_generation_latency_ms": null, | |
| "avg_generation_latency_ms": null, | |
| "generation_tokens_per_sec": null, | |
| "est_generation_cost_per_query_usd": null, | |
| "paraphrase_drop": 0.0, | |
| "shift_drop": 0.0, | |
| "n_queries": 100 | |
| }, | |
| { | |
| "candidate_id": "mistral-small-3.2-24b::hybrid::none", | |
| "benchmark": "dataset-landuse-wa-qa", | |
| "model": "mistral-small-3.2-24b", | |
| "retriever": "hybrid", | |
| "reranker": "none", | |
| "is_neon": true, | |
| "error": null, | |
| "mrr": 1.0, | |
| "hit_at_1": 1.0, | |
| "hit_at_5": 1.0, | |
| "ndcg_at_10": 1.0, | |
| "p50_retrieval_latency_ms": 7.0530083030462265, | |
| "p95_retrieval_latency_ms": 7.611000910401344, | |
| "avg_retrieval_latency_ms": 7.2303250059485435, | |
| "index_build_seconds": 0.008502699434757233, | |
| "est_cost_per_query_usd": 0.0, | |
| "generation_acc": null, | |
| "p50_generation_latency_ms": null, | |
| "avg_generation_latency_ms": null, | |
| "generation_tokens_per_sec": null, | |
| "est_generation_cost_per_query_usd": null, | |
| "paraphrase_drop": 0.0, | |
| "shift_drop": 1.0, | |
| "n_queries": 100 | |
| }, | |
| { | |
| "candidate_id": "mistral-small-3.2-24b::hybrid::none", | |
| "benchmark": "dataset-shakespeare-qa", | |
| "model": "mistral-small-3.2-24b", | |
| "retriever": "hybrid", | |
| "reranker": "none", | |
| "is_neon": true, | |
| "error": null, | |
| "mrr": 1.0, | |
| "hit_at_1": 1.0, | |
| "hit_at_5": 1.0, | |
| "ndcg_at_10": 1.0, | |
| "p50_retrieval_latency_ms": 7.434654980897903, | |
| "p95_retrieval_latency_ms": 7.705735974013805, | |
| "avg_retrieval_latency_ms": 7.511973651126027, | |
| "index_build_seconds": 0.053165786899626255, | |
| "est_cost_per_query_usd": 0.0, | |
| "generation_acc": null, | |
| "p50_generation_latency_ms": null, | |
| "avg_generation_latency_ms": null, | |
| "generation_tokens_per_sec": null, | |
| "est_generation_cost_per_query_usd": null, | |
| "paraphrase_drop": 0.0, | |
| "shift_drop": 0.0, | |
| "n_queries": 100 | |
| }, | |
| { | |
| "candidate_id": "mistral-small-3.2-24b::hybrid::none", | |
| "benchmark": "dataset-futurewise-qa", | |
| "model": "mistral-small-3.2-24b", | |
| "retriever": "hybrid", | |
| "reranker": "none", | |
| "is_neon": true, | |
| "error": null, | |
| "mrr": 0.09780952380952382, | |
| "hit_at_1": 0.03, | |
| "hit_at_5": 0.19, | |
| "ndcg_at_10": 0.13218233672282742, | |
| "p50_retrieval_latency_ms": 8.097118698060513, | |
| "p95_retrieval_latency_ms": 8.389534428715706, | |
| "avg_retrieval_latency_ms": 8.127077231183648, | |
| "index_build_seconds": 0.08655050490051508, | |
| "est_cost_per_query_usd": 0.0, | |
| "generation_acc": null, | |
| "p50_generation_latency_ms": null, | |
| "avg_generation_latency_ms": null, | |
| "generation_tokens_per_sec": null, | |
| "est_generation_cost_per_query_usd": null, | |
| "paraphrase_drop": 0.03988095238095239, | |
| "shift_drop": 0.04168707482993198, | |
| "n_queries": 100 | |
| }, | |
| { | |
| "candidate_id": "mistral-small-3.2-24b::hybrid::none", | |
| "benchmark": "dataset-belebele-qa", | |
| "model": "mistral-small-3.2-24b", | |
| "retriever": "hybrid", | |
| "reranker": "none", | |
| "is_neon": true, | |
| "error": null, | |
| "mrr": 0.9678571428571429, | |
| "hit_at_1": 0.95, | |
| "hit_at_5": 0.9928571428571429, | |
| "ndcg_at_10": 0.974179086645947, | |
| "p50_retrieval_latency_ms": 7.46503658592701, | |
| "p95_retrieval_latency_ms": 7.775686681270599, | |
| "avg_retrieval_latency_ms": 7.389138266444206, | |
| "index_build_seconds": 0.004063928499817848, | |
| "est_cost_per_query_usd": 0.0, | |
| "generation_acc": null, | |
| "p50_generation_latency_ms": null, | |
| "avg_generation_latency_ms": null, | |
| "generation_tokens_per_sec": null, | |
| "est_generation_cost_per_query_usd": null, | |
| "paraphrase_drop": 0.06440476190476185, | |
| "shift_drop": 0.0035714285714285587, | |
| "n_queries": 140 | |
| }, | |
| { | |
| "candidate_id": "granite-4-30b::hybrid::none", | |
| "benchmark": "dataset-neonai-qa", | |
| "model": "granite-4-30b", | |
| "retriever": "hybrid", | |
| "reranker": "none", | |
| "is_neon": true, | |
| "error": null, | |
| "mrr": 0.5988452380952382, | |
| "hit_at_1": 0.54, | |
| "hit_at_5": 0.68, | |
| "ndcg_at_10": 0.6323249003639557, | |
| "p50_retrieval_latency_ms": 7.0448704063892365, | |
| "p95_retrieval_latency_ms": 7.770040072500706, | |
| "avg_retrieval_latency_ms": 7.286299178376794, | |
| "index_build_seconds": 0.0033295685425400734, | |
| "est_cost_per_query_usd": 0.0, | |
| "generation_acc": null, | |
| "p50_generation_latency_ms": null, | |
| "avg_generation_latency_ms": null, | |
| "generation_tokens_per_sec": null, | |
| "est_generation_cost_per_query_usd": null, | |
| "paraphrase_drop": 0.07990079365079361, | |
| "shift_drop": 0.3071190476190476, | |
| "n_queries": 100 | |
| }, | |
| { | |
| "candidate_id": "granite-4-30b::hybrid::none", | |
| "benchmark": "dataset-nucleotidings-qa", | |
| "model": "granite-4-30b", | |
| "retriever": "hybrid", | |
| "reranker": "none", | |
| "is_neon": true, | |
| "error": null, | |
| "mrr": 0.048261904761904756, | |
| "hit_at_1": 0.01, | |
| "hit_at_5": 0.1, | |
| "ndcg_at_10": 0.06569076601728688, | |
| "p50_retrieval_latency_ms": 7.591321133077145, | |
| "p95_retrieval_latency_ms": 8.210630156099796, | |
| "avg_retrieval_latency_ms": 7.772924480959773, | |
| "index_build_seconds": 0.03287146147340536, | |
| "est_cost_per_query_usd": 0.0, | |
| "generation_acc": null, | |
| "p50_generation_latency_ms": null, | |
| "avg_generation_latency_ms": null, | |
| "generation_tokens_per_sec": null, | |
| "est_generation_cost_per_query_usd": null, | |
| "paraphrase_drop": 0.015111111111111103, | |
| "shift_drop": 0.048261904761904756, | |
| "n_queries": 100 | |
| }, | |
| { | |
| "candidate_id": "granite-4-30b::hybrid::none", | |
| "benchmark": "dataset-logistics-qa", | |
| "model": "granite-4-30b", | |
| "retriever": "hybrid", | |
| "reranker": "none", | |
| "is_neon": true, | |
| "error": null, | |
| "mrr": 1.0, | |
| "hit_at_1": 1.0, | |
| "hit_at_5": 1.0, | |
| "ndcg_at_10": 1.0, | |
| "p50_retrieval_latency_ms": 6.952958181500435, | |
| "p95_retrieval_latency_ms": 7.622065953910351, | |
| "avg_retrieval_latency_ms": 7.116861520335078, | |
| "index_build_seconds": 0.009383691474795341, | |
| "est_cost_per_query_usd": 0.0, | |
| "generation_acc": null, | |
| "p50_generation_latency_ms": null, | |
| "avg_generation_latency_ms": null, | |
| "generation_tokens_per_sec": null, | |
| "est_generation_cost_per_query_usd": null, | |
| "paraphrase_drop": 0.0, | |
| "shift_drop": 0.0, | |
| "n_queries": 100 | |
| }, | |
| { | |
| "candidate_id": "granite-4-30b::hybrid::none", | |
| "benchmark": "dataset-landuse-wa-qa", | |
| "model": "granite-4-30b", | |
| "retriever": "hybrid", | |
| "reranker": "none", | |
| "is_neon": true, | |
| "error": null, | |
| "mrr": 1.0, | |
| "hit_at_1": 1.0, | |
| "hit_at_5": 1.0, | |
| "ndcg_at_10": 1.0, | |
| "p50_retrieval_latency_ms": 7.011697627604008, | |
| "p95_retrieval_latency_ms": 7.574110291898251, | |
| "avg_retrieval_latency_ms": 7.0873454213142395, | |
| "index_build_seconds": 0.008607922121882439, | |
| "est_cost_per_query_usd": 0.0, | |
| "generation_acc": null, | |
| "p50_generation_latency_ms": null, | |
| "avg_generation_latency_ms": null, | |
| "generation_tokens_per_sec": null, | |
| "est_generation_cost_per_query_usd": null, | |
| "paraphrase_drop": 0.0, | |
| "shift_drop": 1.0, | |
| "n_queries": 100 | |
| }, | |
| { | |
| "candidate_id": "granite-4-30b::hybrid::none", | |
| "benchmark": "dataset-shakespeare-qa", | |
| "model": "granite-4-30b", | |
| "retriever": "hybrid", | |
| "reranker": "none", | |
| "is_neon": true, | |
| "error": null, | |
| "mrr": 1.0, | |
| "hit_at_1": 1.0, | |
| "hit_at_5": 1.0, | |
| "ndcg_at_10": 1.0, | |
| "p50_retrieval_latency_ms": 7.527725771069527, | |
| "p95_retrieval_latency_ms": 7.810373790562153, | |
| "avg_retrieval_latency_ms": 7.593083288520575, | |
| "index_build_seconds": 0.052798391319811344, | |
| "est_cost_per_query_usd": 0.0, | |
| "generation_acc": null, | |
| "p50_generation_latency_ms": null, | |
| "avg_generation_latency_ms": null, | |
| "generation_tokens_per_sec": null, | |
| "est_generation_cost_per_query_usd": null, | |
| "paraphrase_drop": 0.0, | |
| "shift_drop": 0.0, | |
| "n_queries": 100 | |
| }, | |
| { | |
| "candidate_id": "granite-4-30b::hybrid::none", | |
| "benchmark": "dataset-futurewise-qa", | |
| "model": "granite-4-30b", | |
| "retriever": "hybrid", | |
| "reranker": "none", | |
| "is_neon": true, | |
| "error": null, | |
| "mrr": 0.09780952380952382, | |
| "hit_at_1": 0.03, | |
| "hit_at_5": 0.19, | |
| "ndcg_at_10": 0.13218233672282742, | |
| "p50_retrieval_latency_ms": 7.9466793686151505, | |
| "p95_retrieval_latency_ms": 8.466493338346481, | |
| "avg_retrieval_latency_ms": 7.993034860119224, | |
| "index_build_seconds": 0.08808730728924274, | |
| "est_cost_per_query_usd": 0.0, | |
| "generation_acc": null, | |
| "p50_generation_latency_ms": null, | |
| "avg_generation_latency_ms": null, | |
| "generation_tokens_per_sec": null, | |
| "est_generation_cost_per_query_usd": null, | |
| "paraphrase_drop": 0.03988095238095239, | |
| "shift_drop": 0.04168707482993198, | |
| "n_queries": 100 | |
| }, | |
| { | |
| "candidate_id": "granite-4-30b::hybrid::none", | |
| "benchmark": "dataset-belebele-qa", | |
| "model": "granite-4-30b", | |
| "retriever": "hybrid", | |
| "reranker": "none", | |
| "is_neon": true, | |
| "error": null, | |
| "mrr": 0.9678571428571429, | |
| "hit_at_1": 0.95, | |
| "hit_at_5": 0.9928571428571429, | |
| "ndcg_at_10": 0.974179086645947, | |
| "p50_retrieval_latency_ms": 7.436822168529034, | |
| "p95_retrieval_latency_ms": 7.707064040005207, | |
| "avg_retrieval_latency_ms": 7.361321929576142, | |
| "index_build_seconds": 0.004397517070174217, | |
| "est_cost_per_query_usd": 0.0, | |
| "generation_acc": null, | |
| "p50_generation_latency_ms": null, | |
| "avg_generation_latency_ms": null, | |
| "generation_tokens_per_sec": null, | |
| "est_generation_cost_per_query_usd": null, | |
| "paraphrase_drop": 0.06440476190476185, | |
| "shift_drop": 0.0035714285714285587, | |
| "n_queries": 140 | |
| } | |
| ], | |
| "profiles": [ | |
| { | |
| "name": "enterprise_news", | |
| "metric": "mrr", | |
| "gates": [ | |
| "p50 ≤ 200 ms", | |
| "$/q ≤ 0.001", | |
| "MRR ≥ 0.85" | |
| ], | |
| "corpus_profile": "news_articles", | |
| "query_mix": { | |
| "extractive": 0.7, | |
| "paraphrased": 0.25, | |
| "multi_hop": 0.05 | |
| }, | |
| "deployment_constraint": "single_gpu_10gb", | |
| "quality_weight": 0.7 | |
| }, | |
| { | |
| "name": "language_following", | |
| "metric": "generation_acc", | |
| "gates": [ | |
| "p50 ≤ 60000 ms", | |
| "gen ≤ 12000 ms", | |
| "$/q ≤ 0.02", | |
| "gen ≥ 0.5" | |
| ], | |
| "corpus_profile": "mixed_html", | |
| "query_mix": { | |
| "extractive": 0.5, | |
| "paraphrased": 0.3, | |
| "multi_hop": 0.2 | |
| }, | |
| "deployment_constraint": "multi_gpu", | |
| "quality_weight": 0.9 | |
| }, | |
| { | |
| "name": "model_efficiency", | |
| "metric": "generation_acc", | |
| "gates": [ | |
| "p50 ≤ 500 ms", | |
| "gen ≤ 8000 ms", | |
| "$/q ≤ 0.01", | |
| "gen ≥ 0.85" | |
| ], | |
| "corpus_profile": "technical_papers", | |
| "query_mix": { | |
| "extractive": 0.4, | |
| "paraphrased": 0.4, | |
| "multi_hop": 0.2 | |
| }, | |
| "deployment_constraint": "multi_gpu", | |
| "quality_weight": 0.65 | |
| }, | |
| { | |
| "name": "multilingual_qa", | |
| "metric": "generation_acc", | |
| "gates": [ | |
| "p50 ≤ 1500 ms", | |
| "gen ≤ 12000 ms", | |
| "$/q ≤ 0.02", | |
| "gen ≥ 0.25" | |
| ], | |
| "corpus_profile": "mixed_html", | |
| "query_mix": { | |
| "extractive": 0.5, | |
| "paraphrased": 0.3, | |
| "multi_hop": 0.2 | |
| }, | |
| "deployment_constraint": "multi_gpu", | |
| "quality_weight": 0.7 | |
| }, | |
| { | |
| "name": "science_qa", | |
| "metric": "mrr", | |
| "gates": [ | |
| "p50 ≤ 1500 ms", | |
| "$/q ≤ 0.005", | |
| "MRR ≥ 0.8" | |
| ], | |
| "corpus_profile": "technical_papers", | |
| "query_mix": { | |
| "extractive": 0.3, | |
| "paraphrased": 0.4, | |
| "multi_hop": 0.3 | |
| }, | |
| "deployment_constraint": "single_gpu_10gb", | |
| "quality_weight": 0.9 | |
| } | |
| ], | |
| "picks": { | |
| "dataset-neonai-qa": { | |
| "enterprise_news": { | |
| "pick": null, | |
| "verdict": "all disqualified" | |
| }, | |
| "language_following": { | |
| "pick": "devstral-123b::hybrid::none", | |
| "model": "devstral-123b", | |
| "retriever": "hybrid", | |
| "reranker": "none", | |
| "mrr": 0.5988452380952382, | |
| "metric": "generation_acc", | |
| "metric_val": 0.92, | |
| "p50": 7.858959957957268, | |
| "verdict": "winner clear" | |
| }, | |
| "model_efficiency": { | |
| "pick": "devstral-123b::hybrid::none", | |
| "model": "devstral-123b", | |
| "retriever": "hybrid", | |
| "reranker": "none", | |
| "mrr": 0.5988452380952382, | |
| "metric": "generation_acc", | |
| "metric_val": 0.92, | |
| "p50": 7.858959957957268, | |
| "verdict": "winner clear" | |
| }, | |
| "multilingual_qa": { | |
| "pick": "devstral-123b::hybrid::none", | |
| "model": "devstral-123b", | |
| "retriever": "hybrid", | |
| "reranker": "none", | |
| "mrr": 0.5988452380952382, | |
| "metric": "generation_acc", | |
| "metric_val": 0.92, | |
| "p50": 7.858959957957268, | |
| "verdict": "winner clear" | |
| }, | |
| "science_qa": { | |
| "pick": null, | |
| "verdict": "all disqualified" | |
| } | |
| }, | |
| "dataset-nucleotidings-qa": { | |
| "enterprise_news": { | |
| "pick": null, | |
| "verdict": "all disqualified" | |
| }, | |
| "language_following": { | |
| "pick": "devstral-123b::hybrid::none", | |
| "model": "devstral-123b", | |
| "retriever": "hybrid", | |
| "reranker": "none", | |
| "mrr": 0.048261904761904756, | |
| "metric": "generation_acc", | |
| "metric_val": 0.66, | |
| "p50": 8.428102359175682, | |
| "verdict": "winner clear" | |
| }, | |
| "model_efficiency": { | |
| "pick": "retrieval-only::bm25-only::none", | |
| "model": "retrieval-only", | |
| "retriever": "bm25-only", | |
| "reranker": "none", | |
| "mrr": 0.03176190476190476, | |
| "metric": "generation_acc", | |
| "metric_val": null, | |
| "p50": 0.28319843113422394, | |
| "verdict": "inconclusive" | |
| }, | |
| "multilingual_qa": { | |
| "pick": "devstral-123b::hybrid::none", | |
| "model": "devstral-123b", | |
| "retriever": "hybrid", | |
| "reranker": "none", | |
| "mrr": 0.048261904761904756, | |
| "metric": "generation_acc", | |
| "metric_val": 0.66, | |
| "p50": 8.428102359175682, | |
| "verdict": "winner clear" | |
| }, | |
| "science_qa": { | |
| "pick": null, | |
| "verdict": "all disqualified" | |
| } | |
| }, | |
| "dataset-logistics-qa": { | |
| "enterprise_news": { | |
| "pick": "magistral-small::hybrid::none", | |
| "model": "magistral-small", | |
| "retriever": "hybrid", | |
| "reranker": "none", | |
| "mrr": 1.0, | |
| "metric": "mrr", | |
| "metric_val": 1.0, | |
| "p50": 6.913617253303528, | |
| "verdict": "inconclusive" | |
| }, | |
| "language_following": { | |
| "pick": "devstral-123b::hybrid::none", | |
| "model": "devstral-123b", | |
| "retriever": "hybrid", | |
| "reranker": "none", | |
| "mrr": 1.0, | |
| "metric": "generation_acc", | |
| "metric_val": 0.71, | |
| "p50": 7.7611226588487625, | |
| "verdict": "inconclusive" | |
| }, | |
| "model_efficiency": { | |
| "pick": "magistral-small::hybrid::none", | |
| "model": "magistral-small", | |
| "retriever": "hybrid", | |
| "reranker": "none", | |
| "mrr": 1.0, | |
| "metric": "generation_acc", | |
| "metric_val": null, | |
| "p50": 6.913617253303528, | |
| "verdict": "inconclusive" | |
| }, | |
| "multilingual_qa": { | |
| "pick": "devstral-123b::hybrid::none", | |
| "model": "devstral-123b", | |
| "retriever": "hybrid", | |
| "reranker": "none", | |
| "mrr": 1.0, | |
| "metric": "generation_acc", | |
| "metric_val": 0.71, | |
| "p50": 7.7611226588487625, | |
| "verdict": "inconclusive" | |
| }, | |
| "science_qa": { | |
| "pick": "magistral-small::hybrid::none", | |
| "model": "magistral-small", | |
| "retriever": "hybrid", | |
| "reranker": "none", | |
| "mrr": 1.0, | |
| "metric": "mrr", | |
| "metric_val": 1.0, | |
| "p50": 6.913617253303528, | |
| "verdict": "inconclusive" | |
| } | |
| }, | |
| "dataset-landuse-wa-qa": { | |
| "enterprise_news": { | |
| "pick": "devstral-small-24b::hybrid::none", | |
| "model": "devstral-small-24b", | |
| "retriever": "hybrid", | |
| "reranker": "none", | |
| "mrr": 1.0, | |
| "metric": "mrr", | |
| "metric_val": 1.0, | |
| "p50": 6.944989785552025, | |
| "verdict": "inconclusive" | |
| }, | |
| "language_following": { | |
| "pick": "devstral-123b::hybrid::none", | |
| "model": "devstral-123b", | |
| "retriever": "hybrid", | |
| "reranker": "none", | |
| "mrr": 1.0, | |
| "metric": "generation_acc", | |
| "metric_val": 0.89, | |
| "p50": 7.857654243707657, | |
| "verdict": "inconclusive" | |
| }, | |
| "model_efficiency": { | |
| "pick": "devstral-123b::hybrid::none", | |
| "model": "devstral-123b", | |
| "retriever": "hybrid", | |
| "reranker": "none", | |
| "mrr": 1.0, | |
| "metric": "generation_acc", | |
| "metric_val": 0.89, | |
| "p50": 7.857654243707657, | |
| "verdict": "inconclusive" | |
| }, | |
| "multilingual_qa": { | |
| "pick": "devstral-123b::hybrid::none", | |
| "model": "devstral-123b", | |
| "retriever": "hybrid", | |
| "reranker": "none", | |
| "mrr": 1.0, | |
| "metric": "generation_acc", | |
| "metric_val": 0.89, | |
| "p50": 7.857654243707657, | |
| "verdict": "inconclusive" | |
| }, | |
| "science_qa": { | |
| "pick": "devstral-small-24b::hybrid::none", | |
| "model": "devstral-small-24b", | |
| "retriever": "hybrid", | |
| "reranker": "none", | |
| "mrr": 1.0, | |
| "metric": "mrr", | |
| "metric_val": 1.0, | |
| "p50": 6.944989785552025, | |
| "verdict": "inconclusive" | |
| } | |
| }, | |
| "dataset-shakespeare-qa": { | |
| "enterprise_news": { | |
| "pick": "devstral-small-24b::hybrid::none", | |
| "model": "devstral-small-24b", | |
| "retriever": "hybrid", | |
| "reranker": "none", | |
| "mrr": 1.0, | |
| "metric": "mrr", | |
| "metric_val": 1.0, | |
| "p50": 7.386316545307636, | |
| "verdict": "inconclusive" | |
| }, | |
| "language_following": { | |
| "pick": "devstral-123b::hybrid::none", | |
| "model": "devstral-123b", | |
| "retriever": "hybrid", | |
| "reranker": "none", | |
| "mrr": 1.0, | |
| "metric": "generation_acc", | |
| "metric_val": 0.71, | |
| "p50": 8.404421620070934, | |
| "verdict": "inconclusive" | |
| }, | |
| "model_efficiency": { | |
| "pick": "devstral-small-24b::hybrid::none", | |
| "model": "devstral-small-24b", | |
| "retriever": "hybrid", | |
| "reranker": "none", | |
| "mrr": 1.0, | |
| "metric": "generation_acc", | |
| "metric_val": null, | |
| "p50": 7.386316545307636, | |
| "verdict": "inconclusive" | |
| }, | |
| "multilingual_qa": { | |
| "pick": "devstral-123b::hybrid::none", | |
| "model": "devstral-123b", | |
| "retriever": "hybrid", | |
| "reranker": "none", | |
| "mrr": 1.0, | |
| "metric": "generation_acc", | |
| "metric_val": 0.71, | |
| "p50": 8.404421620070934, | |
| "verdict": "inconclusive" | |
| }, | |
| "science_qa": { | |
| "pick": "devstral-small-24b::hybrid::none", | |
| "model": "devstral-small-24b", | |
| "retriever": "hybrid", | |
| "reranker": "none", | |
| "mrr": 1.0, | |
| "metric": "mrr", | |
| "metric_val": 1.0, | |
| "p50": 7.386316545307636, | |
| "verdict": "inconclusive" | |
| } | |
| }, | |
| "dataset-futurewise-qa": { | |
| "enterprise_news": { | |
| "pick": null, | |
| "verdict": "all disqualified" | |
| }, | |
| "language_following": { | |
| "pick": "devstral-123b::hybrid::none", | |
| "model": "devstral-123b", | |
| "retriever": "hybrid", | |
| "reranker": "none", | |
| "mrr": 0.09780952380952382, | |
| "metric": "generation_acc", | |
| "metric_val": 0.77, | |
| "p50": 8.942721411585808, | |
| "verdict": "winner clear" | |
| }, | |
| "model_efficiency": { | |
| "pick": "devstral-small-24b::hybrid::none", | |
| "model": "devstral-small-24b", | |
| "retriever": "hybrid", | |
| "reranker": "none", | |
| "mrr": 0.09780952380952382, | |
| "metric": "generation_acc", | |
| "metric_val": null, | |
| "p50": 7.762560620903969, | |
| "verdict": "inconclusive" | |
| }, | |
| "multilingual_qa": { | |
| "pick": "devstral-123b::hybrid::none", | |
| "model": "devstral-123b", | |
| "retriever": "hybrid", | |
| "reranker": "none", | |
| "mrr": 0.09780952380952382, | |
| "metric": "generation_acc", | |
| "metric_val": 0.77, | |
| "p50": 8.942721411585808, | |
| "verdict": "winner clear" | |
| }, | |
| "science_qa": { | |
| "pick": null, | |
| "verdict": "all disqualified" | |
| } | |
| }, | |
| "dataset-belebele-qa": { | |
| "enterprise_news": { | |
| "pick": "devstral-small-24b::hybrid::none", | |
| "model": "devstral-small-24b", | |
| "retriever": "hybrid", | |
| "reranker": "none", | |
| "mrr": 0.9678571428571429, | |
| "metric": "mrr", | |
| "metric_val": 0.95, | |
| "p50": 7.34444335103035, | |
| "verdict": "inconclusive" | |
| }, | |
| "language_following": { | |
| "pick": "devstral-123b::hybrid::none", | |
| "model": "devstral-123b", | |
| "retriever": "hybrid", | |
| "reranker": "none", | |
| "mrr": 0.9678571428571429, | |
| "metric": "generation_acc", | |
| "metric_val": 0.9, | |
| "p50": 8.227682672441006, | |
| "verdict": "inconclusive" | |
| }, | |
| "model_efficiency": { | |
| "pick": "devstral-123b::hybrid::none", | |
| "model": "devstral-123b", | |
| "retriever": "hybrid", | |
| "reranker": "none", | |
| "mrr": 0.9678571428571429, | |
| "metric": "generation_acc", | |
| "metric_val": 0.9, | |
| "p50": 8.227682672441006, | |
| "verdict": "inconclusive" | |
| }, | |
| "multilingual_qa": { | |
| "pick": "devstral-123b::hybrid::none", | |
| "model": "devstral-123b", | |
| "retriever": "hybrid", | |
| "reranker": "none", | |
| "mrr": 0.9678571428571429, | |
| "metric": "generation_acc", | |
| "metric_val": 0.9, | |
| "p50": 8.227682672441006, | |
| "verdict": "inconclusive" | |
| }, | |
| "science_qa": { | |
| "pick": "devstral-small-24b::hybrid::none", | |
| "model": "devstral-small-24b", | |
| "retriever": "hybrid", | |
| "reranker": "none", | |
| "mrr": 0.9678571428571429, | |
| "metric": "mrr", | |
| "metric_val": 0.95, | |
| "p50": 7.34444335103035, | |
| "verdict": "inconclusive" | |
| } | |
| } | |
| }, | |
| "bench_tables": { | |
| "dataset-neonai-qa": { | |
| "order": [ | |
| "retrieval-only::hybrid::none", | |
| "retrieval-only::hybrid-web::none", | |
| "devstral-123b::hybrid::none", | |
| "ministral-14b-reasoning::hybrid::none", | |
| "magistral-small::hybrid::none", | |
| "devstral-small-24b::hybrid::none", | |
| "mistral-small-3.2-24b::hybrid::none", | |
| "granite-4-30b::hybrid::none", | |
| "retrieval-only::hybrid-triple::none", | |
| "retrieval-only::hybrid-bge-large::none", | |
| "retrieval-only::hybrid-bge-large::minilm", | |
| "retrieval-only::bm25-only::minilm", | |
| "retrieval-only::hybrid::minilm", | |
| "retrieval-only::hybrid-web::minilm", | |
| "retrieval-only::hybrid-triple::minilm", | |
| "retrieval-only::splade-only::minilm", | |
| "retrieval-only::embedding-bge-large::minilm", | |
| "retrieval-only::embedding-bge-large::none", | |
| "retrieval-only::embedding-bge-m3::minilm", | |
| "retrieval-only::embedding-bge-m3::none", | |
| "retrieval-only::hybrid::mxbai", | |
| "retrieval-only::hybrid-web::mxbai", | |
| "retrieval-only::hybrid-bge-large::mxbai", | |
| "retrieval-only::hybrid-triple::mxbai", | |
| "retrieval-only::bm25-only::none", | |
| "retrieval-only::embedding-bge-large::mxbai", | |
| "retrieval-only::splade-only::none", | |
| "retrieval-only::embedding-bge-m3::mxbai", | |
| "retrieval-only::bm25-only::mxbai", | |
| "retrieval-only::splade-only::mxbai" | |
| ], | |
| "pareto": [ | |
| "devstral-123b::hybrid::none", | |
| "devstral-small-24b::hybrid::none", | |
| "granite-4-30b::hybrid::none", | |
| "magistral-small::hybrid::none", | |
| "retrieval-only::bm25-only::none", | |
| "retrieval-only::hybrid-bge-large::none", | |
| "retrieval-only::hybrid::none", | |
| "retrieval-only::splade-only::minilm", | |
| "retrieval-only::splade-only::mxbai", | |
| "retrieval-only::splade-only::none" | |
| ], | |
| "dq": { | |
| "retrieval-only::hybrid::none": "accuracy", | |
| "retrieval-only::hybrid-web::none": "latency", | |
| "devstral-123b::hybrid::none": "cost", | |
| "ministral-14b-reasoning::hybrid::none": "accuracy", | |
| "magistral-small::hybrid::none": "accuracy", | |
| "devstral-small-24b::hybrid::none": "accuracy", | |
| "mistral-small-3.2-24b::hybrid::none": "accuracy", | |
| "granite-4-30b::hybrid::none": "accuracy", | |
| "retrieval-only::hybrid-triple::none": "accuracy", | |
| "retrieval-only::hybrid-bge-large::none": "accuracy", | |
| "retrieval-only::hybrid-bge-large::minilm": "accuracy", | |
| "retrieval-only::bm25-only::minilm": "accuracy", | |
| "retrieval-only::hybrid::minilm": "accuracy", | |
| "retrieval-only::hybrid-web::minilm": "latency", | |
| "retrieval-only::hybrid-triple::minilm": "accuracy", | |
| "retrieval-only::splade-only::minilm": "accuracy", | |
| "retrieval-only::embedding-bge-large::minilm": "accuracy", | |
| "retrieval-only::embedding-bge-large::none": "accuracy", | |
| "retrieval-only::embedding-bge-m3::minilm": "accuracy", | |
| "retrieval-only::embedding-bge-m3::none": "accuracy", | |
| "retrieval-only::hybrid::mxbai": "latency", | |
| "retrieval-only::hybrid-web::mxbai": "latency", | |
| "retrieval-only::hybrid-bge-large::mxbai": "latency", | |
| "retrieval-only::hybrid-triple::mxbai": "latency", | |
| "retrieval-only::bm25-only::none": "accuracy", | |
| "retrieval-only::embedding-bge-large::mxbai": "latency", | |
| "retrieval-only::splade-only::none": "accuracy", | |
| "retrieval-only::embedding-bge-m3::mxbai": "latency", | |
| "retrieval-only::bm25-only::mxbai": "latency", | |
| "retrieval-only::splade-only::mxbai": "latency" | |
| } | |
| }, | |
| "dataset-nucleotidings-qa": { | |
| "order": [ | |
| "retrieval-only::hybrid-bge-large::minilm", | |
| "retrieval-only::hybrid-triple::minilm", | |
| "retrieval-only::hybrid::minilm", | |
| "retrieval-only::hybrid-web::minilm", | |
| "retrieval-only::hybrid-bge-large::mxbai", | |
| "retrieval-only::hybrid-triple::none", | |
| "retrieval-only::splade-only::minilm", | |
| "retrieval-only::embedding-bge-m3::minilm", | |
| "retrieval-only::hybrid::mxbai", | |
| "retrieval-only::hybrid-web::mxbai", | |
| "retrieval-only::hybrid-triple::mxbai", | |
| "retrieval-only::bm25-only::minilm", | |
| "retrieval-only::embedding-bge-m3::mxbai", | |
| "retrieval-only::splade-only::mxbai", | |
| "retrieval-only::hybrid::none", | |
| "retrieval-only::hybrid-web::none", | |
| "devstral-123b::hybrid::none", | |
| "ministral-14b-reasoning::hybrid::none", | |
| "magistral-small::hybrid::none", | |
| "devstral-small-24b::hybrid::none", | |
| "mistral-small-3.2-24b::hybrid::none", | |
| "granite-4-30b::hybrid::none", | |
| "retrieval-only::embedding-bge-large::mxbai", | |
| "retrieval-only::embedding-bge-large::minilm", | |
| "retrieval-only::splade-only::none", | |
| "retrieval-only::bm25-only::mxbai", | |
| "retrieval-only::embedding-bge-m3::none", | |
| "retrieval-only::bm25-only::none", | |
| "retrieval-only::embedding-bge-large::none", | |
| "retrieval-only::hybrid-bge-large::none" | |
| ], | |
| "pareto": [ | |
| "devstral-123b::hybrid::none", | |
| "retrieval-only::bm25-only::minilm", | |
| "retrieval-only::bm25-only::none", | |
| "retrieval-only::embedding-bge-m3::minilm", | |
| "retrieval-only::hybrid-bge-large::minilm", | |
| "retrieval-only::hybrid-triple::none", | |
| "retrieval-only::hybrid::minilm", | |
| "retrieval-only::hybrid::none", | |
| "retrieval-only::splade-only::minilm", | |
| "retrieval-only::splade-only::none" | |
| ], | |
| "dq": { | |
| "retrieval-only::hybrid-bge-large::minilm": "accuracy", | |
| "retrieval-only::hybrid-triple::minilm": "accuracy", | |
| "retrieval-only::hybrid::minilm": "accuracy", | |
| "retrieval-only::hybrid-web::minilm": "latency", | |
| "retrieval-only::hybrid-bge-large::mxbai": "latency", | |
| "retrieval-only::hybrid-triple::none": "accuracy", | |
| "retrieval-only::splade-only::minilm": "accuracy", | |
| "retrieval-only::embedding-bge-m3::minilm": "accuracy", | |
| "retrieval-only::hybrid::mxbai": "latency", | |
| "retrieval-only::hybrid-web::mxbai": "latency", | |
| "retrieval-only::hybrid-triple::mxbai": "latency", | |
| "retrieval-only::bm25-only::minilm": "accuracy", | |
| "retrieval-only::embedding-bge-m3::mxbai": "latency", | |
| "retrieval-only::splade-only::mxbai": "latency", | |
| "retrieval-only::hybrid::none": "accuracy", | |
| "retrieval-only::hybrid-web::none": "latency", | |
| "devstral-123b::hybrid::none": "cost", | |
| "ministral-14b-reasoning::hybrid::none": "accuracy", | |
| "magistral-small::hybrid::none": "accuracy", | |
| "devstral-small-24b::hybrid::none": "accuracy", | |
| "mistral-small-3.2-24b::hybrid::none": "accuracy", | |
| "granite-4-30b::hybrid::none": "accuracy", | |
| "retrieval-only::embedding-bge-large::mxbai": "latency", | |
| "retrieval-only::embedding-bge-large::minilm": "accuracy", | |
| "retrieval-only::splade-only::none": "accuracy", | |
| "retrieval-only::bm25-only::mxbai": "latency", | |
| "retrieval-only::embedding-bge-m3::none": "accuracy", | |
| "retrieval-only::bm25-only::none": "accuracy", | |
| "retrieval-only::embedding-bge-large::none": "accuracy", | |
| "retrieval-only::hybrid-bge-large::none": "accuracy" | |
| } | |
| }, | |
| "dataset-logistics-qa": { | |
| "order": [ | |
| "devstral-123b::hybrid::none", | |
| "ministral-14b-reasoning::hybrid::none", | |
| "magistral-small::hybrid::none", | |
| "devstral-small-24b::hybrid::none", | |
| "mistral-small-3.2-24b::hybrid::none", | |
| "granite-4-30b::hybrid::none" | |
| ], | |
| "pareto": [ | |
| "magistral-small::hybrid::none", | |
| "ministral-14b-reasoning::hybrid::none" | |
| ], | |
| "dq": { | |
| "devstral-123b::hybrid::none": "cost" | |
| } | |
| }, | |
| "dataset-landuse-wa-qa": { | |
| "order": [ | |
| "devstral-123b::hybrid::none", | |
| "ministral-14b-reasoning::hybrid::none", | |
| "magistral-small::hybrid::none", | |
| "devstral-small-24b::hybrid::none", | |
| "mistral-small-3.2-24b::hybrid::none", | |
| "granite-4-30b::hybrid::none" | |
| ], | |
| "pareto": [ | |
| "devstral-small-24b::hybrid::none", | |
| "magistral-small::hybrid::none" | |
| ], | |
| "dq": { | |
| "devstral-123b::hybrid::none": "cost" | |
| } | |
| }, | |
| "dataset-shakespeare-qa": { | |
| "order": [ | |
| "devstral-123b::hybrid::none", | |
| "ministral-14b-reasoning::hybrid::none", | |
| "magistral-small::hybrid::none", | |
| "devstral-small-24b::hybrid::none", | |
| "mistral-small-3.2-24b::hybrid::none", | |
| "granite-4-30b::hybrid::none" | |
| ], | |
| "pareto": [ | |
| "devstral-123b::hybrid::none", | |
| "devstral-small-24b::hybrid::none", | |
| "granite-4-30b::hybrid::none", | |
| "magistral-small::hybrid::none", | |
| "mistral-small-3.2-24b::hybrid::none" | |
| ], | |
| "dq": { | |
| "devstral-123b::hybrid::none": "accuracy" | |
| } | |
| }, | |
| "dataset-futurewise-qa": { | |
| "order": [ | |
| "devstral-123b::hybrid::none", | |
| "ministral-14b-reasoning::hybrid::none", | |
| "magistral-small::hybrid::none", | |
| "devstral-small-24b::hybrid::none", | |
| "mistral-small-3.2-24b::hybrid::none", | |
| "granite-4-30b::hybrid::none" | |
| ], | |
| "pareto": [ | |
| "devstral-123b::hybrid::none", | |
| "devstral-small-24b::hybrid::none", | |
| "ministral-14b-reasoning::hybrid::none" | |
| ], | |
| "dq": { | |
| "devstral-123b::hybrid::none": "accuracy", | |
| "ministral-14b-reasoning::hybrid::none": "accuracy", | |
| "magistral-small::hybrid::none": "accuracy", | |
| "devstral-small-24b::hybrid::none": "accuracy", | |
| "mistral-small-3.2-24b::hybrid::none": "accuracy", | |
| "granite-4-30b::hybrid::none": "accuracy" | |
| } | |
| }, | |
| "dataset-belebele-qa": { | |
| "order": [ | |
| "devstral-123b::hybrid::none", | |
| "ministral-14b-reasoning::hybrid::none", | |
| "magistral-small::hybrid::none", | |
| "devstral-small-24b::hybrid::none", | |
| "mistral-small-3.2-24b::hybrid::none", | |
| "granite-4-30b::hybrid::none" | |
| ], | |
| "pareto": [ | |
| "devstral-small-24b::hybrid::none", | |
| "ministral-14b-reasoning::hybrid::none" | |
| ], | |
| "dq": {} | |
| } | |
| }, | |
| "metric_stats": { | |
| "mrr": { | |
| "retrieval-only::bm25-only::none": { | |
| "wins": 0, | |
| "losses": 0, | |
| "count": 2, | |
| "mean": 0.22009523809523818 | |
| }, | |
| "retrieval-only::bm25-only::minilm": { | |
| "wins": 0, | |
| "losses": 0, | |
| "count": 2, | |
| "mean": 0.2693194444444445 | |
| }, | |
| "retrieval-only::bm25-only::mxbai": { | |
| "wins": 0, | |
| "losses": 0, | |
| "count": 2, | |
| "mean": 0.209015873015873 | |
| }, | |
| "retrieval-only::embedding-bge-m3::none": { | |
| "wins": 0, | |
| "losses": 0, | |
| "count": 2, | |
| "mean": 0.2356309523809524 | |
| }, | |
| "retrieval-only::embedding-bge-m3::minilm": { | |
| "wins": 0, | |
| "losses": 0, | |
| "count": 2, | |
| "mean": 0.24796230158730168 | |
| }, | |
| "retrieval-only::embedding-bge-m3::mxbai": { | |
| "wins": 0, | |
| "losses": 0, | |
| "count": 2, | |
| "mean": 0.2262936507936508 | |
| }, | |
| "retrieval-only::embedding-bge-large::none": { | |
| "wins": 0, | |
| "losses": 0, | |
| "count": 2, | |
| "mean": 0.23683531746031752 | |
| }, | |
| "retrieval-only::embedding-bge-large::minilm": { | |
| "wins": 0, | |
| "losses": 0, | |
| "count": 2, | |
| "mean": 0.24646031746031757 | |
| }, | |
| "retrieval-only::embedding-bge-large::mxbai": { | |
| "wins": 0, | |
| "losses": 0, | |
| "count": 2, | |
| "mean": 0.2276051587301588 | |
| }, | |
| "retrieval-only::hybrid::none": { | |
| "wins": 1, | |
| "losses": 0, | |
| "count": 2, | |
| "mean": 0.32355357142857144 | |
| }, | |
| "retrieval-only::hybrid::minilm": { | |
| "wins": 0, | |
| "losses": 0, | |
| "count": 2, | |
| "mean": 0.2671051587301587 | |
| }, | |
| "retrieval-only::hybrid::mxbai": { | |
| "wins": 0, | |
| "losses": 0, | |
| "count": 2, | |
| "mean": 0.23812103174603175 | |
| }, | |
| "retrieval-only::hybrid-bge-large::none": { | |
| "wins": 0, | |
| "losses": 1, | |
| "count": 2, | |
| "mean": 0.3023313492063492 | |
| }, | |
| "retrieval-only::hybrid-bge-large::minilm": { | |
| "wins": 1, | |
| "losses": 0, | |
| "count": 2, | |
| "mean": 0.2783472222222223 | |
| }, | |
| "retrieval-only::hybrid-bge-large::mxbai": { | |
| "wins": 0, | |
| "losses": 0, | |
| "count": 2, | |
| "mean": 0.23511706349206357 | |
| }, | |
| "retrieval-only::splade-only::none": { | |
| "wins": 0, | |
| "losses": 0, | |
| "count": 2, | |
| "mean": 0.2220694444444445 | |
| }, | |
| "retrieval-only::splade-only::minilm": { | |
| "wins": 0, | |
| "losses": 0, | |
| "count": 2, | |
| "mean": 0.2552103174603175 | |
| }, | |
| "retrieval-only::splade-only::mxbai": { | |
| "wins": 0, | |
| "losses": 1, | |
| "count": 2, | |
| "mean": 0.2106309523809524 | |
| }, | |
| "retrieval-only::hybrid-triple::none": { | |
| "wins": 0, | |
| "losses": 0, | |
| "count": 2, | |
| "mean": 0.32097023809523817 | |
| }, | |
| "retrieval-only::hybrid-triple::minilm": { | |
| "wins": 0, | |
| "losses": 0, | |
| "count": 2, | |
| "mean": 0.2667063492063492 | |
| }, | |
| "retrieval-only::hybrid-triple::mxbai": { | |
| "wins": 0, | |
| "losses": 0, | |
| "count": 2, | |
| "mean": 0.2324484126984128 | |
| }, | |
| "retrieval-only::hybrid-web::none": { | |
| "wins": 1, | |
| "losses": 0, | |
| "count": 2, | |
| "mean": 0.32355357142857144 | |
| }, | |
| "retrieval-only::hybrid-web::minilm": { | |
| "wins": 0, | |
| "losses": 0, | |
| "count": 2, | |
| "mean": 0.2671051587301587 | |
| }, | |
| "retrieval-only::hybrid-web::mxbai": { | |
| "wins": 0, | |
| "losses": 0, | |
| "count": 2, | |
| "mean": 0.23812103174603175 | |
| }, | |
| "devstral-123b::hybrid::none": { | |
| "wins": 6, | |
| "losses": 0, | |
| "count": 7, | |
| "mean": 0.6732534013605443 | |
| }, | |
| "ministral-14b-reasoning::hybrid::none": { | |
| "wins": 6, | |
| "losses": 0, | |
| "count": 7, | |
| "mean": 0.6732534013605443 | |
| }, | |
| "magistral-small::hybrid::none": { | |
| "wins": 6, | |
| "losses": 0, | |
| "count": 7, | |
| "mean": 0.6732534013605443 | |
| }, | |
| "devstral-small-24b::hybrid::none": { | |
| "wins": 6, | |
| "losses": 0, | |
| "count": 7, | |
| "mean": 0.6732534013605443 | |
| }, | |
| "mistral-small-3.2-24b::hybrid::none": { | |
| "wins": 6, | |
| "losses": 0, | |
| "count": 7, | |
| "mean": 0.6732534013605443 | |
| }, | |
| "granite-4-30b::hybrid::none": { | |
| "wins": 6, | |
| "losses": 0, | |
| "count": 7, | |
| "mean": 0.6732534013605443 | |
| } | |
| }, | |
| "h1": { | |
| "retrieval-only::bm25-only::none": { | |
| "wins": 0, | |
| "losses": 1, | |
| "count": 2, | |
| "mean": 0.165 | |
| }, | |
| "retrieval-only::bm25-only::minilm": { | |
| "wins": 0, | |
| "losses": 0, | |
| "count": 2, | |
| "mean": 0.2 | |
| }, | |
| "retrieval-only::bm25-only::mxbai": { | |
| "wins": 0, | |
| "losses": 1, | |
| "count": 2, | |
| "mean": 0.14500000000000002 | |
| }, | |
| "retrieval-only::embedding-bge-m3::none": { | |
| "wins": 0, | |
| "losses": 0, | |
| "count": 2, | |
| "mean": 0.155 | |
| }, | |
| "retrieval-only::embedding-bge-m3::minilm": { | |
| "wins": 0, | |
| "losses": 0, | |
| "count": 2, | |
| "mean": 0.18 | |
| }, | |
| "retrieval-only::embedding-bge-m3::mxbai": { | |
| "wins": 0, | |
| "losses": 0, | |
| "count": 2, | |
| "mean": 0.165 | |
| }, | |
| "retrieval-only::embedding-bge-large::none": { | |
| "wins": 0, | |
| "losses": 1, | |
| "count": 2, | |
| "mean": 0.18 | |
| }, | |
| "retrieval-only::embedding-bge-large::minilm": { | |
| "wins": 0, | |
| "losses": 0, | |
| "count": 2, | |
| "mean": 0.17500000000000002 | |
| }, | |
| "retrieval-only::embedding-bge-large::mxbai": { | |
| "wins": 0, | |
| "losses": 0, | |
| "count": 2, | |
| "mean": 0.17 | |
| }, | |
| "retrieval-only::hybrid::none": { | |
| "wins": 1, | |
| "losses": 0, | |
| "count": 2, | |
| "mean": 0.275 | |
| }, | |
| "retrieval-only::hybrid::minilm": { | |
| "wins": 0, | |
| "losses": 0, | |
| "count": 2, | |
| "mean": 0.19 | |
| }, | |
| "retrieval-only::hybrid::mxbai": { | |
| "wins": 1, | |
| "losses": 0, | |
| "count": 2, | |
| "mean": 0.175 | |
| }, | |
| "retrieval-only::hybrid-bge-large::none": { | |
| "wins": 0, | |
| "losses": 1, | |
| "count": 2, | |
| "mean": 0.26 | |
| }, | |
| "retrieval-only::hybrid-bge-large::minilm": { | |
| "wins": 0, | |
| "losses": 0, | |
| "count": 2, | |
| "mean": 0.185 | |
| }, | |
| "retrieval-only::hybrid-bge-large::mxbai": { | |
| "wins": 0, | |
| "losses": 0, | |
| "count": 2, | |
| "mean": 0.16 | |
| }, | |
| "retrieval-only::splade-only::none": { | |
| "wins": 0, | |
| "losses": 1, | |
| "count": 2, | |
| "mean": 0.165 | |
| }, | |
| "retrieval-only::splade-only::minilm": { | |
| "wins": 0, | |
| "losses": 0, | |
| "count": 2, | |
| "mean": 0.19 | |
| }, | |
| "retrieval-only::splade-only::mxbai": { | |
| "wins": 0, | |
| "losses": 0, | |
| "count": 2, | |
| "mean": 0.155 | |
| }, | |
| "retrieval-only::hybrid-triple::none": { | |
| "wins": 0, | |
| "losses": 0, | |
| "count": 2, | |
| "mean": 0.27 | |
| }, | |
| "retrieval-only::hybrid-triple::minilm": { | |
| "wins": 0, | |
| "losses": 0, | |
| "count": 2, | |
| "mean": 0.19 | |
| }, | |
| "retrieval-only::hybrid-triple::mxbai": { | |
| "wins": 1, | |
| "losses": 0, | |
| "count": 2, | |
| "mean": 0.16999999999999998 | |
| }, | |
| "retrieval-only::hybrid-web::none": { | |
| "wins": 1, | |
| "losses": 0, | |
| "count": 2, | |
| "mean": 0.275 | |
| }, | |
| "retrieval-only::hybrid-web::minilm": { | |
| "wins": 0, | |
| "losses": 0, | |
| "count": 2, | |
| "mean": 0.19 | |
| }, | |
| "retrieval-only::hybrid-web::mxbai": { | |
| "wins": 1, | |
| "losses": 0, | |
| "count": 2, | |
| "mean": 0.175 | |
| }, | |
| "devstral-123b::hybrid::none": { | |
| "wins": 6, | |
| "losses": 0, | |
| "count": 7, | |
| "mean": 0.647142857142857 | |
| }, | |
| "ministral-14b-reasoning::hybrid::none": { | |
| "wins": 6, | |
| "losses": 0, | |
| "count": 7, | |
| "mean": 0.647142857142857 | |
| }, | |
| "magistral-small::hybrid::none": { | |
| "wins": 6, | |
| "losses": 0, | |
| "count": 7, | |
| "mean": 0.647142857142857 | |
| }, | |
| "devstral-small-24b::hybrid::none": { | |
| "wins": 6, | |
| "losses": 0, | |
| "count": 7, | |
| "mean": 0.647142857142857 | |
| }, | |
| "mistral-small-3.2-24b::hybrid::none": { | |
| "wins": 6, | |
| "losses": 0, | |
| "count": 7, | |
| "mean": 0.647142857142857 | |
| }, | |
| "granite-4-30b::hybrid::none": { | |
| "wins": 6, | |
| "losses": 0, | |
| "count": 7, | |
| "mean": 0.647142857142857 | |
| } | |
| }, | |
| "h5": { | |
| "retrieval-only::bm25-only::none": { | |
| "wins": 0, | |
| "losses": 1, | |
| "count": 2, | |
| "mean": 0.28500000000000003 | |
| }, | |
| "retrieval-only::bm25-only::minilm": { | |
| "wins": 0, | |
| "losses": 0, | |
| "count": 2, | |
| "mean": 0.355 | |
| }, | |
| "retrieval-only::bm25-only::mxbai": { | |
| "wins": 0, | |
| "losses": 0, | |
| "count": 2, | |
| "mean": 0.31499999999999995 | |
| }, | |
| "retrieval-only::embedding-bge-m3::none": { | |
| "wins": 0, | |
| "losses": 0, | |
| "count": 2, | |
| "mean": 0.345 | |
| }, | |
| "retrieval-only::embedding-bge-m3::minilm": { | |
| "wins": 0, | |
| "losses": 0, | |
| "count": 2, | |
| "mean": 0.35 | |
| }, | |
| "retrieval-only::embedding-bge-m3::mxbai": { | |
| "wins": 0, | |
| "losses": 0, | |
| "count": 2, | |
| "mean": 0.29500000000000004 | |
| }, | |
| "retrieval-only::embedding-bge-large::none": { | |
| "wins": 0, | |
| "losses": 0, | |
| "count": 2, | |
| "mean": 0.30500000000000005 | |
| }, | |
| "retrieval-only::embedding-bge-large::minilm": { | |
| "wins": 0, | |
| "losses": 0, | |
| "count": 2, | |
| "mean": 0.345 | |
| }, | |
| "retrieval-only::embedding-bge-large::mxbai": { | |
| "wins": 0, | |
| "losses": 0, | |
| "count": 2, | |
| "mean": 0.31499999999999995 | |
| }, | |
| "retrieval-only::hybrid::none": { | |
| "wins": 1, | |
| "losses": 0, | |
| "count": 2, | |
| "mean": 0.39 | |
| }, | |
| "retrieval-only::hybrid::minilm": { | |
| "wins": 0, | |
| "losses": 0, | |
| "count": 2, | |
| "mean": 0.355 | |
| }, | |
| "retrieval-only::hybrid::mxbai": { | |
| "wins": 0, | |
| "losses": 0, | |
| "count": 2, | |
| "mean": 0.315 | |
| }, | |
| "retrieval-only::hybrid-bge-large::none": { | |
| "wins": 0, | |
| "losses": 0, | |
| "count": 2, | |
| "mean": 0.36 | |
| }, | |
| "retrieval-only::hybrid-bge-large::minilm": { | |
| "wins": 1, | |
| "losses": 0, | |
| "count": 2, | |
| "mean": 0.4 | |
| }, | |
| "retrieval-only::hybrid-bge-large::mxbai": { | |
| "wins": 0, | |
| "losses": 0, | |
| "count": 2, | |
| "mean": 0.315 | |
| }, | |
| "retrieval-only::splade-only::none": { | |
| "wins": 0, | |
| "losses": 1, | |
| "count": 2, | |
| "mean": 0.29 | |
| }, | |
| "retrieval-only::splade-only::minilm": { | |
| "wins": 0, | |
| "losses": 0, | |
| "count": 2, | |
| "mean": 0.33999999999999997 | |
| }, | |
| "retrieval-only::splade-only::mxbai": { | |
| "wins": 0, | |
| "losses": 0, | |
| "count": 2, | |
| "mean": 0.29 | |
| }, | |
| "retrieval-only::hybrid-triple::none": { | |
| "wins": 0, | |
| "losses": 0, | |
| "count": 2, | |
| "mean": 0.38 | |
| }, | |
| "retrieval-only::hybrid-triple::minilm": { | |
| "wins": 0, | |
| "losses": 0, | |
| "count": 2, | |
| "mean": 0.37 | |
| }, | |
| "retrieval-only::hybrid-triple::mxbai": { | |
| "wins": 0, | |
| "losses": 0, | |
| "count": 2, | |
| "mean": 0.3 | |
| }, | |
| "retrieval-only::hybrid-web::none": { | |
| "wins": 1, | |
| "losses": 0, | |
| "count": 2, | |
| "mean": 0.39 | |
| }, | |
| "retrieval-only::hybrid-web::minilm": { | |
| "wins": 0, | |
| "losses": 0, | |
| "count": 2, | |
| "mean": 0.355 | |
| }, | |
| "retrieval-only::hybrid-web::mxbai": { | |
| "wins": 0, | |
| "losses": 0, | |
| "count": 2, | |
| "mean": 0.315 | |
| }, | |
| "devstral-123b::hybrid::none": { | |
| "wins": 6, | |
| "losses": 0, | |
| "count": 7, | |
| "mean": 0.7089795918367346 | |
| }, | |
| "ministral-14b-reasoning::hybrid::none": { | |
| "wins": 6, | |
| "losses": 0, | |
| "count": 7, | |
| "mean": 0.7089795918367346 | |
| }, | |
| "magistral-small::hybrid::none": { | |
| "wins": 6, | |
| "losses": 0, | |
| "count": 7, | |
| "mean": 0.7089795918367346 | |
| }, | |
| "devstral-small-24b::hybrid::none": { | |
| "wins": 6, | |
| "losses": 0, | |
| "count": 7, | |
| "mean": 0.7089795918367346 | |
| }, | |
| "mistral-small-3.2-24b::hybrid::none": { | |
| "wins": 6, | |
| "losses": 0, | |
| "count": 7, | |
| "mean": 0.7089795918367346 | |
| }, | |
| "granite-4-30b::hybrid::none": { | |
| "wins": 6, | |
| "losses": 0, | |
| "count": 7, | |
| "mean": 0.7089795918367346 | |
| } | |
| }, | |
| "ndcg": { | |
| "retrieval-only::bm25-only::none": { | |
| "wins": 0, | |
| "losses": 0, | |
| "count": 2, | |
| "mean": 0.2583013442462098 | |
| }, | |
| "retrieval-only::bm25-only::minilm": { | |
| "wins": 0, | |
| "losses": 0, | |
| "count": 2, | |
| "mean": 0.30734957235292343 | |
| }, | |
| "retrieval-only::bm25-only::mxbai": { | |
| "wins": 0, | |
| "losses": 0, | |
| "count": 2, | |
| "mean": 0.251605778206734 | |
| }, | |
| "retrieval-only::embedding-bge-m3::none": { | |
| "wins": 0, | |
| "losses": 0, | |
| "count": 2, | |
| "mean": 0.27830895027512786 | |
| }, | |
| "retrieval-only::embedding-bge-m3::minilm": { | |
| "wins": 0, | |
| "losses": 0, | |
| "count": 2, | |
| "mean": 0.28984281647385224 | |
| }, | |
| "retrieval-only::embedding-bge-m3::mxbai": { | |
| "wins": 0, | |
| "losses": 0, | |
| "count": 2, | |
| "mean": 0.2680014625660154 | |
| }, | |
| "retrieval-only::embedding-bge-large::none": { | |
| "wins": 0, | |
| "losses": 1, | |
| "count": 2, | |
| "mean": 0.2726846673676603 | |
| }, | |
| "retrieval-only::embedding-bge-large::minilm": { | |
| "wins": 0, | |
| "losses": 0, | |
| "count": 2, | |
| "mean": 0.2951497118077595 | |
| }, | |
| "retrieval-only::embedding-bge-large::mxbai": { | |
| "wins": 0, | |
| "losses": 0, | |
| "count": 2, | |
| "mean": 0.26683272244230594 | |
| }, | |
| "retrieval-only::hybrid::none": { | |
| "wins": 1, | |
| "losses": 0, | |
| "count": 2, | |
| "mean": 0.3490078331906213 | |
| }, | |
| "retrieval-only::hybrid::minilm": { | |
| "wins": 0, | |
| "losses": 0, | |
| "count": 2, | |
| "mean": 0.3124858976171995 | |
| }, | |
| "retrieval-only::hybrid::mxbai": { | |
| "wins": 0, | |
| "losses": 0, | |
| "count": 2, | |
| "mean": 0.2816296718223945 | |
| }, | |
| "retrieval-only::hybrid-bge-large::none": { | |
| "wins": 0, | |
| "losses": 0, | |
| "count": 2, | |
| "mean": 0.33097737940028366 | |
| }, | |
| "retrieval-only::hybrid-bge-large::minilm": { | |
| "wins": 1, | |
| "losses": 0, | |
| "count": 2, | |
| "mean": 0.33428433272211305 | |
| }, | |
| "retrieval-only::hybrid-bge-large::mxbai": { | |
| "wins": 0, | |
| "losses": 0, | |
| "count": 2, | |
| "mean": 0.2840729725949876 | |
| }, | |
| "retrieval-only::splade-only::none": { | |
| "wins": 0, | |
| "losses": 0, | |
| "count": 2, | |
| "mean": 0.261045061542026 | |
| }, | |
| "retrieval-only::splade-only::minilm": { | |
| "wins": 0, | |
| "losses": 0, | |
| "count": 2, | |
| "mean": 0.2997848863194258 | |
| }, | |
| "retrieval-only::splade-only::mxbai": { | |
| "wins": 0, | |
| "losses": 1, | |
| "count": 2, | |
| "mean": 0.2458770321015341 | |
| }, | |
| "retrieval-only::hybrid-triple::none": { | |
| "wins": 0, | |
| "losses": 0, | |
| "count": 2, | |
| "mean": 0.34810720388560956 | |
| }, | |
| "retrieval-only::hybrid-triple::minilm": { | |
| "wins": 0, | |
| "losses": 0, | |
| "count": 2, | |
| "mean": 0.31880769420680904 | |
| }, | |
| "retrieval-only::hybrid-triple::mxbai": { | |
| "wins": 0, | |
| "losses": 0, | |
| "count": 2, | |
| "mean": 0.2747457569398053 | |
| }, | |
| "retrieval-only::hybrid-web::none": { | |
| "wins": 1, | |
| "losses": 0, | |
| "count": 2, | |
| "mean": 0.3490078331906213 | |
| }, | |
| "retrieval-only::hybrid-web::minilm": { | |
| "wins": 0, | |
| "losses": 0, | |
| "count": 2, | |
| "mean": 0.3124858976171995 | |
| }, | |
| "retrieval-only::hybrid-web::mxbai": { | |
| "wins": 0, | |
| "losses": 0, | |
| "count": 2, | |
| "mean": 0.2816296718223945 | |
| }, | |
| "devstral-123b::hybrid::none": { | |
| "wins": 6, | |
| "losses": 0, | |
| "count": 7, | |
| "mean": 0.6863395842500024 | |
| }, | |
| "ministral-14b-reasoning::hybrid::none": { | |
| "wins": 6, | |
| "losses": 0, | |
| "count": 7, | |
| "mean": 0.6863395842500024 | |
| }, | |
| "magistral-small::hybrid::none": { | |
| "wins": 6, | |
| "losses": 0, | |
| "count": 7, | |
| "mean": 0.6863395842500024 | |
| }, | |
| "devstral-small-24b::hybrid::none": { | |
| "wins": 6, | |
| "losses": 0, | |
| "count": 7, | |
| "mean": 0.6863395842500024 | |
| }, | |
| "mistral-small-3.2-24b::hybrid::none": { | |
| "wins": 6, | |
| "losses": 0, | |
| "count": 7, | |
| "mean": 0.6863395842500024 | |
| }, | |
| "granite-4-30b::hybrid::none": { | |
| "wins": 6, | |
| "losses": 0, | |
| "count": 7, | |
| "mean": 0.6863395842500024 | |
| } | |
| }, | |
| "p50": { | |
| "retrieval-only::bm25-only::none": { | |
| "wins": 2, | |
| "losses": 0, | |
| "count": 2, | |
| "mean": 0.24090008810162544 | |
| }, | |
| "retrieval-only::bm25-only::minilm": { | |
| "wins": 0, | |
| "losses": 0, | |
| "count": 2, | |
| "mean": 22.695181891322136 | |
| }, | |
| "retrieval-only::bm25-only::mxbai": { | |
| "wins": 0, | |
| "losses": 0, | |
| "count": 2, | |
| "mean": 190.02285366877913 | |
| }, | |
| "retrieval-only::embedding-bge-m3::none": { | |
| "wins": 0, | |
| "losses": 0, | |
| "count": 2, | |
| "mean": 5.813977215439081 | |
| }, | |
| "retrieval-only::embedding-bge-m3::minilm": { | |
| "wins": 0, | |
| "losses": 0, | |
| "count": 2, | |
| "mean": 29.342605732381344 | |
| }, | |
| "retrieval-only::embedding-bge-m3::mxbai": { | |
| "wins": 0, | |
| "losses": 0, | |
| "count": 2, | |
| "mean": 203.89279909431934 | |
| }, | |
| "retrieval-only::embedding-bge-large::none": { | |
| "wins": 0, | |
| "losses": 0, | |
| "count": 2, | |
| "mean": 5.891583859920502 | |
| }, | |
| "retrieval-only::embedding-bge-large::minilm": { | |
| "wins": 0, | |
| "losses": 0, | |
| "count": 2, | |
| "mean": 29.316066298633814 | |
| }, | |
| "retrieval-only::embedding-bge-large::mxbai": { | |
| "wins": 0, | |
| "losses": 0, | |
| "count": 2, | |
| "mean": 204.86171124503016 | |
| }, | |
| "retrieval-only::hybrid::none": { | |
| "wins": 0, | |
| "losses": 0, | |
| "count": 2, | |
| "mean": 6.42356975004077 | |
| }, | |
| "retrieval-only::hybrid::minilm": { | |
| "wins": 0, | |
| "losses": 0, | |
| "count": 2, | |
| "mean": 31.444795429706573 | |
| }, | |
| "retrieval-only::hybrid::mxbai": { | |
| "wins": 0, | |
| "losses": 0, | |
| "count": 2, | |
| "mean": 207.23473047837615 | |
| }, | |
| "retrieval-only::hybrid-bge-large::none": { | |
| "wins": 0, | |
| "losses": 0, | |
| "count": 2, | |
| "mean": 6.400614511221647 | |
| }, | |
| "retrieval-only::hybrid-bge-large::minilm": { | |
| "wins": 0, | |
| "losses": 0, | |
| "count": 2, | |
| "mean": 30.800541397184134 | |
| }, | |
| "retrieval-only::hybrid-bge-large::mxbai": { | |
| "wins": 0, | |
| "losses": 0, | |
| "count": 2, | |
| "mean": 206.9518780335784 | |
| }, | |
| "retrieval-only::splade-only::none": { | |
| "wins": 0, | |
| "losses": 0, | |
| "count": 2, | |
| "mean": 2.768915146589279 | |
| }, | |
| "retrieval-only::splade-only::minilm": { | |
| "wins": 0, | |
| "losses": 0, | |
| "count": 2, | |
| "mean": 26.865416672080755 | |
| }, | |
| "retrieval-only::splade-only::mxbai": { | |
| "wins": 0, | |
| "losses": 0, | |
| "count": 2, | |
| "mean": 202.86574540659785 | |
| }, | |
| "retrieval-only::hybrid-triple::none": { | |
| "wins": 0, | |
| "losses": 0, | |
| "count": 2, | |
| "mean": 9.532728232443333 | |
| }, | |
| "retrieval-only::hybrid-triple::minilm": { | |
| "wins": 0, | |
| "losses": 0, | |
| "count": 2, | |
| "mean": 34.74805224686861 | |
| }, | |
| "retrieval-only::hybrid-triple::mxbai": { | |
| "wins": 0, | |
| "losses": 0, | |
| "count": 2, | |
| "mean": 208.22617365047336 | |
| }, | |
| "retrieval-only::hybrid-web::none": { | |
| "wins": 0, | |
| "losses": 0, | |
| "count": 2, | |
| "mean": 2549.0026334300637 | |
| }, | |
| "retrieval-only::hybrid-web::minilm": { | |
| "wins": 0, | |
| "losses": 0, | |
| "count": 2, | |
| "mean": 5065.05661085248 | |
| }, | |
| "retrieval-only::hybrid-web::mxbai": { | |
| "wins": 0, | |
| "losses": 2, | |
| "count": 2, | |
| "mean": 5197.566072456539 | |
| }, | |
| "devstral-123b::hybrid::none": { | |
| "wins": 0, | |
| "losses": 5, | |
| "count": 7, | |
| "mean": 8.211523560541016 | |
| }, | |
| "ministral-14b-reasoning::hybrid::none": { | |
| "wins": 1, | |
| "losses": 0, | |
| "count": 7, | |
| "mean": 7.412772226546492 | |
| }, | |
| "magistral-small::hybrid::none": { | |
| "wins": 0, | |
| "losses": 0, | |
| "count": 7, | |
| "mean": 7.392910974366324 | |
| }, | |
| "devstral-small-24b::hybrid::none": { | |
| "wins": 4, | |
| "losses": 0, | |
| "count": 7, | |
| "mean": 7.296505916331496 | |
| }, | |
| "mistral-small-3.2-24b::hybrid::none": { | |
| "wins": 0, | |
| "losses": 0, | |
| "count": 7, | |
| "mean": 7.394192075090749 | |
| }, | |
| "granite-4-30b::hybrid::none": { | |
| "wins": 0, | |
| "losses": 0, | |
| "count": 7, | |
| "mean": 7.358867808112076 | |
| } | |
| }, | |
| "p95": { | |
| "retrieval-only::bm25-only::none": { | |
| "wins": 2, | |
| "losses": 0, | |
| "count": 2, | |
| "mean": 0.5216426216065884 | |
| }, | |
| "retrieval-only::bm25-only::minilm": { | |
| "wins": 0, | |
| "losses": 0, | |
| "count": 2, | |
| "mean": 24.78122152388096 | |
| }, | |
| "retrieval-only::bm25-only::mxbai": { | |
| "wins": 0, | |
| "losses": 0, | |
| "count": 2, | |
| "mean": 194.22026490792632 | |
| }, | |
| "retrieval-only::embedding-bge-m3::none": { | |
| "wins": 0, | |
| "losses": 0, | |
| "count": 2, | |
| "mean": 6.5811071544885635 | |
| }, | |
| "retrieval-only::embedding-bge-m3::minilm": { | |
| "wins": 0, | |
| "losses": 0, | |
| "count": 2, | |
| "mean": 32.704496290534735 | |
| }, | |
| "retrieval-only::embedding-bge-m3::mxbai": { | |
| "wins": 0, | |
| "losses": 0, | |
| "count": 2, | |
| "mean": 207.5708662159741 | |
| }, | |
| "retrieval-only::embedding-bge-large::none": { | |
| "wins": 0, | |
| "losses": 0, | |
| "count": 2, | |
| "mean": 6.531184073537588 | |
| }, | |
| "retrieval-only::embedding-bge-large::minilm": { | |
| "wins": 0, | |
| "losses": 0, | |
| "count": 2, | |
| "mean": 31.900114379823208 | |
| }, | |
| "retrieval-only::embedding-bge-large::mxbai": { | |
| "wins": 0, | |
| "losses": 0, | |
| "count": 2, | |
| "mean": 208.62957322970033 | |
| }, | |
| "retrieval-only::hybrid::none": { | |
| "wins": 0, | |
| "losses": 0, | |
| "count": 2, | |
| "mean": 7.1006291545927525 | |
| }, | |
| "retrieval-only::hybrid::minilm": { | |
| "wins": 0, | |
| "losses": 0, | |
| "count": 2, | |
| "mean": 33.13743881881237 | |
| }, | |
| "retrieval-only::hybrid::mxbai": { | |
| "wins": 0, | |
| "losses": 0, | |
| "count": 2, | |
| "mean": 209.9398672580719 | |
| }, | |
| "retrieval-only::hybrid-bge-large::none": { | |
| "wins": 0, | |
| "losses": 0, | |
| "count": 2, | |
| "mean": 7.356016896665096 | |
| }, | |
| "retrieval-only::hybrid-bge-large::minilm": { | |
| "wins": 0, | |
| "losses": 0, | |
| "count": 2, | |
| "mean": 32.61782508343458 | |
| }, | |
| "retrieval-only::hybrid-bge-large::mxbai": { | |
| "wins": 0, | |
| "losses": 0, | |
| "count": 2, | |
| "mean": 210.08411142975092 | |
| }, | |
| "retrieval-only::splade-only::none": { | |
| "wins": 0, | |
| "losses": 0, | |
| "count": 2, | |
| "mean": 3.0150958336889744 | |
| }, | |
| "retrieval-only::splade-only::minilm": { | |
| "wins": 0, | |
| "losses": 0, | |
| "count": 2, | |
| "mean": 28.595024719834328 | |
| }, | |
| "retrieval-only::splade-only::mxbai": { | |
| "wins": 0, | |
| "losses": 0, | |
| "count": 2, | |
| "mean": 206.21620258316398 | |
| }, | |
| "retrieval-only::hybrid-triple::none": { | |
| "wins": 0, | |
| "losses": 0, | |
| "count": 2, | |
| "mean": 10.518142487853765 | |
| }, | |
| "retrieval-only::hybrid-triple::minilm": { | |
| "wins": 0, | |
| "losses": 0, | |
| "count": 2, | |
| "mean": 37.638560868799686 | |
| }, | |
| "retrieval-only::hybrid-triple::mxbai": { | |
| "wins": 0, | |
| "losses": 0, | |
| "count": 2, | |
| "mean": 211.2135230563581 | |
| }, | |
| "retrieval-only::hybrid-web::none": { | |
| "wins": 0, | |
| "losses": 0, | |
| "count": 2, | |
| "mean": 5030.530819203705 | |
| }, | |
| "retrieval-only::hybrid-web::minilm": { | |
| "wins": 0, | |
| "losses": 0, | |
| "count": 2, | |
| "mean": 5073.805775493383 | |
| }, | |
| "retrieval-only::hybrid-web::mxbai": { | |
| "wins": 0, | |
| "losses": 2, | |
| "count": 2, | |
| "mean": 5225.449657533318 | |
| }, | |
| "devstral-123b::hybrid::none": { | |
| "wins": 0, | |
| "losses": 5, | |
| "count": 7, | |
| "mean": 9.212801099887916 | |
| }, | |
| "ministral-14b-reasoning::hybrid::none": { | |
| "wins": 0, | |
| "losses": 0, | |
| "count": 7, | |
| "mean": 7.984842838985579 | |
| }, | |
| "magistral-small::hybrid::none": { | |
| "wins": 1, | |
| "losses": 0, | |
| "count": 7, | |
| "mean": 8.029126961316381 | |
| }, | |
| "devstral-small-24b::hybrid::none": { | |
| "wins": 2, | |
| "losses": 0, | |
| "count": 7, | |
| "mean": 7.915892239127841 | |
| }, | |
| "mistral-small-3.2-24b::hybrid::none": { | |
| "wins": 0, | |
| "losses": 0, | |
| "count": 7, | |
| "mean": 7.930300065449306 | |
| }, | |
| "granite-4-30b::hybrid::none": { | |
| "wins": 2, | |
| "losses": 0, | |
| "count": 7, | |
| "mean": 7.880111091903278 | |
| } | |
| }, | |
| "build": { | |
| "retrieval-only::bm25-only::none": { | |
| "wins": 0, | |
| "losses": 0, | |
| "count": 2, | |
| "mean": 0.6894275164231658 | |
| }, | |
| "retrieval-only::bm25-only::minilm": { | |
| "wins": 0, | |
| "losses": 0, | |
| "count": 2, | |
| "mean": 0.010334129445254803 | |
| }, | |
| "retrieval-only::bm25-only::mxbai": { | |
| "wins": 0, | |
| "losses": 0, | |
| "count": 2, | |
| "mean": 0.01179773872718215 | |
| }, | |
| "retrieval-only::embedding-bge-m3::none": { | |
| "wins": 0, | |
| "losses": 0, | |
| "count": 2, | |
| "mean": 0.02554283756762743 | |
| }, | |
| "retrieval-only::embedding-bge-m3::minilm": { | |
| "wins": 0, | |
| "losses": 0, | |
| "count": 2, | |
| "mean": 0.011171562131494284 | |
| }, | |
| "retrieval-only::embedding-bge-m3::mxbai": { | |
| "wins": 0, | |
| "losses": 0, | |
| "count": 2, | |
| "mean": 0.02657144283875823 | |
| }, | |
| "retrieval-only::embedding-bge-large::none": { | |
| "wins": 0, | |
| "losses": 0, | |
| "count": 2, | |
| "mean": 0.010330421384423971 | |
| }, | |
| "retrieval-only::embedding-bge-large::minilm": { | |
| "wins": 0, | |
| "losses": 0, | |
| "count": 2, | |
| "mean": 0.010938801802694798 | |
| }, | |
| "retrieval-only::embedding-bge-large::mxbai": { | |
| "wins": 0, | |
| "losses": 0, | |
| "count": 2, | |
| "mean": 0.011622074991464615 | |
| }, | |
| "retrieval-only::hybrid::none": { | |
| "wins": 0, | |
| "losses": 0, | |
| "count": 2, | |
| "mean": 0.032247269526124 | |
| }, | |
| "retrieval-only::hybrid::minilm": { | |
| "wins": 0, | |
| "losses": 0, | |
| "count": 2, | |
| "mean": 0.016082720831036568 | |
| }, | |
| "retrieval-only::hybrid::mxbai": { | |
| "wins": 0, | |
| "losses": 0, | |
| "count": 2, | |
| "mean": 0.03033801121637225 | |
| }, | |
| "retrieval-only::hybrid-bge-large::none": { | |
| "wins": 0, | |
| "losses": 0, | |
| "count": 2, | |
| "mean": 0.016932107508182526 | |
| }, | |
| "retrieval-only::hybrid-bge-large::minilm": { | |
| "wins": 0, | |
| "losses": 0, | |
| "count": 2, | |
| "mean": 0.016773953568190336 | |
| }, | |
| "retrieval-only::hybrid-bge-large::mxbai": { | |
| "wins": 0, | |
| "losses": 0, | |
| "count": 2, | |
| "mean": 0.016854459885507822 | |
| }, | |
| "retrieval-only::splade-only::none": { | |
| "wins": 0, | |
| "losses": 0, | |
| "count": 2, | |
| "mean": 0.007179258391261101 | |
| }, | |
| "retrieval-only::splade-only::minilm": { | |
| "wins": 1, | |
| "losses": 0, | |
| "count": 2, | |
| "mean": 0.006352534517645836 | |
| }, | |
| "retrieval-only::splade-only::mxbai": { | |
| "wins": 1, | |
| "losses": 0, | |
| "count": 2, | |
| "mean": 0.01976611651480198 | |
| }, | |
| "retrieval-only::hybrid-triple::none": { | |
| "wins": 0, | |
| "losses": 0, | |
| "count": 2, | |
| "mean": 0.018045139964669943 | |
| }, | |
| "retrieval-only::hybrid-triple::minilm": { | |
| "wins": 0, | |
| "losses": 0, | |
| "count": 2, | |
| "mean": 0.018401450011879206 | |
| }, | |
| "retrieval-only::hybrid-triple::mxbai": { | |
| "wins": 0, | |
| "losses": 0, | |
| "count": 2, | |
| "mean": 0.018003772478550673 | |
| }, | |
| "retrieval-only::hybrid-web::none": { | |
| "wins": 0, | |
| "losses": 2, | |
| "count": 2, | |
| "mean": 15.669111443683505 | |
| }, | |
| "retrieval-only::hybrid-web::minilm": { | |
| "wins": 0, | |
| "losses": 0, | |
| "count": 2, | |
| "mean": 14.033660519868135 | |
| }, | |
| "retrieval-only::hybrid-web::mxbai": { | |
| "wins": 0, | |
| "losses": 0, | |
| "count": 2, | |
| "mean": 13.982121753040701 | |
| }, | |
| "devstral-123b::hybrid::none": { | |
| "wins": 2, | |
| "losses": 2, | |
| "count": 7, | |
| "mean": 0.033393271932644505 | |
| }, | |
| "ministral-14b-reasoning::hybrid::none": { | |
| "wins": 2, | |
| "losses": 1, | |
| "count": 7, | |
| "mean": 0.03156101557293108 | |
| }, | |
| "magistral-small::hybrid::none": { | |
| "wins": 1, | |
| "losses": 0, | |
| "count": 7, | |
| "mean": 0.02820786434624876 | |
| }, | |
| "devstral-small-24b::hybrid::none": { | |
| "wins": 0, | |
| "losses": 0, | |
| "count": 7, | |
| "mean": 0.027854623539107188 | |
| }, | |
| "mistral-small-3.2-24b::hybrid::none": { | |
| "wins": 0, | |
| "losses": 0, | |
| "count": 7, | |
| "mean": 0.028560412781579152 | |
| }, | |
| "granite-4-30b::hybrid::none": { | |
| "wins": 0, | |
| "losses": 2, | |
| "count": 7, | |
| "mean": 0.02849655132740736 | |
| } | |
| }, | |
| "gen": { | |
| "devstral-123b::hybrid::none": { | |
| "wins": 7, | |
| "losses": 0, | |
| "count": 7, | |
| "mean": 0.7942857142857144 | |
| } | |
| }, | |
| "gen_ms": { | |
| "devstral-123b::hybrid::none": { | |
| "wins": 7, | |
| "losses": 0, | |
| "count": 7, | |
| "mean": 582.9223734991891 | |
| } | |
| }, | |
| "tok_s": { | |
| "devstral-123b::hybrid::none": { | |
| "wins": 7, | |
| "losses": 0, | |
| "count": 7, | |
| "mean": 636.8282663437029 | |
| } | |
| }, | |
| "gen_cost": { | |
| "devstral-123b::hybrid::none": { | |
| "wins": 7, | |
| "losses": 0, | |
| "count": 7, | |
| "mean": 0.0009117746938775508 | |
| } | |
| }, | |
| "cost": { | |
| "retrieval-only::bm25-only::none": { | |
| "wins": 2, | |
| "losses": 0, | |
| "count": 2, | |
| "mean": 0.0 | |
| }, | |
| "retrieval-only::bm25-only::minilm": { | |
| "wins": 2, | |
| "losses": 0, | |
| "count": 2, | |
| "mean": 0.0 | |
| }, | |
| "retrieval-only::bm25-only::mxbai": { | |
| "wins": 2, | |
| "losses": 0, | |
| "count": 2, | |
| "mean": 0.0 | |
| }, | |
| "retrieval-only::embedding-bge-m3::none": { | |
| "wins": 2, | |
| "losses": 0, | |
| "count": 2, | |
| "mean": 0.0 | |
| }, | |
| "retrieval-only::embedding-bge-m3::minilm": { | |
| "wins": 2, | |
| "losses": 0, | |
| "count": 2, | |
| "mean": 0.0 | |
| }, | |
| "retrieval-only::embedding-bge-m3::mxbai": { | |
| "wins": 2, | |
| "losses": 0, | |
| "count": 2, | |
| "mean": 0.0 | |
| }, | |
| "retrieval-only::embedding-bge-large::none": { | |
| "wins": 2, | |
| "losses": 0, | |
| "count": 2, | |
| "mean": 0.0 | |
| }, | |
| "retrieval-only::embedding-bge-large::minilm": { | |
| "wins": 2, | |
| "losses": 0, | |
| "count": 2, | |
| "mean": 0.0 | |
| }, | |
| "retrieval-only::embedding-bge-large::mxbai": { | |
| "wins": 2, | |
| "losses": 0, | |
| "count": 2, | |
| "mean": 0.0 | |
| }, | |
| "retrieval-only::hybrid::none": { | |
| "wins": 2, | |
| "losses": 0, | |
| "count": 2, | |
| "mean": 0.0 | |
| }, | |
| "retrieval-only::hybrid::minilm": { | |
| "wins": 2, | |
| "losses": 0, | |
| "count": 2, | |
| "mean": 0.0 | |
| }, | |
| "retrieval-only::hybrid::mxbai": { | |
| "wins": 2, | |
| "losses": 0, | |
| "count": 2, | |
| "mean": 0.0 | |
| }, | |
| "retrieval-only::hybrid-bge-large::none": { | |
| "wins": 2, | |
| "losses": 0, | |
| "count": 2, | |
| "mean": 0.0 | |
| }, | |
| "retrieval-only::hybrid-bge-large::minilm": { | |
| "wins": 2, | |
| "losses": 0, | |
| "count": 2, | |
| "mean": 0.0 | |
| }, | |
| "retrieval-only::hybrid-bge-large::mxbai": { | |
| "wins": 2, | |
| "losses": 0, | |
| "count": 2, | |
| "mean": 0.0 | |
| }, | |
| "retrieval-only::splade-only::none": { | |
| "wins": 2, | |
| "losses": 0, | |
| "count": 2, | |
| "mean": 0.0 | |
| }, | |
| "retrieval-only::splade-only::minilm": { | |
| "wins": 2, | |
| "losses": 0, | |
| "count": 2, | |
| "mean": 0.0 | |
| }, | |
| "retrieval-only::splade-only::mxbai": { | |
| "wins": 2, | |
| "losses": 0, | |
| "count": 2, | |
| "mean": 0.0 | |
| }, | |
| "retrieval-only::hybrid-triple::none": { | |
| "wins": 2, | |
| "losses": 0, | |
| "count": 2, | |
| "mean": 0.0 | |
| }, | |
| "retrieval-only::hybrid-triple::minilm": { | |
| "wins": 2, | |
| "losses": 0, | |
| "count": 2, | |
| "mean": 0.0 | |
| }, | |
| "retrieval-only::hybrid-triple::mxbai": { | |
| "wins": 2, | |
| "losses": 0, | |
| "count": 2, | |
| "mean": 0.0 | |
| }, | |
| "retrieval-only::hybrid-web::none": { | |
| "wins": 2, | |
| "losses": 0, | |
| "count": 2, | |
| "mean": 0.0 | |
| }, | |
| "retrieval-only::hybrid-web::minilm": { | |
| "wins": 2, | |
| "losses": 0, | |
| "count": 2, | |
| "mean": 0.0 | |
| }, | |
| "retrieval-only::hybrid-web::mxbai": { | |
| "wins": 2, | |
| "losses": 0, | |
| "count": 2, | |
| "mean": 0.0 | |
| }, | |
| "devstral-123b::hybrid::none": { | |
| "wins": 0, | |
| "losses": 7, | |
| "count": 7, | |
| "mean": 0.0009117746938775508 | |
| }, | |
| "ministral-14b-reasoning::hybrid::none": { | |
| "wins": 7, | |
| "losses": 0, | |
| "count": 7, | |
| "mean": 0.0 | |
| }, | |
| "magistral-small::hybrid::none": { | |
| "wins": 7, | |
| "losses": 0, | |
| "count": 7, | |
| "mean": 0.0 | |
| }, | |
| "devstral-small-24b::hybrid::none": { | |
| "wins": 7, | |
| "losses": 0, | |
| "count": 7, | |
| "mean": 0.0 | |
| }, | |
| "mistral-small-3.2-24b::hybrid::none": { | |
| "wins": 7, | |
| "losses": 0, | |
| "count": 7, | |
| "mean": 0.0 | |
| }, | |
| "granite-4-30b::hybrid::none": { | |
| "wins": 7, | |
| "losses": 0, | |
| "count": 7, | |
| "mean": 0.0 | |
| } | |
| }, | |
| "par": { | |
| "retrieval-only::bm25-only::none": { | |
| "wins": 0, | |
| "losses": 0, | |
| "count": 2, | |
| "mean": 0.03598809523809535 | |
| }, | |
| "retrieval-only::bm25-only::minilm": { | |
| "wins": 0, | |
| "losses": 0, | |
| "count": 2, | |
| "mean": 0.02162698412698408 | |
| }, | |
| "retrieval-only::bm25-only::mxbai": { | |
| "wins": 0, | |
| "losses": 1, | |
| "count": 2, | |
| "mean": 0.02560119047619044 | |
| }, | |
| "retrieval-only::embedding-bge-m3::none": { | |
| "wins": 0, | |
| "losses": 0, | |
| "count": 2, | |
| "mean": 0.05380158730158733 | |
| }, | |
| "retrieval-only::embedding-bge-m3::minilm": { | |
| "wins": 0, | |
| "losses": 0, | |
| "count": 2, | |
| "mean": 0.022214285714285728 | |
| }, | |
| "retrieval-only::embedding-bge-m3::mxbai": { | |
| "wins": 0, | |
| "losses": 0, | |
| "count": 2, | |
| "mean": 0.05740873015873019 | |
| }, | |
| "retrieval-only::embedding-bge-large::none": { | |
| "wins": 0, | |
| "losses": 0, | |
| "count": 2, | |
| "mean": 0.04175396825396826 | |
| }, | |
| "retrieval-only::embedding-bge-large::minilm": { | |
| "wins": 0, | |
| "losses": 0, | |
| "count": 2, | |
| "mean": 0.01264880952380959 | |
| }, | |
| "retrieval-only::embedding-bge-large::mxbai": { | |
| "wins": 1, | |
| "losses": 0, | |
| "count": 2, | |
| "mean": 0.05530555555555562 | |
| }, | |
| "retrieval-only::hybrid::none": { | |
| "wins": 0, | |
| "losses": 0, | |
| "count": 2, | |
| "mean": 0.04750595238095236 | |
| }, | |
| "retrieval-only::hybrid::minilm": { | |
| "wins": 0, | |
| "losses": 0, | |
| "count": 2, | |
| "mean": 0.017311507936507882 | |
| }, | |
| "retrieval-only::hybrid::mxbai": { | |
| "wins": 0, | |
| "losses": 0, | |
| "count": 2, | |
| "mean": 0.06067063492063492 | |
| }, | |
| "retrieval-only::hybrid-bge-large::none": { | |
| "wins": 0, | |
| "losses": 0, | |
| "count": 2, | |
| "mean": 0.03945833333333328 | |
| }, | |
| "retrieval-only::hybrid-bge-large::minilm": { | |
| "wins": 1, | |
| "losses": 0, | |
| "count": 2, | |
| "mean": 0.03327380952380961 | |
| }, | |
| "retrieval-only::hybrid-bge-large::mxbai": { | |
| "wins": 0, | |
| "losses": 0, | |
| "count": 2, | |
| "mean": 0.06022619047619054 | |
| }, | |
| "retrieval-only::splade-only::none": { | |
| "wins": 0, | |
| "losses": 1, | |
| "count": 2, | |
| "mean": 0.002821428571428624 | |
| }, | |
| "retrieval-only::splade-only::minilm": { | |
| "wins": 0, | |
| "losses": 0, | |
| "count": 2, | |
| "mean": 0.016950396825396827 | |
| }, | |
| "retrieval-only::splade-only::mxbai": { | |
| "wins": 0, | |
| "losses": 0, | |
| "count": 2, | |
| "mean": 0.04500198412698413 | |
| }, | |
| "retrieval-only::hybrid-triple::none": { | |
| "wins": 0, | |
| "losses": 0, | |
| "count": 2, | |
| "mean": 0.050505952380952374 | |
| }, | |
| "retrieval-only::hybrid-triple::minilm": { | |
| "wins": 0, | |
| "losses": 0, | |
| "count": 2, | |
| "mean": 0.015744047619047637 | |
| }, | |
| "retrieval-only::hybrid-triple::mxbai": { | |
| "wins": 0, | |
| "losses": 0, | |
| "count": 2, | |
| "mean": 0.052242063492063576 | |
| }, | |
| "retrieval-only::hybrid-web::none": { | |
| "wins": 0, | |
| "losses": 0, | |
| "count": 2, | |
| "mean": 0.04750595238095236 | |
| }, | |
| "retrieval-only::hybrid-web::minilm": { | |
| "wins": 0, | |
| "losses": 0, | |
| "count": 2, | |
| "mean": 0.017311507936507882 | |
| }, | |
| "retrieval-only::hybrid-web::mxbai": { | |
| "wins": 0, | |
| "losses": 0, | |
| "count": 2, | |
| "mean": 0.06067063492063492 | |
| }, | |
| "devstral-123b::hybrid::none": { | |
| "wins": 5, | |
| "losses": 0, | |
| "count": 7, | |
| "mean": 0.028471088435374135 | |
| }, | |
| "ministral-14b-reasoning::hybrid::none": { | |
| "wins": 5, | |
| "losses": 0, | |
| "count": 7, | |
| "mean": 0.028471088435374135 | |
| }, | |
| "magistral-small::hybrid::none": { | |
| "wins": 5, | |
| "losses": 0, | |
| "count": 7, | |
| "mean": 0.028471088435374135 | |
| }, | |
| "devstral-small-24b::hybrid::none": { | |
| "wins": 5, | |
| "losses": 0, | |
| "count": 7, | |
| "mean": 0.028471088435374135 | |
| }, | |
| "mistral-small-3.2-24b::hybrid::none": { | |
| "wins": 5, | |
| "losses": 0, | |
| "count": 7, | |
| "mean": 0.028471088435374135 | |
| }, | |
| "granite-4-30b::hybrid::none": { | |
| "wins": 5, | |
| "losses": 0, | |
| "count": 7, | |
| "mean": 0.028471088435374135 | |
| } | |
| }, | |
| "shift": { | |
| "retrieval-only::bm25-only::none": { | |
| "wins": 0, | |
| "losses": 1, | |
| "count": 2, | |
| "mean": 0.067654761904762 | |
| }, | |
| "retrieval-only::bm25-only::minilm": { | |
| "wins": 0, | |
| "losses": 0, | |
| "count": 2, | |
| "mean": 0.15779166666666675 | |
| }, | |
| "retrieval-only::bm25-only::mxbai": { | |
| "wins": 0, | |
| "losses": 0, | |
| "count": 2, | |
| "mean": 0.10380753968253964 | |
| }, | |
| "retrieval-only::embedding-bge-m3::none": { | |
| "wins": 0, | |
| "losses": 1, | |
| "count": 2, | |
| "mean": 0.06060119047619052 | |
| }, | |
| "retrieval-only::embedding-bge-m3::minilm": { | |
| "wins": 0, | |
| "losses": 0, | |
| "count": 2, | |
| "mean": 0.14105753968253976 | |
| }, | |
| "retrieval-only::embedding-bge-m3::mxbai": { | |
| "wins": 0, | |
| "losses": 0, | |
| "count": 2, | |
| "mean": 0.11650198412698412 | |
| }, | |
| "retrieval-only::embedding-bge-large::none": { | |
| "wins": 0, | |
| "losses": 0, | |
| "count": 2, | |
| "mean": 0.09962301587301593 | |
| }, | |
| "retrieval-only::embedding-bge-large::minilm": { | |
| "wins": 0, | |
| "losses": 0, | |
| "count": 2, | |
| "mean": 0.1678293650793652 | |
| }, | |
| "retrieval-only::embedding-bge-large::mxbai": { | |
| "wins": 0, | |
| "losses": 0, | |
| "count": 2, | |
| "mean": 0.10281349206349212 | |
| }, | |
| "retrieval-only::hybrid::none": { | |
| "wins": 0, | |
| "losses": 0, | |
| "count": 2, | |
| "mean": 0.17769047619047618 | |
| }, | |
| "retrieval-only::hybrid::minilm": { | |
| "wins": 0, | |
| "losses": 0, | |
| "count": 2, | |
| "mean": 0.15861309523809525 | |
| }, | |
| "retrieval-only::hybrid::mxbai": { | |
| "wins": 0, | |
| "losses": 0, | |
| "count": 2, | |
| "mean": 0.11200992063492066 | |
| }, | |
| "retrieval-only::hybrid-bge-large::none": { | |
| "wins": 0, | |
| "losses": 0, | |
| "count": 2, | |
| "mean": 0.12747023809523814 | |
| }, | |
| "retrieval-only::hybrid-bge-large::minilm": { | |
| "wins": 0, | |
| "losses": 0, | |
| "count": 2, | |
| "mean": 0.16234948979591846 | |
| }, | |
| "retrieval-only::hybrid-bge-large::mxbai": { | |
| "wins": 0, | |
| "losses": 0, | |
| "count": 2, | |
| "mean": 0.09213945578231302 | |
| }, | |
| "retrieval-only::splade-only::none": { | |
| "wins": 0, | |
| "losses": 0, | |
| "count": 2, | |
| "mean": 0.16295238095238102 | |
| }, | |
| "retrieval-only::splade-only::minilm": { | |
| "wins": 1, | |
| "losses": 0, | |
| "count": 2, | |
| "mean": 0.19781944444444455 | |
| }, | |
| "retrieval-only::splade-only::mxbai": { | |
| "wins": 0, | |
| "losses": 0, | |
| "count": 2, | |
| "mean": 0.08979761904761907 | |
| }, | |
| "retrieval-only::hybrid-triple::none": { | |
| "wins": 0, | |
| "losses": 0, | |
| "count": 2, | |
| "mean": 0.16739880952380956 | |
| }, | |
| "retrieval-only::hybrid-triple::minilm": { | |
| "wins": 1, | |
| "losses": 0, | |
| "count": 2, | |
| "mean": 0.18809523809523815 | |
| }, | |
| "retrieval-only::hybrid-triple::mxbai": { | |
| "wins": 0, | |
| "losses": 0, | |
| "count": 2, | |
| "mean": 0.11244841269841283 | |
| }, | |
| "retrieval-only::hybrid-web::none": { | |
| "wins": 0, | |
| "losses": 0, | |
| "count": 2, | |
| "mean": 0.17769047619047618 | |
| }, | |
| "retrieval-only::hybrid-web::minilm": { | |
| "wins": 0, | |
| "losses": 0, | |
| "count": 2, | |
| "mean": 0.15861309523809525 | |
| }, | |
| "retrieval-only::hybrid-web::mxbai": { | |
| "wins": 0, | |
| "losses": 0, | |
| "count": 2, | |
| "mean": 0.11200992063492066 | |
| }, | |
| "devstral-123b::hybrid::none": { | |
| "wins": 5, | |
| "losses": 0, | |
| "count": 7, | |
| "mean": 0.2000913508260447 | |
| }, | |
| "ministral-14b-reasoning::hybrid::none": { | |
| "wins": 5, | |
| "losses": 0, | |
| "count": 7, | |
| "mean": 0.2000913508260447 | |
| }, | |
| "magistral-small::hybrid::none": { | |
| "wins": 5, | |
| "losses": 0, | |
| "count": 7, | |
| "mean": 0.2000913508260447 | |
| }, | |
| "devstral-small-24b::hybrid::none": { | |
| "wins": 5, | |
| "losses": 0, | |
| "count": 7, | |
| "mean": 0.2000913508260447 | |
| }, | |
| "mistral-small-3.2-24b::hybrid::none": { | |
| "wins": 5, | |
| "losses": 0, | |
| "count": 7, | |
| "mean": 0.2000913508260447 | |
| }, | |
| "granite-4-30b::hybrid::none": { | |
| "wins": 5, | |
| "losses": 0, | |
| "count": 7, | |
| "mean": 0.2000913508260447 | |
| } | |
| } | |
| }, | |
| "metrics_def": [ | |
| { | |
| "key": "mrr", | |
| "label": "MRR", | |
| "field": "mrr", | |
| "higher": true, | |
| "fmt": "f3", | |
| "tip": "MRR", | |
| "scale": 1 | |
| }, | |
| { | |
| "key": "h1", | |
| "label": "H@1", | |
| "field": "hit_at_1", | |
| "higher": true, | |
| "fmt": "f2", | |
| "tip": "H@1", | |
| "scale": 1 | |
| }, | |
| { | |
| "key": "h5", | |
| "label": "H@5", | |
| "field": "hit_at_5", | |
| "higher": true, | |
| "fmt": "f2", | |
| "tip": "H@5", | |
| "scale": 1 | |
| }, | |
| { | |
| "key": "ndcg", | |
| "label": "nDCG", | |
| "field": "ndcg_at_10", | |
| "higher": true, | |
| "fmt": "f3", | |
| "tip": "nDCG", | |
| "scale": 1 | |
| }, | |
| { | |
| "key": "p50", | |
| "label": "P50 ms", | |
| "field": "p50_retrieval_latency_ms", | |
| "higher": false, | |
| "fmt": "g0", | |
| "tip": "P50", | |
| "scale": 1 | |
| }, | |
| { | |
| "key": "p95", | |
| "label": "P95 ms", | |
| "field": "p95_retrieval_latency_ms", | |
| "higher": false, | |
| "fmt": "g0", | |
| "tip": "P95", | |
| "scale": 1 | |
| }, | |
| { | |
| "key": "build", | |
| "label": "Build s", | |
| "field": "index_build_seconds", | |
| "higher": false, | |
| "fmt": "g1", | |
| "tip": "build", | |
| "scale": 1 | |
| }, | |
| { | |
| "key": "gen", | |
| "label": "gen", | |
| "field": "generation_acc", | |
| "higher": true, | |
| "fmt": "f3", | |
| "tip": "gen", | |
| "scale": 1 | |
| }, | |
| { | |
| "key": "gen_ms", | |
| "label": "gen ms", | |
| "field": "p50_generation_latency_ms", | |
| "higher": false, | |
| "fmt": "f0", | |
| "tip": "gen ms", | |
| "scale": 1 | |
| }, | |
| { | |
| "key": "tok_s", | |
| "label": "tok/s", | |
| "field": "generation_tokens_per_sec", | |
| "higher": true, | |
| "fmt": "f0", | |
| "tip": "tok/s", | |
| "scale": 1 | |
| }, | |
| { | |
| "key": "gen_cost", | |
| "label": "gen $", | |
| "field": "est_generation_cost_per_query_usd", | |
| "higher": false, | |
| "fmt": "f5", | |
| "tip": "gen $", | |
| "scale": 1 | |
| }, | |
| { | |
| "key": "cost", | |
| "label": "$/kq", | |
| "field": "est_cost_per_query_usd", | |
| "higher": false, | |
| "fmt": "f2", | |
| "tip": "$/kq", | |
| "scale": 1000 | |
| }, | |
| { | |
| "key": "par", | |
| "label": "par", | |
| "field": "paraphrase_drop", | |
| "higher": true, | |
| "fmt": "f3", | |
| "tip": "par", | |
| "scale": 1 | |
| }, | |
| { | |
| "key": "shift", | |
| "label": "shift", | |
| "field": "shift_drop", | |
| "higher": true, | |
| "fmt": "f3", | |
| "tip": "shift", | |
| "scale": 1 | |
| } | |
| ], | |
| "glossary": { | |
| "model": "The generation model in the pipeline. 'retrieval-only' means no LLM generation step was run for this row.", | |
| "retriever": "Retrieval strategy used to fetch candidate documents (e.g. hybrid = BM25 + dense embeddings; bm25-only = lexical; splade-only = learned sparse).", | |
| "reranker": "Cross-encoder reranker applied after retrieval ('none' = no reranking; minilm / mxbai are reranker models).", | |
| "MRR": "Mean Reciprocal Rank — average of 1/rank for the first relevant doc retrieved per query. Range 0–1; higher is better.", | |
| "H@1": "Hit@1 — fraction of queries where the top-1 retrieved doc is relevant. Higher is better.", | |
| "H@5": "Hit@5 — fraction of queries where any of the top-5 retrieved docs is relevant. Higher is better.", | |
| "nDCG": "Normalized Discounted Cumulative Gain @10 — position-weighted relevance of the top-10 results. Higher is better.", | |
| "P50": "Median (50th percentile) per-query retrieval latency in milliseconds. Lower is better.", | |
| "P95": "95th percentile retrieval latency — the tail (worst-case) cost per query, in milliseconds. Lower is better.", | |
| "build": "Index build time in seconds for this benchmark corpus. Lower is better.", | |
| "gen": "Generation score (task-dependent): for multiple-choice benchmarks, the fraction of questions answered correctly given retrieved context; for the language-following benchmark, the fraction of responses written in the user's language. Higher is better.", | |
| "gen ms": "Median per-query generation latency (time-to-first-token + generation) in milliseconds. Lower is better.", | |
| "tok/s": "LLM output tokens generated per second on this benchmark. Higher is better.", | |
| "gen $": "Estimated generation cost per query in USD (token usage × model price). Lower is better.", | |
| "$/kq": "Total estimated cost per 1,000 queries in USD (retrieval + generation). Lower is better.", | |
| "par": "Paraphrase robustness — MRR on paraphrased queries minus MRR on originals. Near zero is expected; large negative means fragile exact-match retrieval.", | |
| "shift": "Held-out (domain-shifted) slice MRR — measures out-of-distribution generalization on an unseen query subset.", | |
| "n": "Number of queries evaluated for this row.", | |
| "verdict": "Whether the winning candidate is statistically distinct from the runner-up. 'winner clear' = winner's Wilson CI lower bound exceeds runner-up's upper bound; 'inconclusive' = confidence intervals overlap (within noise); 'no signal' = too few valid candidates; 'all disqualified' = every candidate failed a hard gate.", | |
| "Pareto": "Pareto front — a pipeline not dominated by any other on (accuracy, latency, build-cost) simultaneously. Marked with ★.", | |
| "DQ": "Disqualified — violated one of the profile's hard gates: DQ:latency (p50 over budget), DQ:cost (cost per query over ceiling), or DQ:accuracy (MRR or generation score below the floor).", | |
| "pick": "The router's single recommended pipeline (model / retriever / reranker) for this client profile on this benchmark.", | |
| "profile": "A client requirements file (not a model) declaring latency / cost / accuracy hard gates plus utility weights used to rank surviving candidates." | |
| }, | |
| "provenance": { | |
| "merged_from": [ | |
| "reports/weekly/2026-06-29/retrieval/raw/run.json", | |
| "reports/weekly/2026-06-29/models/devstral-123b/raw/run.json", | |
| "reports/weekly/2026-06-29/models/ministral-14b-reasoning/raw/run.json", | |
| "reports/weekly/2026-06-29/models/magistral-small/raw/run.json", | |
| "reports/weekly/2026-06-29/models/devstral-small-24b/raw/run.json", | |
| "reports/weekly/2026-06-29/models/mistral-small-3.2-24b/raw/run.json", | |
| "reports/weekly/2026-06-29/models/granite-4-30b/raw/run.json" | |
| ], | |
| "merged_at": "2026-06-29T15:28:40.311551+00:00", | |
| "weekly": true | |
| }, | |
| "n_candidates": 30, | |
| "n_queries": 100 | |
| } |