{ "schema_version": 1, "suite_questions": 120, "selected_records": 6000, "public_answer_records": 900, "configurations": [ { "id": "gemma4-e4b/off", "suite_id": "tiny", "model": "Gemma 4 E4B", "mode": "Thinking off", "model_key": "google/gemma-4-e4b", "selected_variant": "google/gemma-4-e4b@q4_k_m", "quantization": "Q4_K_M", "adapter": "native-reasoning-v1", "generation": { "context_length": 32768, "max_output_tokens": 8192, "temperature": 1.0, "top_p": 0.95, "top_k": 64, "min_p": 0, "repeat_penalty": 1 }, "thinking": "off", "reasoning_effort": "not separately specified", "generation_seed": null, "generation_seed_status": "not set; native API does not expose it", "runtime_version": "not verified", "exact_gpu_placement": "not verified", "source_id": "tiny/gemma-off", "batch": "tiny", "total_parameters": 7996156490, "active_parameters": null, "effective_parameters": 4500000000, "size_class": "medium", "parameter_count_basis": "Hugging Face safetensors.total for the pinned official artifact", "parameter_source_url": "https://huggingface.co/api/models/google/gemma-4-E4B-it/revision/ee0ef6023621cff504d758262d4e04895a5af4a2", "parameter_card_url": "https://huggingface.co/google/gemma-4-E4B-it/blob/ee0ef6023621cff504d758262d4e04895a5af4a2/README.md", "parameter_card_sha256": "b21e4f69614ccd77baa2f3797d05311040dee07b989cb9f0d25111aa4b605b2c", "parameter_checked_date": "2026-10-21", "parameter_notes": "Effective count is 4.5B; it is not an active-parameter measurement. Total includes embeddings and modality encoders." }, { "id": "gemma4-e4b/on", "suite_id": "tiny", "model": "Gemma 4 E4B", "mode": "Thinking on", "model_key": "google/gemma-4-e4b", "selected_variant": "google/gemma-4-e4b@q4_k_m", "quantization": "Q4_K_M", "adapter": "native-reasoning-v1", "generation": { "context_length": 32768, "max_output_tokens": 8192, "temperature": 1.0, "top_p": 0.95, "top_k": 64, "min_p": 0, "repeat_penalty": 1 }, "thinking": "on", "reasoning_effort": "not separately specified", "generation_seed": null, "generation_seed_status": "not set; native API does not expose it", "runtime_version": "not verified", "exact_gpu_placement": "not verified", "source_id": "tiny/gemma-on", "batch": "tiny", "total_parameters": 7996156490, "active_parameters": null, "effective_parameters": 4500000000, "size_class": "medium", "parameter_count_basis": "Hugging Face safetensors.total for the pinned official artifact", "parameter_source_url": "https://huggingface.co/api/models/google/gemma-4-E4B-it/revision/ee0ef6023621cff504d758262d4e04895a5af4a2", "parameter_card_url": "https://huggingface.co/google/gemma-4-E4B-it/blob/ee0ef6023621cff504d758262d4e04895a5af4a2/README.md", "parameter_card_sha256": "b21e4f69614ccd77baa2f3797d05311040dee07b989cb9f0d25111aa4b605b2c", "parameter_checked_date": "2026-10-21", "parameter_notes": "Effective count is 4.5B; it is not an active-parameter measurement. Total includes embeddings and modality encoders." }, { "id": "lfm2.5-2.6b/native", "suite_id": "tiny", "model": "LFM2.5 2.6B", "mode": "Native thinking", "model_key": "lfm2.5-2.6b", "selected_variant": "not verified", "quantization": "Q4_K_M", "adapter": "native-default-thinking-v1", "generation": { "context_length": 32768, "max_output_tokens": 8192, "temperature": 0.1, "top_p": 1.0, "top_k": 50, "min_p": 0, "repeat_penalty": 1.1 }, "thinking": "on", "reasoning_effort": "not separately specified", "generation_seed": null, "generation_seed_status": "not set; native API does not expose it", "runtime_version": "not verified", "exact_gpu_placement": "not verified", "source_id": "tiny/lfm-on", "batch": "tiny", "total_parameters": 2697198592, "active_parameters": null, "effective_parameters": null, "size_class": "small", "parameter_count_basis": "Hugging Face safetensors.total for the pinned official artifact", "parameter_source_url": "https://huggingface.co/api/models/LiquidAI/LFM2.5-2.6B/revision/654f9463ce32b05d0429d76fe1f580b27d4c1ac0", "parameter_card_url": "https://huggingface.co/LiquidAI/LFM2.5-2.6B/blob/654f9463ce32b05d0429d76fe1f580b27d4c1ac0/README.md", "parameter_card_sha256": "003ceb63ec63a5b06b17f5e7b7a7d907695f037ad18720cabbcf8d8d68813f24", "parameter_checked_date": "2026-10-21", "parameter_notes": "Card rounds the total to 2.69B. Hybrid dense model; no separate active count is published." }, { "id": "gemma4-e2b/off", "suite_id": "tiny", "model": "Gemma 4 E2B", "mode": "Thinking off", "model_key": "google/gemma-4-e2b", "selected_variant": "google/gemma-4-e2b@q4_k_m", "quantization": "Q4_K_M", "adapter": "native-reasoning-v1", "generation": { "context_length": 65536, "max_output_tokens": 32768, "temperature": 1.0, "top_p": 0.95, "top_k": 64, "min_p": 0, "repeat_penalty": 1 }, "thinking": "off", "reasoning_effort": "not separately specified", "generation_seed": null, "generation_seed_status": "not set; native API does not expose it", "runtime_version": "not verified", "exact_gpu_placement": "not verified", "source_id": "tiny/gemmae2-off", "batch": "tiny", "total_parameters": 5123178051, "active_parameters": null, "effective_parameters": 2300000000, "size_class": "small", "parameter_count_basis": "Hugging Face safetensors.total for the pinned official artifact", "parameter_source_url": "https://huggingface.co/api/models/google/gemma-4-E2B-it/revision/3e22461f65e89153144f8adb70e3b8c2cc9845a7", "parameter_card_url": "https://huggingface.co/google/gemma-4-E2B-it/blob/3e22461f65e89153144f8adb70e3b8c2cc9845a7/README.md", "parameter_card_sha256": "3e0608a6be80e3eb040a5aa6c76707809503deb85be6360184a118fb6156526c", "parameter_checked_date": "2026-10-21", "parameter_notes": "Effective count is 2.3B; it is not an active-parameter measurement. Total includes embeddings and modality encoders." }, { "id": "gemma4-e2b/on", "suite_id": "tiny", "model": "Gemma 4 E2B", "mode": "Thinking on", "model_key": "google/gemma-4-e2b", "selected_variant": "google/gemma-4-e2b@q4_k_m", "quantization": "Q4_K_M", "adapter": "native-reasoning-v1", "generation": { "context_length": 65536, "max_output_tokens": 32768, "temperature": 1.0, "top_p": 0.95, "top_k": 64, "min_p": 0, "repeat_penalty": 1 }, "thinking": "on", "reasoning_effort": "not separately specified", "generation_seed": null, "generation_seed_status": "not set; native API does not expose it", "runtime_version": "not verified", "exact_gpu_placement": "not verified", "source_id": "tiny/gemmae2-on", "batch": "tiny", "total_parameters": 5123178051, "active_parameters": null, "effective_parameters": 2300000000, "size_class": "small", "parameter_count_basis": "Hugging Face safetensors.total for the pinned official artifact", "parameter_source_url": "https://huggingface.co/api/models/google/gemma-4-E2B-it/revision/3e22461f65e89153144f8adb70e3b8c2cc9845a7", "parameter_card_url": "https://huggingface.co/google/gemma-4-E2B-it/blob/3e22461f65e89153144f8adb70e3b8c2cc9845a7/README.md", "parameter_card_sha256": "3e0608a6be80e3eb040a5aa6c76707809503deb85be6360184a118fb6156526c", "parameter_checked_date": "2026-10-21", "parameter_notes": "Effective count is 2.3B; it is not an active-parameter measurement. Total includes embeddings and modality encoders." }, { "id": "k2-horizon-3.7b/low", "suite_id": "tiny", "model": "K2 Horizon 3.7B", "mode": "Low reasoning effort", "model_key": "k2-horizon-3.7b", "selected_variant": "not verified", "quantization": "Q4_K_M", "adapter": "k2-creator-low-text-v1", "generation": { "context_length": 65536, "max_output_tokens": 32768, "temperature": 1.0, "top_p": 0.95, "top_k": 0, "min_p": 0, "repeat_penalty": 1 }, "thinking": "on", "reasoning_effort": "not separately specified", "generation_seed": null, "generation_seed_status": "not set; native API does not expose it", "runtime_version": "not verified", "exact_gpu_placement": "not verified", "source_id": "tiny/k2-low", "batch": "tiny", "total_parameters": 5058255360, "active_parameters": 3700000000, "effective_parameters": null, "size_class": "small", "parameter_count_basis": "Hugging Face safetensors.total for the pinned official artifact", "parameter_source_url": "https://huggingface.co/api/models/IFM/K2-Horizon-3.7B/revision/9d280ff742235796925c9fe3cf243dc1beea5c08", "parameter_card_url": "https://huggingface.co/IFM/K2-Horizon-3.7B/blob/9d280ff742235796925c9fe3cf243dc1beea5c08/README.md", "parameter_card_sha256": "e0868f280e0509f75a208ca54a328808a8652d7466f5f7b76b5e8d561e9a0c3d", "parameter_checked_date": "2026-10-21", "parameter_notes": "Card reports a 3.7B core and labels 3.7B activated parameters. Artifact total also counts embeddings/output weights; active is the card convention." }, { "id": "granite4.2-3b/off", "suite_id": "tiny", "model": "Granite 4.2 3B", "mode": "Thinking off", "model_key": "granite-4.2-3b", "selected_variant": "not verified", "quantization": "Q4_K_M", "adapter": "granite-creator-text-v1", "generation": { "context_length": 65536, "max_output_tokens": 32768, "temperature": 1.0, "top_p": 0.95, "top_k": 0, "min_p": 0, "repeat_penalty": 1 }, "thinking": "off", "reasoning_effort": "not separately specified", "generation_seed": null, "generation_seed_status": "not set; native API does not expose it", "runtime_version": "not verified", "exact_gpu_placement": "not verified", "source_id": "tiny/granite-off", "batch": "tiny", "total_parameters": 3659737600, "active_parameters": null, "effective_parameters": null, "size_class": "small", "parameter_count_basis": "Hugging Face safetensors.total for the pinned official artifact", "parameter_source_url": "https://huggingface.co/api/models/ibm-granite/granite-4.2-3b/revision/4e9c275e232f10211ec88644d106a1f6a1b60e06", "parameter_card_url": "https://huggingface.co/ibm-granite/granite-4.2-3b/blob/4e9c275e232f10211ec88644d106a1f6a1b60e06/README.md", "parameter_card_sha256": "8f22ddce440f0c15ecb58fc9f8501b882baf5f2b7ab0d13872fe77d88c1ad792", "parameter_checked_date": "2026-10-21", "parameter_notes": "Dense model; artifact total includes separate input/output embeddings. No separate active count is published." }, { "id": "granite4.2-3b/on", "suite_id": "tiny", "model": "Granite 4.2 3B", "mode": "Thinking on", "model_key": "granite-4.2-3b", "selected_variant": "not verified", "quantization": "Q4_K_M", "adapter": "granite-creator-text-v1", "generation": { "context_length": 65536, "max_output_tokens": 32768, "temperature": 1.0, "top_p": 0.95, "top_k": 0, "min_p": 0, "repeat_penalty": 1 }, "thinking": "on", "reasoning_effort": "not separately specified", "generation_seed": null, "generation_seed_status": "not set; native API does not expose it", "runtime_version": "not verified", "exact_gpu_placement": "not verified", "source_id": "tiny/granite-on", "batch": "tiny", "total_parameters": 3659737600, "active_parameters": null, "effective_parameters": null, "size_class": "small", "parameter_count_basis": "Hugging Face safetensors.total for the pinned official artifact", "parameter_source_url": "https://huggingface.co/api/models/ibm-granite/granite-4.2-3b/revision/4e9c275e232f10211ec88644d106a1f6a1b60e06", "parameter_card_url": "https://huggingface.co/ibm-granite/granite-4.2-3b/blob/4e9c275e232f10211ec88644d106a1f6a1b60e06/README.md", "parameter_card_sha256": "8f22ddce440f0c15ecb58fc9f8501b882baf5f2b7ab0d13872fe77d88c1ad792", "parameter_checked_date": "2026-10-21", "parameter_notes": "Dense model; artifact total includes separate input/output embeddings. No separate active count is published." }, { "id": "qwen3.5-4b/off", "suite_id": "medium", "model": "Qwen3.5 4B", "mode": "Thinking off", "model_key": "qwen3.5-4b", "selected_variant": "not verified", "quantization": "Q4_K_M", "adapter": "qwen35-official-text-v1", "generation": { "context_length": 65536, "max_output_tokens": 32768, "temperature": 0.7, "top_p": 0.8, "top_k": 20, "min_p": 0, "repeat_penalty": 1, "presence_penalty": 1.5 }, "thinking": "off", "reasoning_effort": "not separately specified", "generation_seed": null, "generation_seed_status": "not set; native API does not expose it", "runtime_version": "not verified", "exact_gpu_placement": "not verified", "source_id": "medium/qwen4-off", "batch": "medium", "total_parameters": 4659865088, "active_parameters": null, "effective_parameters": null, "size_class": "small", "parameter_count_basis": "Hugging Face safetensors.total for the pinned official artifact", "parameter_source_url": "https://huggingface.co/api/models/Qwen/Qwen3.5-4B/revision/851bf6e806efd8d0a36b00ddf55e13ccb7b8cd0a", "parameter_card_url": "https://huggingface.co/Qwen/Qwen3.5-4B/blob/851bf6e806efd8d0a36b00ddf55e13ccb7b8cd0a/README.md", "parameter_card_sha256": "1406be1b6b8fd8a6545870da516912804756593628a1d0fb0a7965211e82a7bb", "parameter_checked_date": "2026-10-21", "parameter_notes": "Dense language model is labelled 4B; artifact total includes the vision components. No separate active count is published." }, { "id": "qwen3.5-4b/on", "suite_id": "medium", "model": "Qwen3.5 4B", "mode": "Thinking on", "model_key": "qwen3.5-4b", "selected_variant": "not verified", "quantization": "Q4_K_M", "adapter": "qwen35-official-text-v1", "generation": { "context_length": 65536, "max_output_tokens": 32768, "temperature": 1.0, "top_p": 0.95, "top_k": 20, "min_p": 0, "repeat_penalty": 1, "presence_penalty": 1.5 }, "thinking": "on", "reasoning_effort": "not separately specified", "generation_seed": null, "generation_seed_status": "not set; native API does not expose it", "runtime_version": "not verified", "exact_gpu_placement": "not verified", "source_id": "medium/qwen4-on", "batch": "medium", "total_parameters": 4659865088, "active_parameters": null, "effective_parameters": null, "size_class": "small", "parameter_count_basis": "Hugging Face safetensors.total for the pinned official artifact", "parameter_source_url": "https://huggingface.co/api/models/Qwen/Qwen3.5-4B/revision/851bf6e806efd8d0a36b00ddf55e13ccb7b8cd0a", "parameter_card_url": "https://huggingface.co/Qwen/Qwen3.5-4B/blob/851bf6e806efd8d0a36b00ddf55e13ccb7b8cd0a/README.md", "parameter_card_sha256": "1406be1b6b8fd8a6545870da516912804756593628a1d0fb0a7965211e82a7bb", "parameter_checked_date": "2026-10-21", "parameter_notes": "Dense language model is labelled 4B; artifact total includes the vision components. No separate active count is published." }, { "id": "granite4.2-8b/on", "suite_id": "medium", "model": "Granite 4.2 8B", "mode": "Thinking on", "model_key": "granite-4.2-8b", "selected_variant": "not verified", "quantization": "Q4_K_M", "adapter": "granite-creator-text-v1", "generation": { "context_length": 65536, "max_output_tokens": 32768, "temperature": 1.0, "top_p": 0.95, "top_k": 0, "min_p": 0, "repeat_penalty": 1 }, "thinking": "on", "reasoning_effort": "not separately specified", "generation_seed": null, "generation_seed_status": "not set; native API does not expose it", "runtime_version": "not verified", "exact_gpu_placement": "not verified", "source_id": "medium/granite-on", "batch": "medium", "total_parameters": 8791592960, "active_parameters": null, "effective_parameters": null, "size_class": "medium", "parameter_count_basis": "Hugging Face safetensors.total for the pinned official artifact", "parameter_source_url": "https://huggingface.co/api/models/ibm-granite/granite-4.2-8b/revision/a7042144125ce5ae64f89610e8289647fc000b9a", "parameter_card_url": "https://huggingface.co/ibm-granite/granite-4.2-8b/blob/a7042144125ce5ae64f89610e8289647fc000b9a/README.md", "parameter_card_sha256": "01396b1a8596586986fea22cfaa86d0255d19113d71fe09e0b69cc2bfa4307da", "parameter_checked_date": "2026-10-21", "parameter_notes": "Dense model; artifact total includes separate input/output embeddings. No separate active count is published." }, { "id": "granite4.2-8b/off", "suite_id": "medium", "model": "Granite 4.2 8B", "mode": "Thinking off", "model_key": "granite-4.2-8b", "selected_variant": "not verified", "quantization": "Q4_K_M", "adapter": "granite-creator-text-v1", "generation": { "context_length": 65536, "max_output_tokens": 32768, "temperature": 1.0, "top_p": 0.95, "top_k": 0, "min_p": 0, "repeat_penalty": 1 }, "thinking": "off", "reasoning_effort": "not separately specified", "generation_seed": null, "generation_seed_status": "not set; native API does not expose it", "runtime_version": "not verified", "exact_gpu_placement": "not verified", "source_id": "medium/granite-off", "batch": "medium", "total_parameters": 8791592960, "active_parameters": null, "effective_parameters": null, "size_class": "medium", "parameter_count_basis": "Hugging Face safetensors.total for the pinned official artifact", "parameter_source_url": "https://huggingface.co/api/models/ibm-granite/granite-4.2-8b/revision/a7042144125ce5ae64f89610e8289647fc000b9a", "parameter_card_url": "https://huggingface.co/ibm-granite/granite-4.2-8b/blob/a7042144125ce5ae64f89610e8289647fc000b9a/README.md", "parameter_card_sha256": "01396b1a8596586986fea22cfaa86d0255d19113d71fe09e0b69cc2bfa4307da", "parameter_checked_date": "2026-10-21", "parameter_notes": "Dense model; artifact total includes separate input/output embeddings. No separate active count is published." }, { "id": "lfm2.5-8b-a1b/native", "suite_id": "medium", "model": "LFM2.5-8B-A1B", "mode": "Native thinking", "model_key": "lfm2.5-8b-a1b", "selected_variant": "not verified", "quantization": "Q4_K_M", "adapter": "native-default-thinking-v1", "generation": { "context_length": 65536, "max_output_tokens": 32768, "temperature": 0.2, "top_p": 1.0, "top_k": 80, "min_p": 0, "repeat_penalty": 1.05 }, "thinking": "on", "reasoning_effort": "not separately specified", "generation_seed": null, "generation_seed_status": "not set; native API does not expose it", "runtime_version": "not verified", "exact_gpu_placement": "not verified", "source_id": "medium/lfm8-on", "batch": "medium", "total_parameters": 8467856832, "active_parameters": 1500000000, "effective_parameters": null, "size_class": "medium", "parameter_count_basis": "Hugging Face safetensors.total for the pinned official artifact", "parameter_source_url": "https://huggingface.co/api/models/LiquidAI/LFM2.5-8B-A1B/revision/5dd22602c2e9f6a097b1de4c4efe0658b605015c", "parameter_card_url": "https://huggingface.co/LiquidAI/LFM2.5-8B-A1B/blob/5dd22602c2e9f6a097b1de4c4efe0658b605015c/README.md", "parameter_card_sha256": "eb40f5396ef5200a14240f8c0523b269bcc114f40b153ac2fa9017dad391e59b", "parameter_checked_date": "2026-10-21", "parameter_notes": "Card reports 8.3B total and 1.5B active; active count is not the A1B label rounded to an integer." }, { "id": "qwen3.5-9b/off", "suite_id": "medium", "model": "Qwen3.5 9B", "mode": "Thinking off", "model_key": "qwen/qwen3.5-9b", "selected_variant": "qwen/qwen3.5-9b@q4_k_m", "quantization": "Q4_K_M", "adapter": "qwen35-official-text-v1", "generation": { "context_length": 65536, "max_output_tokens": 32768, "temperature": 0.7, "top_p": 0.8, "top_k": 20, "min_p": 0, "repeat_penalty": 1, "presence_penalty": 1.5 }, "thinking": "off", "reasoning_effort": "not separately specified", "generation_seed": null, "generation_seed_status": "not set; native API does not expose it", "runtime_version": "not verified", "exact_gpu_placement": "not verified", "source_id": "medium/qwen9-off", "batch": "medium", "total_parameters": 9653104368, "active_parameters": null, "effective_parameters": null, "size_class": "medium", "parameter_count_basis": "Hugging Face safetensors.total for the pinned official artifact", "parameter_source_url": "https://huggingface.co/api/models/Qwen/Qwen3.5-9B/revision/c202236235762e1c871ad0ccb60c8ee5ba337b9a", "parameter_card_url": "https://huggingface.co/Qwen/Qwen3.5-9B/blob/c202236235762e1c871ad0ccb60c8ee5ba337b9a/README.md", "parameter_card_sha256": "c5f5a8c2dddab69cfbf05279235aa5fddb137939a06539c4c7637aa900fef6d0", "parameter_checked_date": "2026-10-21", "parameter_notes": "Dense language model is labelled 9B; artifact total includes the vision components. No separate active count is published." }, { "id": "qwen3.5-9b/on", "suite_id": "medium", "model": "Qwen3.5 9B", "mode": "Thinking on", "model_key": "qwen/qwen3.5-9b", "selected_variant": "qwen/qwen3.5-9b@q4_k_m", "quantization": "Q4_K_M", "adapter": "qwen35-official-text-v1", "generation": { "context_length": 65536, "max_output_tokens": 32768, "temperature": 1.0, "top_p": 0.95, "top_k": 20, "min_p": 0, "repeat_penalty": 1, "presence_penalty": 1.5 }, "thinking": "on", "reasoning_effort": "not separately specified", "generation_seed": null, "generation_seed_status": "not set; native API does not expose it", "runtime_version": "not verified", "exact_gpu_placement": "not verified", "source_id": "medium/qwen9-on", "batch": "medium", "total_parameters": 9653104368, "active_parameters": null, "effective_parameters": null, "size_class": "medium", "parameter_count_basis": "Hugging Face safetensors.total for the pinned official artifact", "parameter_source_url": "https://huggingface.co/api/models/Qwen/Qwen3.5-9B/revision/c202236235762e1c871ad0ccb60c8ee5ba337b9a", "parameter_card_url": "https://huggingface.co/Qwen/Qwen3.5-9B/blob/c202236235762e1c871ad0ccb60c8ee5ba337b9a/README.md", "parameter_card_sha256": "c5f5a8c2dddab69cfbf05279235aa5fddb137939a06539c4c7637aa900fef6d0", "parameter_checked_date": "2026-10-21", "parameter_notes": "Dense language model is labelled 9B; artifact total includes the vision components. No separate active count is published." }, { "id": "granite4.2-30b/on", "suite_id": "large", "model": "Granite 4.2 30B", "mode": "Thinking on", "model_key": "granite-4.2-30b", "selected_variant": "not verified", "quantization": "Q4_K_M", "adapter": "granite-creator-text-v1", "generation": { "context_length": 65536, "max_output_tokens": 32768, "temperature": 1.0, "top_p": 0.95, "top_k": 0, "min_p": 0, "repeat_penalty": 1.0 }, "thinking": "on", "reasoning_effort": "not separately specified", "generation_seed": null, "generation_seed_status": "not set; native API does not expose it", "runtime_version": "not verified", "exact_gpu_placement": "not verified", "source_id": "large/granite-on", "batch": "large", "total_parameters": 29276770304, "active_parameters": null, "effective_parameters": null, "size_class": "large", "parameter_count_basis": "Hugging Face safetensors.total for the pinned official artifact", "parameter_source_url": "https://huggingface.co/api/models/ibm-granite/granite-4.2-30b/revision/9e668ce1c538387ef24d3644e9b0606647762636", "parameter_card_url": "https://huggingface.co/ibm-granite/granite-4.2-30b/blob/9e668ce1c538387ef24d3644e9b0606647762636/README.md", "parameter_card_sha256": "10d0e2b083c888814dc92a6308ca34d1c6fe01f344b54ff54905a8cbe5a80b06", "parameter_checked_date": "2026-10-21", "parameter_notes": "Dense model; artifact total includes separate input/output embeddings. No separate active count is published." }, { "id": "granite4.2-30b/off", "suite_id": "large", "model": "Granite 4.2 30B", "mode": "Thinking off", "model_key": "granite-4.2-30b", "selected_variant": "not verified", "quantization": "Q4_K_M", "adapter": "granite-creator-text-v1", "generation": { "context_length": 65536, "max_output_tokens": 32768, "temperature": 1.0, "top_p": 0.95, "top_k": 0, "min_p": 0, "repeat_penalty": 1.0 }, "thinking": "off", "reasoning_effort": "not separately specified", "generation_seed": null, "generation_seed_status": "not set; native API does not expose it", "runtime_version": "not verified", "exact_gpu_placement": "not verified", "source_id": "large/granite-off", "batch": "large", "total_parameters": 29276770304, "active_parameters": null, "effective_parameters": null, "size_class": "large", "parameter_count_basis": "Hugging Face safetensors.total for the pinned official artifact", "parameter_source_url": "https://huggingface.co/api/models/ibm-granite/granite-4.2-30b/revision/9e668ce1c538387ef24d3644e9b0606647762636", "parameter_card_url": "https://huggingface.co/ibm-granite/granite-4.2-30b/blob/9e668ce1c538387ef24d3644e9b0606647762636/README.md", "parameter_card_sha256": "10d0e2b083c888814dc92a6308ca34d1c6fe01f344b54ff54905a8cbe5a80b06", "parameter_checked_date": "2026-10-21", "parameter_notes": "Dense model; artifact total includes separate input/output embeddings. No separate active count is published." }, { "id": "lfm2.5-24b-a2b/native", "suite_id": "large", "model": "LFM2.5-24B-A2B", "mode": "Native thinking", "model_key": "lfm2.5-24b-a2b", "selected_variant": "not verified", "quantization": "Q4_K_M", "adapter": "native-default-thinking-v1", "generation": { "context_length": 65536, "max_output_tokens": 32768, "temperature": 0.1, "top_p": 1.0, "top_k": 50, "min_p": 0, "repeat_penalty": 1.05 }, "thinking": "on", "reasoning_effort": "not separately specified", "generation_seed": null, "generation_seed_status": "not set; native API does not expose it", "runtime_version": "not verified", "exact_gpu_placement": "not verified", "source_id": "large/lfm24-on", "batch": "large", "total_parameters": 23971585280, "active_parameters": 2450000000, "effective_parameters": null, "size_class": "large", "parameter_count_basis": "Hugging Face safetensors.total for the pinned official artifact", "parameter_source_url": "https://huggingface.co/api/models/LiquidAI/LFM2.5-24B-A2B/revision/3fcfdbfa1a1b6d8166846a6323e69f1e1254440d", "parameter_card_url": "https://huggingface.co/LiquidAI/LFM2.5-24B-A2B/blob/3fcfdbfa1a1b6d8166846a6323e69f1e1254440d/README.md", "parameter_card_sha256": "e5e174cc7f03caf4621e2372395ea558765bf5cddbcf052a6e591090b491af69", "parameter_checked_date": "2026-10-21", "parameter_notes": "Card reports 24.0B total and 2.45B active." }, { "id": "qwen3.8-27b/off", "suite_id": "large", "model": "Qwen3.8 27B", "mode": "Thinking off", "model_key": "qwen/qwen3.8-27b", "selected_variant": "qwen/qwen3.8-27b@q4_k_m", "quantization": "Q4_K_M", "adapter": "qwen38-creator-text-v1", "generation": { "context_length": 65536, "max_output_tokens": 32768, "temperature": 0.7, "top_p": 0.8, "top_k": 20, "min_p": 0, "repeat_penalty": 1.0, "presence_penalty": 1.5 }, "thinking": "off", "reasoning_effort": "off", "generation_seed": null, "generation_seed_status": "not set; native API does not expose it", "runtime_version": "not verified", "exact_gpu_placement": "not verified", "source_id": "large/qwen27-off", "batch": "large", "total_parameters": 27781427952, "active_parameters": null, "effective_parameters": null, "size_class": "large", "parameter_count_basis": "Hugging Face safetensors.total for the pinned official artifact", "parameter_source_url": "https://huggingface.co/api/models/Qwen/Qwen3.8-27B/revision/1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0", "parameter_card_url": "https://huggingface.co/Qwen/Qwen3.8-27B/blob/1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0/README.md", "parameter_card_sha256": "57e4bdb258ee1a7d2635c5174ebd4e56abe392505cdb5f8bbb356b0dc4293641", "parameter_checked_date": "2026-10-21", "parameter_notes": "Dense language model is labelled 27B; artifact total includes the vision components. No separate active count is published." }, { "id": "qwen3.8-27b/on", "suite_id": "large", "model": "Qwen3.8 27B", "mode": "Low thinking effort", "model_key": "qwen/qwen3.8-27b", "selected_variant": "qwen/qwen3.8-27b@q4_k_m", "quantization": "Q4_K_M", "adapter": "qwen38-creator-text-v1", "generation": { "context_length": 65536, "max_output_tokens": 32768, "temperature": 1.0, "top_p": 0.95, "top_k": 20, "min_p": 0, "repeat_penalty": 1.0, "presence_penalty": 0.0 }, "thinking": "on", "reasoning_effort": "low", "generation_seed": null, "generation_seed_status": "not set; native API does not expose it", "runtime_version": "not verified", "exact_gpu_placement": "not verified", "source_id": "large/qwen27-on", "batch": "large", "total_parameters": 27781427952, "active_parameters": null, "effective_parameters": null, "size_class": "large", "parameter_count_basis": "Hugging Face safetensors.total for the pinned official artifact", "parameter_source_url": "https://huggingface.co/api/models/Qwen/Qwen3.8-27B/revision/1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0", "parameter_card_url": "https://huggingface.co/Qwen/Qwen3.8-27B/blob/1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0/README.md", "parameter_card_sha256": "57e4bdb258ee1a7d2635c5174ebd4e56abe392505cdb5f8bbb356b0dc4293641", "parameter_checked_date": "2026-10-21", "parameter_notes": "Dense language model is labelled 27B; artifact total includes the vision components. No separate active count is published." }, { "id": "gemma4-26b-a4b/off", "suite_id": "large", "model": "Gemma 4 26B-A4B", "mode": "Thinking off", "model_key": "google/gemma-4-26b-a4b", "selected_variant": "google/gemma-4-26b-a4b@q4_k_m", "quantization": "Q4_K_M", "adapter": "native-reasoning-v1", "generation": { "context_length": 65536, "max_output_tokens": 32768, "temperature": 1.0, "top_p": 0.95, "top_k": 64, "min_p": 0, "repeat_penalty": 1.0 }, "thinking": "off", "reasoning_effort": "not separately specified", "generation_seed": null, "generation_seed_status": "not set; native API does not expose it", "runtime_version": "not verified", "exact_gpu_placement": "not verified", "source_id": "large/gemma26-off", "batch": "large", "total_parameters": 25805936206, "active_parameters": 3800000000, "effective_parameters": null, "size_class": "large", "parameter_count_basis": "Hugging Face safetensors.total for the pinned official artifact", "parameter_source_url": "https://huggingface.co/api/models/google/gemma-4-26B-A4B-it/revision/4d7ae4984b7db7de8f8457170b3f1a419ee76d52", "parameter_card_url": "https://huggingface.co/google/gemma-4-26B-A4B-it/blob/4d7ae4984b7db7de8f8457170b3f1a419ee76d52/README.md", "parameter_card_sha256": "4f463225c4fe49ec120115b0f9bb6713ff1556293a909903ffdae3df7bf974da", "parameter_checked_date": "2026-10-21", "parameter_notes": "Card gives 25.2B language-model total and 3.8B active; artifact total also includes the vision encoder." }, { "id": "gemma4-26b-a4b/on", "suite_id": "large", "model": "Gemma 4 26B-A4B", "mode": "Thinking on", "model_key": "google/gemma-4-26b-a4b", "selected_variant": "google/gemma-4-26b-a4b@q4_k_m", "quantization": "Q4_K_M", "adapter": "native-reasoning-v1", "generation": { "context_length": 65536, "max_output_tokens": 32768, "temperature": 1.0, "top_p": 0.95, "top_k": 64, "min_p": 0, "repeat_penalty": 1.0 }, "thinking": "on", "reasoning_effort": "not separately specified", "generation_seed": null, "generation_seed_status": "not set; native API does not expose it", "runtime_version": "not verified", "exact_gpu_placement": "not verified", "source_id": "large/gemma26-on", "batch": "large", "total_parameters": 25805936206, "active_parameters": 3800000000, "effective_parameters": null, "size_class": "large", "parameter_count_basis": "Hugging Face safetensors.total for the pinned official artifact", "parameter_source_url": "https://huggingface.co/api/models/google/gemma-4-26B-A4B-it/revision/4d7ae4984b7db7de8f8457170b3f1a419ee76d52", "parameter_card_url": "https://huggingface.co/google/gemma-4-26B-A4B-it/blob/4d7ae4984b7db7de8f8457170b3f1a419ee76d52/README.md", "parameter_card_sha256": "4f463225c4fe49ec120115b0f9bb6713ff1556293a909903ffdae3df7bf974da", "parameter_checked_date": "2026-10-21", "parameter_notes": "Card gives 25.2B language-model total and 3.8B active; artifact total also includes the vision encoder." }, { "id": "glm4.7-flash/off", "suite_id": "large", "model": "GLM-4.7 Flash", "mode": "Thinking off", "model_key": "zai-org/glm-4.7-flash", "selected_variant": "zai-org/glm-4.7-flash@q4_k_m", "quantization": "Q4_K_M", "adapter": "glm47-creator-text-v1", "generation": { "context_length": 65536, "max_output_tokens": 32768, "temperature": 1.0, "top_p": 0.95, "top_k": 0, "min_p": 0, "repeat_penalty": 1.0, "presence_penalty": 0.0 }, "thinking": "off", "reasoning_effort": "not separately specified", "generation_seed": null, "generation_seed_status": "not set; native API does not expose it", "runtime_version": "not verified", "exact_gpu_placement": "not verified", "source_id": "large/glm30-off", "batch": "large", "total_parameters": 31221488576, "active_parameters": 3000000000, "effective_parameters": null, "size_class": "large", "parameter_count_basis": "Hugging Face safetensors.total for the pinned official artifact", "parameter_source_url": "https://huggingface.co/api/models/zai-org/GLM-4.7-Flash/revision/7dd20894a642a0aa287e9827cb1a1f7f91386b67", "parameter_card_url": "https://huggingface.co/zai-org/GLM-4.7-Flash/blob/7dd20894a642a0aa287e9827cb1a1f7f91386b67/README.md", "parameter_card_sha256": "7bfcbced42741ceb3fff8809f8d6c84ada9696850e257c4e53e6c2fe9ef4f9bf", "parameter_checked_date": "2026-10-21", "parameter_notes": "Card labels the model 30B-A3B; artifact total is about 31.22B." }, { "id": "glm4.7-flash/on", "suite_id": "large", "model": "GLM-4.7 Flash", "mode": "Thinking on", "model_key": "zai-org/glm-4.7-flash", "selected_variant": "zai-org/glm-4.7-flash@q4_k_m", "quantization": "Q4_K_M", "adapter": "glm47-creator-text-v1", "generation": { "context_length": 65536, "max_output_tokens": 32768, "temperature": 1.0, "top_p": 0.95, "top_k": 0, "min_p": 0, "repeat_penalty": 1.0, "presence_penalty": 0.0 }, "thinking": "on", "reasoning_effort": "not separately specified", "generation_seed": null, "generation_seed_status": "not set; native API does not expose it", "runtime_version": "not verified", "exact_gpu_placement": "not verified", "source_id": "large/glm30-on", "batch": "large", "total_parameters": 31221488576, "active_parameters": 3000000000, "effective_parameters": null, "size_class": "large", "parameter_count_basis": "Hugging Face safetensors.total for the pinned official artifact", "parameter_source_url": "https://huggingface.co/api/models/zai-org/GLM-4.7-Flash/revision/7dd20894a642a0aa287e9827cb1a1f7f91386b67", "parameter_card_url": "https://huggingface.co/zai-org/GLM-4.7-Flash/blob/7dd20894a642a0aa287e9827cb1a1f7f91386b67/README.md", "parameter_card_sha256": "7bfcbced42741ceb3fff8809f8d6c84ada9696850e257c4e53e6c2fe9ef4f9bf", "parameter_checked_date": "2026-10-21", "parameter_notes": "Card labels the model 30B-A3B; artifact total is about 31.22B." }, { "id": "muse-glimmer/low", "suite_id": "large", "model": "Muse Glimmer", "mode": "Low thinking effort", "model_key": "meta/muse-glimmer", "selected_variant": "meta/muse-glimmer@q4_k_m", "quantization": "Q4_K_M", "adapter": "native-effort-v1", "generation": { "context_length": 65536, "max_output_tokens": 32768, "temperature": 1.0, "top_p": 0.95, "top_k": 64, "min_p": 0, "repeat_penalty": 1.0 }, "thinking": "on", "reasoning_effort": "low", "generation_seed": null, "generation_seed_status": "not set; native API does not expose it", "runtime_version": "not verified", "exact_gpu_placement": "not verified", "source_id": "large/muse-low", "batch": "large", "total_parameters": 29776626688, "active_parameters": null, "effective_parameters": null, "size_class": "large", "parameter_count_basis": "Hugging Face safetensors.total for the pinned official artifact", "parameter_source_url": "https://huggingface.co/api/models/meta-models/Muse-Glimmer-30B/revision/7af58019cf472021aae5480731c9ae8e141b56c7", "parameter_card_url": "https://huggingface.co/meta-models/Muse-Glimmer-30B/blob/7af58019cf472021aae5480731c9ae8e141b56c7/README.md", "parameter_card_sha256": "2ba646dcbe5b3b97daedc59e64ef19505312fa99b6598cacc0b82d93f4d74163", "parameter_checked_date": "2026-10-21", "parameter_notes": "Card describes a dense model and rounds total including vision to about 29.6B; artifact metadata gives about 29.78B. No separate active count is published." } ], "cells": [ { "configuration": "gemma4-e4b/off", "condition": "no_source", "records": 120, "quality": { "planned": 120, "answers_returned": 120, "operational_failures": 0, "graded": 120, "exact_claim_scores": 120, "mean_claim_score": 35.9, "fully_correct": 7, "profile_assessed": 120, "mean_fully_met_claim_coverage": 0.3015277777777778, "answers_with_material_errors": 111, "material_error_assessed": 120, "pending_reviews": 0, "operationally_flagged_answers": 0, "output_ceiling_hits": 0, "thinking_marker_breaches": 0, "direct_failures": 0, "regenerated_answers": 0, "mean_output_tokens": 517.6083333333333, "mean_client_seconds": 5.604219952783144 }, "timing": { "recorded_cases": 120, "complete_timing_cases": 120, "missing_or_partial_timing_cases": 0, "mean_seconds": 5.604219952783144, "median_seconds": 5.011316405492835, "p90_seconds": 8.151041434000945, "p95_seconds": 10.610105722997105, "p99_seconds": 15.031943841000611, "max_seconds": 21.645217498997226, "nonempty_finals": 120, "no_final_cases": 0, "output_ceiling_hits": 0, "retried_cases": 0, "request_failure_attempts": 0, "final_returned_median_seconds": 5.011316405492835, "final_returned_p95_seconds": 10.610105722997105, "first_token_timing_cases": 120, "first_token_p95_seconds": 0.204363, "observed_above_seconds": { "120": { "count": 0, "denominator": 120 }, "180": { "count": 0, "denominator": 120 }, "300": { "count": 0, "denominator": 120 }, "900": { "count": 0, "denominator": 120 }, "1800": { "count": 0, "denominator": 120 } }, "p25_seconds": 3.9278582879924215, "p75_seconds": 6.702583901002072, "final_answer_completion": { "count": 120, "mean": 5.604219952783144, "median": 5.011316405492835, "p95": 10.610105722997105, "maximum": 21.645217498997226, "p25": 3.9278582879924215, "p75": 6.702583901002072 }, "above_30_seconds": 0, "bins": [ 58, 54, 7, 1, 0 ] }, "output_tokens": { "count": 120, "mean": 517.6083333333333, "median": 466.0, "p95": 911, "maximum": 1448 }, "input_tokens": { "count": 120, "mean": 122.33333333333333, "median": 114.0, "p95": 181, "maximum": 292 }, "thinking_tokens": { "count": 120, "mean": 0, "median": 0.0, "p95": 0, "maximum": 0 }, "server_tokens_per_second": { "count": 120, "mean": 133.90520917858413, "median": 134.47019797048185, "p95": 135.44155696816824, "maximum": 135.9396015794887 }, "first_token_seconds": { "count": 120, "mean": 0.08948276666666667, "median": 0.0572235, "p95": 0.204363, "maximum": 0.336395 }, "no_final": 0 }, { "configuration": "gemma4-e4b/off", "condition": "supplied_passage", "records": 120, "quality": { "planned": 120, "answers_returned": 120, "operational_failures": 0, "graded": 120, "exact_claim_scores": 120, "mean_claim_score": 87.11666666666666, "fully_correct": 78, "profile_assessed": 119, "mean_fully_met_claim_coverage": 0.8484722222222222, "answers_with_material_errors": 36, "material_error_assessed": 119, "pending_reviews": 1, "operationally_flagged_answers": 0, "output_ceiling_hits": 0, "thinking_marker_breaches": 0, "direct_failures": 0, "regenerated_answers": 0, "mean_output_tokens": 383.68333333333334, "mean_client_seconds": 4.342100253925612 }, "timing": { "recorded_cases": 120, "complete_timing_cases": 120, "missing_or_partial_timing_cases": 0, "mean_seconds": 4.342100253925612, "median_seconds": 3.9901296970056137, "p90_seconds": 7.080524860008154, "p95_seconds": 8.767960596000194, "p99_seconds": 10.123264835994632, "max_seconds": 11.100527478993172, "nonempty_finals": 120, "no_final_cases": 0, "output_ceiling_hits": 0, "retried_cases": 0, "request_failure_attempts": 0, "final_returned_median_seconds": 3.9901296970056137, "final_returned_p95_seconds": 8.767960596000194, "first_token_timing_cases": 120, "first_token_p95_seconds": 0.31690199999999996, "observed_above_seconds": { "120": { "count": 0, "denominator": 120 }, "180": { "count": 0, "denominator": 120 }, "300": { "count": 0, "denominator": 120 }, "900": { "count": 0, "denominator": 120 }, "1800": { "count": 0, "denominator": 120 } }, "p25_seconds": 2.95872265601065, "p75_seconds": 5.111951914004749, "final_answer_completion": { "count": 120, "mean": 4.342100253925612, "median": 3.9901296970056137, "p95": 8.767960596000194, "maximum": 11.100527478993172, "p25": 2.95872265601065, "p75": 5.111951914004749 }, "above_30_seconds": 0, "bins": [ 85, 32, 3, 0, 0 ] }, "output_tokens": { "count": 120, "mean": 383.68333333333334, "median": 331.0, "p95": 892, "maximum": 1319 }, "input_tokens": { "count": 120, "mean": 357.1666666666667, "median": 343.5, "p95": 575, "maximum": 999 }, "thinking_tokens": { "count": 120, "mean": 0, "median": 0.0, "p95": 0, "maximum": 0 }, "server_tokens_per_second": { "count": 120, "mean": 133.32863204484724, "median": 133.75490822189965, "p95": 134.76205728125854, "maximum": 135.30124980596332 }, "first_token_seconds": { "count": 120, "mean": 0.14091244166666667, "median": 0.100392, "p95": 0.31690199999999996, "maximum": 0.7095629999999999 }, "no_final": 0 }, { "configuration": "gemma4-e4b/on", "condition": "no_source", "records": 120, "quality": { "planned": 120, "answers_returned": 120, "operational_failures": 0, "graded": 120, "exact_claim_scores": 120, "mean_claim_score": 39.266666666666666, "fully_correct": 11, "profile_assessed": 120, "mean_fully_met_claim_coverage": 0.33791666666666664, "answers_with_material_errors": 105, "material_error_assessed": 120, "pending_reviews": 2, "operationally_flagged_answers": 0, "output_ceiling_hits": 0, "thinking_marker_breaches": 0, "direct_failures": 0, "regenerated_answers": 0, "mean_output_tokens": 1058.3166666666666, "mean_client_seconds": 9.414645599692154 }, "timing": { "recorded_cases": 120, "complete_timing_cases": 120, "missing_or_partial_timing_cases": 0, "mean_seconds": 9.414645599692154, "median_seconds": 9.312436531996354, "p90_seconds": 11.989284109993605, "p95_seconds": 13.034773176012095, "p99_seconds": 15.148238533001859, "max_seconds": 15.985178886010544, "nonempty_finals": 120, "no_final_cases": 0, "output_ceiling_hits": 0, "retried_cases": 0, "request_failure_attempts": 0, "final_returned_median_seconds": 9.312436531996354, "final_returned_p95_seconds": 13.034773176012095, "first_token_timing_cases": 120, "first_token_p95_seconds": 0.17994200000000002, "observed_above_seconds": { "120": { "count": 0, "denominator": 120 }, "180": { "count": 0, "denominator": 120 }, "300": { "count": 0, "denominator": 120 }, "900": { "count": 0, "denominator": 120 }, "1800": { "count": 0, "denominator": 120 } }, "p25_seconds": 7.977075734001119, "p75_seconds": 10.960555558995111, "final_answer_completion": { "count": 120, "mean": 9.414645599692154, "median": 9.312436531996354, "p95": 13.034773176012095, "maximum": 15.985178886010544, "p25": 7.977075734001119, "p75": 10.960555558995111 }, "above_30_seconds": 0, "bins": [ 2, 66, 52, 0, 0 ] }, "output_tokens": { "count": 120, "mean": 1058.3166666666666, "median": 1037.0, "p95": 1439, "maximum": 1812 }, "input_tokens": { "count": 120, "mean": 124.33333333333333, "median": 116.0, "p95": 183, "maximum": 294 }, "thinking_tokens": { "count": 120, "mean": 734.8083333333333, "median": 736.0, "p95": 1056, "maximum": 1326 }, "server_tokens_per_second": { "count": 120, "mean": 133.11994704429705, "median": 133.31215827286997, "p95": 134.58588224102053, "maximum": 135.39883958759444 }, "first_token_seconds": { "count": 120, "mean": 0.085348875, "median": 0.0597655, "p95": 0.17994200000000002, "maximum": 0.361241 }, "no_final": 0 }, { "configuration": "gemma4-e4b/on", "condition": "supplied_passage", "records": 120, "quality": { "planned": 120, "answers_returned": 120, "operational_failures": 0, "graded": 120, "exact_claim_scores": 120, "mean_claim_score": 93.65833333333333, "fully_correct": 91, "profile_assessed": 120, "mean_fully_met_claim_coverage": 0.91375, "answers_with_material_errors": 23, "material_error_assessed": 120, "pending_reviews": 0, "operationally_flagged_answers": 0, "output_ceiling_hits": 0, "thinking_marker_breaches": 0, "direct_failures": 0, "regenerated_answers": 0, "mean_output_tokens": 801.6916666666667, "mean_client_seconds": 7.350844491072947 }, "timing": { "recorded_cases": 120, "complete_timing_cases": 120, "missing_or_partial_timing_cases": 0, "mean_seconds": 7.350844491072947, "median_seconds": 7.0208621914935065, "p90_seconds": 9.934603402012726, "p95_seconds": 11.056239970988827, "p99_seconds": 13.79085455800032, "max_seconds": 16.293478806997882, "nonempty_finals": 120, "no_final_cases": 0, "output_ceiling_hits": 0, "retried_cases": 0, "request_failure_attempts": 0, "final_returned_median_seconds": 7.0208621914935065, "final_returned_p95_seconds": 11.056239970988827, "first_token_timing_cases": 120, "first_token_p95_seconds": 0.306333, "observed_above_seconds": { "120": { "count": 0, "denominator": 120 }, "180": { "count": 0, "denominator": 120 }, "300": { "count": 0, "denominator": 120 }, "900": { "count": 0, "denominator": 120 }, "1800": { "count": 0, "denominator": 120 } }, "p25_seconds": 5.995619124994846, "p75_seconds": 8.448429267999018, "final_answer_completion": { "count": 120, "mean": 7.350844491072947, "median": 7.0208621914935065, "p95": 11.056239970988827, "maximum": 16.293478806997882, "p25": 5.995619124994846, "p75": 8.448429267999018 }, "above_30_seconds": 0, "bins": [ 13, 95, 12, 0, 0 ] }, "output_tokens": { "count": 120, "mean": 801.6916666666667, "median": 776.0, "p95": 1279, "maximum": 1881 }, "input_tokens": { "count": 120, "mean": 359.1666666666667, "median": 345.5, "p95": 577, "maximum": 1001 }, "thinking_tokens": { "count": 120, "mean": 532.9083333333333, "median": 533.5, "p95": 876, "maximum": 1341 }, "server_tokens_per_second": { "count": 120, "mean": 133.28725227149147, "median": 133.3475955788746, "p95": 134.6043072675431, "maximum": 136.01653680881853 }, "first_token_seconds": { "count": 120, "mean": 0.13695515, "median": 0.102504, "p95": 0.306333, "maximum": 0.379988 }, "no_final": 0 }, { "configuration": "lfm2.5-2.6b/native", "condition": "no_source", "records": 120, "quality": { "planned": 120, "answers_returned": 120, "operational_failures": 0, "graded": 120, "exact_claim_scores": 120, "mean_claim_score": 28.4, "fully_correct": 1, "profile_assessed": 119, "mean_fully_met_claim_coverage": 0.23180555555555554, "answers_with_material_errors": 118, "material_error_assessed": 119, "pending_reviews": 8, "operationally_flagged_answers": 0, "output_ceiling_hits": 0, "thinking_marker_breaches": 0, "direct_failures": 0, "regenerated_answers": 0, "mean_output_tokens": 2392, "mean_client_seconds": 9.818030036818284 }, "timing": { "recorded_cases": 120, "complete_timing_cases": 120, "missing_or_partial_timing_cases": 0, "mean_seconds": 9.818030036818284, "median_seconds": 8.30514467699686, "p90_seconds": 18.262600780988578, "p95_seconds": 23.245326806005323, "p99_seconds": 27.825941012997646, "max_seconds": 30.81625905999681, "nonempty_finals": 120, "no_final_cases": 0, "output_ceiling_hits": 0, "retried_cases": 0, "request_failure_attempts": 0, "final_returned_median_seconds": 8.30514467699686, "final_returned_p95_seconds": 23.245326806005323, "first_token_timing_cases": 120, "first_token_p95_seconds": 0.037697, "observed_above_seconds": { "120": { "count": 0, "denominator": 120 }, "180": { "count": 0, "denominator": 120 }, "300": { "count": 0, "denominator": 120 }, "900": { "count": 0, "denominator": 120 }, "1800": { "count": 0, "denominator": 120 } }, "p25_seconds": 5.7175636330212, "p75_seconds": 11.722402907005744, "final_answer_completion": { "count": 120, "mean": 9.818030036818284, "median": 8.30514467699686, "p95": 23.245326806005323, "maximum": 30.81625905999681, "p25": 5.7175636330212, "p75": 11.722402907005744 }, "above_30_seconds": 1, "bins": [ 16, 60, 36, 7, 1 ] }, "output_tokens": { "count": 120, "mean": 2392, "median": 2016.5, "p95": 5608, "maximum": 7506 }, "input_tokens": { "count": 120, "mean": 125.6, "median": 117.0, "p95": 182, "maximum": 302 }, "thinking_tokens": { "count": 120, "mean": 1948.9083333333333, "median": 1587.5, "p95": 5120, "maximum": 7051 }, "server_tokens_per_second": { "count": 120, "mean": 247.66187037053248, "median": 247.8866057309287, "p95": 253.51327614889644, "maximum": 253.88756069453208 }, "first_token_seconds": { "count": 120, "mean": 0.027623858333333334, "median": 0.025594, "p95": 0.037697, "maximum": 0.043299 }, "no_final": 0 }, { "configuration": "lfm2.5-2.6b/native", "condition": "supplied_passage", "records": 120, "quality": { "planned": 120, "answers_returned": 120, "operational_failures": 0, "graded": 120, "exact_claim_scores": 120, "mean_claim_score": 74.03333333333333, "fully_correct": 32, "profile_assessed": 120, "mean_fully_met_claim_coverage": 0.7061111111111111, "answers_with_material_errors": 87, "material_error_assessed": 120, "pending_reviews": 1, "operationally_flagged_answers": 2, "output_ceiling_hits": 2, "thinking_marker_breaches": 0, "direct_failures": 0, "regenerated_answers": 0, "mean_output_tokens": 1733.9666666666667, "mean_client_seconds": 7.186745574226127 }, "timing": { "recorded_cases": 120, "complete_timing_cases": 120, "missing_or_partial_timing_cases": 0, "mean_seconds": 7.186745574226127, "median_seconds": 4.9920281510130735, "p90_seconds": 14.25175850300002, "p95_seconds": 21.39023340199492, "p99_seconds": 33.112668914021924, "max_seconds": 34.40301293801167, "nonempty_finals": 120, "no_final_cases": 0, "output_ceiling_hits": 2, "retried_cases": 0, "request_failure_attempts": 0, "final_returned_median_seconds": 4.9920281510130735, "final_returned_p95_seconds": 21.39023340199492, "first_token_timing_cases": 120, "first_token_p95_seconds": 0.075354, "observed_above_seconds": { "120": { "count": 0, "denominator": 120 }, "180": { "count": 0, "denominator": 120 }, "300": { "count": 0, "denominator": 120 }, "900": { "count": 0, "denominator": 120 }, "1800": { "count": 0, "denominator": 120 } }, "p25_seconds": 3.5915746570099145, "p75_seconds": 7.964336066012038, "final_answer_completion": { "count": 120, "mean": 7.186745574226127, "median": 4.9920281510130735, "p95": 21.39023340199492, "maximum": 34.40301293801167, "p25": 3.5915746570099145, "p75": 7.964336066012038 }, "above_30_seconds": 3, "bins": [ 60, 39, 14, 4, 3 ] }, "output_tokens": { "count": 120, "mean": 1733.9666666666667, "median": 1211.0, "p95": 5153, "maximum": 8192 }, "input_tokens": { "count": 120, "mean": 343.3833333333333, "median": 333.0, "p95": 547, "maximum": 962 }, "thinking_tokens": { "count": 120, "mean": 1337.2916666666667, "median": 782.5, "p95": 4427, "maximum": 7968 }, "server_tokens_per_second": { "count": 120, "mean": 247.95628686900147, "median": 249.19722947242758, "p95": 253.64509597520498, "maximum": 254.28273300620634 }, "first_token_seconds": { "count": 120, "mean": 0.04637028333333333, "median": 0.042829000000000006, "p95": 0.075354, "maximum": 0.106171 }, "no_final": 0 }, { "configuration": "gemma4-e2b/off", "condition": "no_source", "records": 120, "quality": { "planned": 120, "answers_returned": 120, "operational_failures": 0, "graded": 120, "exact_claim_scores": 120, "mean_claim_score": 29.566666666666666, "fully_correct": 8, "profile_assessed": 120, "mean_fully_met_claim_coverage": 0.24180555555555555, "answers_with_material_errors": 109, "material_error_assessed": 120, "pending_reviews": 1, "operationally_flagged_answers": 0, "output_ceiling_hits": 0, "thinking_marker_breaches": 0, "direct_failures": 0, "regenerated_answers": 0, "mean_output_tokens": 464.825, "mean_client_seconds": 2.3145293871925485 }, "timing": { "recorded_cases": 120, "complete_timing_cases": 120, "missing_or_partial_timing_cases": 0, "mean_seconds": 2.3145293871925485, "median_seconds": 2.089287222508574, "p90_seconds": 3.4263049159781076, "p95_seconds": 3.953539179987274, "p99_seconds": 4.970880577981006, "max_seconds": 8.249619626032654, "nonempty_finals": 120, "no_final_cases": 0, "output_ceiling_hits": 0, "retried_cases": 0, "request_failure_attempts": 0, "final_returned_median_seconds": 2.089287222508574, "final_returned_p95_seconds": 3.953539179987274, "first_token_timing_cases": 120, "first_token_p95_seconds": 0.054, "observed_above_seconds": { "120": { "count": 0, "denominator": 120 }, "180": { "count": 0, "denominator": 120 }, "300": { "count": 0, "denominator": 120 }, "900": { "count": 0, "denominator": 120 }, "1800": { "count": 0, "denominator": 120 } }, "p25_seconds": 1.6495677439961582, "p75_seconds": 2.7630482550011948, "final_answer_completion": { "count": 120, "mean": 2.3145293871925485, "median": 2.089287222508574, "p95": 3.953539179987274, "maximum": 8.249619626032654, "p25": 1.6495677439961582, "p75": 2.7630482550011948 }, "above_30_seconds": 0, "bins": [ 119, 1, 0, 0, 0 ] }, "output_tokens": { "count": 120, "mean": 464.825, "median": 421.5, "p95": 772, "maximum": 1690 }, "input_tokens": { "count": 120, "mean": 122.33333333333333, "median": 114.0, "p95": 181, "maximum": 292 }, "thinking_tokens": { "count": 120, "mean": 0, "median": 0.0, "p95": 0, "maximum": 0 }, "server_tokens_per_second": { "count": 120, "mean": 207.84264211372872, "median": 207.94488563550817, "p95": 210.38072196397118, "maximum": 211.3734196534171 }, "first_token_seconds": { "count": 120, "mean": 0.04626144166666667, "median": 0.0405905, "p95": 0.054, "maximum": 0.493073 }, "no_final": 0 }, { "configuration": "gemma4-e2b/off", "condition": "supplied_passage", "records": 120, "quality": { "planned": 120, "answers_returned": 120, "operational_failures": 0, "graded": 120, "exact_claim_scores": 120, "mean_claim_score": 79.84166666666667, "fully_correct": 58, "profile_assessed": 120, "mean_fully_met_claim_coverage": 0.7609722222222223, "answers_with_material_errors": 57, "material_error_assessed": 120, "pending_reviews": 0, "operationally_flagged_answers": 0, "output_ceiling_hits": 0, "thinking_marker_breaches": 0, "direct_failures": 0, "regenerated_answers": 0, "mean_output_tokens": 380.0416666666667, "mean_client_seconds": 1.9184900485633989 }, "timing": { "recorded_cases": 120, "complete_timing_cases": 120, "missing_or_partial_timing_cases": 0, "mean_seconds": 1.9184900485633989, "median_seconds": 1.7609122325084172, "p90_seconds": 3.2197031659889035, "p95_seconds": 4.364872126025148, "p99_seconds": 5.043119853013195, "max_seconds": 5.117597347998526, "nonempty_finals": 120, "no_final_cases": 0, "output_ceiling_hits": 0, "retried_cases": 0, "request_failure_attempts": 0, "final_returned_median_seconds": 1.7609122325084172, "final_returned_p95_seconds": 4.364872126025148, "first_token_timing_cases": 120, "first_token_p95_seconds": 0.104512, "observed_above_seconds": { "120": { "count": 0, "denominator": 120 }, "180": { "count": 0, "denominator": 120 }, "300": { "count": 0, "denominator": 120 }, "900": { "count": 0, "denominator": 120 }, "1800": { "count": 0, "denominator": 120 } }, "p25_seconds": 1.1391726629808545, "p75_seconds": 2.367357023002114, "final_answer_completion": { "count": 120, "mean": 1.9184900485633989, "median": 1.7609122325084172, "p95": 4.364872126025148, "maximum": 5.117597347998526, "p25": 1.1391726629808545, "p75": 2.367357023002114 }, "above_30_seconds": 0, "bins": [ 118, 2, 0, 0, 0 ] }, "output_tokens": { "count": 120, "mean": 380.0416666666667, "median": 345.0, "p95": 875, "maximum": 1041 }, "input_tokens": { "count": 120, "mean": 357.1666666666667, "median": 343.5, "p95": 575, "maximum": 999 }, "thinking_tokens": { "count": 120, "mean": 0, "median": 0.0, "p95": 0, "maximum": 0 }, "server_tokens_per_second": { "count": 120, "mean": 206.93643655442708, "median": 207.052891701266, "p95": 209.02668301512233, "maximum": 210.65130791776687 }, "first_token_seconds": { "count": 120, "mean": 0.06213989166666667, "median": 0.057375499999999996, "p95": 0.104512, "maximum": 0.159791 }, "no_final": 0 }, { "configuration": "gemma4-e2b/on", "condition": "no_source", "records": 120, "quality": { "planned": 120, "answers_returned": 120, "operational_failures": 0, "graded": 120, "exact_claim_scores": 120, "mean_claim_score": 28.825, "fully_correct": 10, "profile_assessed": 120, "mean_fully_met_claim_coverage": 0.245, "answers_with_material_errors": 107, "material_error_assessed": 120, "pending_reviews": 1, "operationally_flagged_answers": 0, "output_ceiling_hits": 0, "thinking_marker_breaches": 0, "direct_failures": 0, "regenerated_answers": 0, "mean_output_tokens": 960.3166666666667, "mean_client_seconds": 4.707328449542304 }, "timing": { "recorded_cases": 120, "complete_timing_cases": 120, "missing_or_partial_timing_cases": 0, "mean_seconds": 4.707328449542304, "median_seconds": 4.522743173496565, "p90_seconds": 5.958411746949423, "p95_seconds": 6.925639235007111, "p99_seconds": 8.067634281003848, "max_seconds": 10.996482214017306, "nonempty_finals": 120, "no_final_cases": 0, "output_ceiling_hits": 0, "retried_cases": 0, "request_failure_attempts": 0, "final_returned_median_seconds": 4.522743173496565, "final_returned_p95_seconds": 6.925639235007111, "first_token_timing_cases": 120, "first_token_p95_seconds": 0.047621000000000004, "observed_above_seconds": { "120": { "count": 0, "denominator": 120 }, "180": { "count": 0, "denominator": 120 }, "300": { "count": 0, "denominator": 120 }, "900": { "count": 0, "denominator": 120 }, "1800": { "count": 0, "denominator": 120 } }, "p25_seconds": 3.9347740469966084, "p75_seconds": 5.372798163967673, "final_answer_completion": { "count": 120, "mean": 4.707328449542304, "median": 4.522743173496565, "p95": 6.925639235007111, "maximum": 10.996482214017306, "p25": 3.9347740469966084, "p75": 5.372798163967673 }, "above_30_seconds": 0, "bins": [ 82, 37, 1, 0, 0 ] }, "output_tokens": { "count": 120, "mean": 960.3166666666667, "median": 924.0, "p95": 1421, "maximum": 2245 }, "input_tokens": { "count": 120, "mean": 124.33333333333333, "median": 116.0, "p95": 183, "maximum": 294 }, "thinking_tokens": { "count": 120, "mean": 675.5166666666667, "median": 650.0, "p95": 955, "maximum": 1495 }, "server_tokens_per_second": { "count": 120, "mean": 206.87896469619108, "median": 207.06801845829352, "p95": 207.7996237615422, "maximum": 208.43860687828126 }, "first_token_seconds": { "count": 120, "mean": 0.042842958333333334, "median": 0.040534, "p95": 0.047621000000000004, "maximum": 0.131977 }, "no_final": 0 }, { "configuration": "gemma4-e2b/on", "condition": "supplied_passage", "records": 120, "quality": { "planned": 120, "answers_returned": 120, "operational_failures": 0, "graded": 120, "exact_claim_scores": 120, "mean_claim_score": 86.84166666666667, "fully_correct": 76, "profile_assessed": 120, "mean_fully_met_claim_coverage": 0.8352777777777778, "answers_with_material_errors": 33, "material_error_assessed": 120, "pending_reviews": 0, "operationally_flagged_answers": 0, "output_ceiling_hits": 0, "thinking_marker_breaches": 0, "direct_failures": 0, "regenerated_answers": 0, "mean_output_tokens": 837.1916666666667, "mean_client_seconds": 4.139805731367475 }, "timing": { "recorded_cases": 120, "complete_timing_cases": 120, "missing_or_partial_timing_cases": 0, "mean_seconds": 4.139805731367475, "median_seconds": 3.8559084575099405, "p90_seconds": 5.956665334000718, "p95_seconds": 6.569992751989048, "p99_seconds": 8.511655547015835, "max_seconds": 10.442087322997395, "nonempty_finals": 120, "no_final_cases": 0, "output_ceiling_hits": 0, "retried_cases": 0, "request_failure_attempts": 0, "final_returned_median_seconds": 3.8559084575099405, "final_returned_p95_seconds": 6.569992751989048, "first_token_timing_cases": 120, "first_token_p95_seconds": 0.100721, "observed_above_seconds": { "120": { "count": 0, "denominator": 120 }, "180": { "count": 0, "denominator": 120 }, "300": { "count": 0, "denominator": 120 }, "900": { "count": 0, "denominator": 120 }, "1800": { "count": 0, "denominator": 120 } }, "p25_seconds": 3.1147124849958345, "p75_seconds": 4.667062469001394, "final_answer_completion": { "count": 120, "mean": 4.139805731367475, "median": 3.8559084575099405, "p95": 6.569992751989048, "maximum": 10.442087322997395, "p25": 3.1147124849958345, "p75": 4.667062469001394 }, "above_30_seconds": 0, "bins": [ 95, 24, 1, 0, 0 ] }, "output_tokens": { "count": 120, "mean": 837.1916666666667, "median": 784.5, "p95": 1334, "maximum": 2090 }, "input_tokens": { "count": 120, "mean": 359.1666666666667, "median": 345.5, "p95": 577, "maximum": 1001 }, "thinking_tokens": { "count": 120, "mean": 573.875, "median": 540.0, "p95": 953, "maximum": 1488 }, "server_tokens_per_second": { "count": 120, "mean": 206.59109639725347, "median": 206.71082036677552, "p95": 207.63291649887796, "maximum": 208.10760814989032 }, "first_token_seconds": { "count": 120, "mean": 0.06271653333333334, "median": 0.060014, "p95": 0.100721, "maximum": 0.130471 }, "no_final": 0 }, { "configuration": "k2-horizon-3.7b/low", "condition": "no_source", "records": 120, "quality": { "planned": 120, "answers_returned": 120, "operational_failures": 0, "graded": 120, "exact_claim_scores": 120, "mean_claim_score": 22.1, "fully_correct": 1, "profile_assessed": 120, "mean_fully_met_claim_coverage": 0.17097222222222222, "answers_with_material_errors": 115, "material_error_assessed": 120, "pending_reviews": 7, "operationally_flagged_answers": 24, "output_ceiling_hits": 23, "thinking_marker_breaches": 0, "direct_failures": 1, "regenerated_answers": 0, "mean_output_tokens": 6853.35, "mean_client_seconds": 106.64435930660848 }, "timing": { "recorded_cases": 120, "complete_timing_cases": 120, "missing_or_partial_timing_cases": 0, "mean_seconds": 106.86878929514181, "median_seconds": 5.0726538874987455, "p90_seconds": 522.565193628001, "p95_seconds": 525.5132120510007, "p99_seconds": 539.4491788619998, "max_seconds": 555.1688007489993, "nonempty_finals": 119, "no_final_cases": 1, "output_ceiling_hits": 23, "retried_cases": 3, "request_failure_attempts": 0, "final_returned_median_seconds": 5.03245187399807, "final_returned_p95_seconds": 525.8819249480002, "first_token_timing_cases": 0, "first_token_p95_seconds": null, "observed_above_seconds": { "120": { "count": 24, "denominator": 120 }, "180": { "count": 23, "denominator": 120 }, "300": { "count": 23, "denominator": 120 }, "900": { "count": 0, "denominator": 120 }, "1800": { "count": 0, "denominator": 120 } }, "p25_seconds": 2.9458580930004246, "p75_seconds": 12.673349598000641, "final_answer_completion": { "count": 119, "mean": 107.60143212683208, "median": 5.03245187399807, "p95": 525.8819249480002, "maximum": 555.1688007489993, "p25": 2.9458580930004246, "p75": 12.673349598000641 }, "above_30_seconds": 27, "bins": [ 57, 27, 8, 1, 27 ] }, "output_tokens": { "count": 120, "mean": 6853.35, "median": 506.5, "p95": 32768, "maximum": 32768 }, "input_tokens": { "count": 120, "mean": 121.18333333333334, "median": 113.0, "p95": 179, "maximum": 288 }, "thinking_tokens": { "count": 0, "mean": null, "median": null, "p95": null, "maximum": null }, "server_tokens_per_second": { "count": 0, "mean": null, "median": null, "p95": null, "maximum": null }, "first_token_seconds": { "count": 0, "mean": null, "median": null, "p95": null, "maximum": null }, "no_final": 1 }, { "configuration": "k2-horizon-3.7b/low", "condition": "supplied_passage", "records": 120, "quality": { "planned": 120, "answers_returned": 120, "operational_failures": 0, "graded": 120, "exact_claim_scores": 120, "mean_claim_score": 70.7, "fully_correct": 33, "profile_assessed": 119, "mean_fully_met_claim_coverage": 0.6630555555555555, "answers_with_material_errors": 81, "material_error_assessed": 118, "pending_reviews": 8, "operationally_flagged_answers": 6, "output_ceiling_hits": 6, "thinking_marker_breaches": 0, "direct_failures": 0, "regenerated_answers": 0, "mean_output_tokens": 2106.5583333333334, "mean_client_seconds": 31.19163628355847 }, "timing": { "recorded_cases": 120, "complete_timing_cases": 120, "missing_or_partial_timing_cases": 0, "mean_seconds": 31.19163628355847, "median_seconds": 3.2705849434996708, "p90_seconds": 8.163189139000679, "p95_seconds": 89.80997591900086, "p99_seconds": 528.7268088199999, "max_seconds": 529.2080322930015, "nonempty_finals": 120, "no_final_cases": 0, "output_ceiling_hits": 6, "retried_cases": 0, "request_failure_attempts": 0, "final_returned_median_seconds": 3.2705849434996708, "final_returned_p95_seconds": 89.80997591900086, "first_token_timing_cases": 0, "first_token_p95_seconds": null, "observed_above_seconds": { "120": { "count": 6, "denominator": 120 }, "180": { "count": 6, "denominator": 120 }, "300": { "count": 6, "denominator": 120 }, "900": { "count": 0, "denominator": 120 }, "1800": { "count": 0, "denominator": 120 } }, "p25_seconds": 2.1468084669977543, "p75_seconds": 4.6954603560006944, "final_answer_completion": { "count": 120, "mean": 31.19163628355847, "median": 3.2705849434996708, "p95": 89.80997591900086, "maximum": 529.2080322930015, "p25": 2.1468084669977543, "p75": 4.6954603560006944 }, "above_30_seconds": 8, "bins": [ 94, 14, 3, 1, 8 ] }, "output_tokens": { "count": 120, "mean": 2106.5583333333334, "median": 288.0, "p95": 8904, "maximum": 32768 }, "input_tokens": { "count": 120, "mean": 328.4916666666667, "median": 318.5, "p95": 522, "maximum": 915 }, "thinking_tokens": { "count": 0, "mean": null, "median": null, "p95": null, "maximum": null }, "server_tokens_per_second": { "count": 0, "mean": null, "median": null, "p95": null, "maximum": null }, "first_token_seconds": { "count": 0, "mean": null, "median": null, "p95": null, "maximum": null }, "no_final": 0 }, { "configuration": "granite4.2-3b/off", "condition": "no_source", "records": 120, "quality": { "planned": 120, "answers_returned": 120, "operational_failures": 0, "graded": 120, "exact_claim_scores": 120, "mean_claim_score": 33.858333333333334, "fully_correct": 2, "profile_assessed": 120, "mean_fully_met_claim_coverage": 0.27819444444444447, "answers_with_material_errors": 118, "material_error_assessed": 120, "pending_reviews": 3, "operationally_flagged_answers": 89, "output_ceiling_hits": 0, "thinking_marker_breaches": 89, "direct_failures": 0, "regenerated_answers": 0, "mean_output_tokens": 2866.6916666666666, "mean_client_seconds": 17.536824373217314 }, "timing": { "recorded_cases": 120, "complete_timing_cases": 120, "missing_or_partial_timing_cases": 0, "mean_seconds": 17.536824373217314, "median_seconds": 11.627608879498439, "p90_seconds": 35.80614589099423, "p95_seconds": 52.12359667501005, "p99_seconds": 97.73319554699992, "max_seconds": 100.92142155799957, "nonempty_finals": 120, "no_final_cases": 0, "output_ceiling_hits": 0, "retried_cases": 0, "request_failure_attempts": 0, "final_returned_median_seconds": 11.627608879498439, "final_returned_p95_seconds": 52.12359667501005, "first_token_timing_cases": 0, "first_token_p95_seconds": null, "observed_above_seconds": { "120": { "count": 0, "denominator": 120 }, "180": { "count": 0, "denominator": 120 }, "300": { "count": 0, "denominator": 120 }, "900": { "count": 0, "denominator": 120 }, "1800": { "count": 0, "denominator": 120 } }, "p25_seconds": 5.959842812997522, "p75_seconds": 20.273498154987465, "final_answer_completion": { "count": 120, "mean": 17.536824373217314, "median": 11.627608879498439, "p95": 52.12359667501005, "maximum": 100.92142155799957, "p25": 5.959842812997522, "p75": 20.273498154987465 }, "above_30_seconds": 22, "bins": [ 25, 26, 37, 10, 22 ] }, "output_tokens": { "count": 120, "mean": 2866.6916666666666, "median": 1994.0, "p95": 8270, "maximum": 15282 }, "input_tokens": { "count": 120, "mean": 123.20833333333333, "median": 115.0, "p95": 181, "maximum": 295 }, "thinking_tokens": { "count": 0, "mean": null, "median": null, "p95": null, "maximum": null }, "server_tokens_per_second": { "count": 0, "mean": null, "median": null, "p95": null, "maximum": null }, "first_token_seconds": { "count": 0, "mean": null, "median": null, "p95": null, "maximum": null }, "no_final": 0 }, { "configuration": "granite4.2-3b/off", "condition": "supplied_passage", "records": 120, "quality": { "planned": 120, "answers_returned": 120, "operational_failures": 0, "graded": 120, "exact_claim_scores": 120, "mean_claim_score": 84.55833333333334, "fully_correct": 49, "profile_assessed": 119, "mean_fully_met_claim_coverage": 0.8165277777777777, "answers_with_material_errors": 69, "material_error_assessed": 119, "pending_reviews": 1, "operationally_flagged_answers": 67, "output_ceiling_hits": 0, "thinking_marker_breaches": 67, "direct_failures": 0, "regenerated_answers": 0, "mean_output_tokens": 1832.55, "mean_client_seconds": 11.2999894266919 }, "timing": { "recorded_cases": 120, "complete_timing_cases": 120, "missing_or_partial_timing_cases": 0, "mean_seconds": 11.2999894266919, "median_seconds": 6.8670859089979785, "p90_seconds": 19.910446284004138, "p95_seconds": 37.386897885997314, "p99_seconds": 83.05889781400037, "max_seconds": 135.73921225199592, "nonempty_finals": 120, "no_final_cases": 0, "output_ceiling_hits": 0, "retried_cases": 0, "request_failure_attempts": 0, "final_returned_median_seconds": 6.8670859089979785, "final_returned_p95_seconds": 37.386897885997314, "first_token_timing_cases": 0, "first_token_p95_seconds": null, "observed_above_seconds": { "120": { "count": 1, "denominator": 120 }, "180": { "count": 0, "denominator": 120 }, "300": { "count": 0, "denominator": 120 }, "900": { "count": 0, "denominator": 120 }, "1800": { "count": 0, "denominator": 120 } }, "p25_seconds": 2.4767175600136397, "p75_seconds": 12.979480125999544, "final_answer_completion": { "count": 120, "mean": 11.2999894266919, "median": 6.8670859089979785, "p95": 37.386897885997314, "maximum": 135.73921225199592, "p25": 2.4767175600136397, "p75": 12.979480125999544 }, "above_30_seconds": 8, "bins": [ 51, 23, 34, 4, 8 ] }, "output_tokens": { "count": 120, "mean": 1832.55, "median": 1157.0, "p95": 6097, "maximum": 19672 }, "input_tokens": { "count": 120, "mean": 358.39166666666665, "median": 346.5, "p95": 578, "maximum": 1005 }, "thinking_tokens": { "count": 0, "mean": null, "median": null, "p95": null, "maximum": null }, "server_tokens_per_second": { "count": 0, "mean": null, "median": null, "p95": null, "maximum": null }, "first_token_seconds": { "count": 0, "mean": null, "median": null, "p95": null, "maximum": null }, "no_final": 0 }, { "configuration": "granite4.2-3b/on", "condition": "no_source", "records": 120, "quality": { "planned": 120, "answers_returned": 120, "operational_failures": 0, "graded": 120, "exact_claim_scores": 120, "mean_claim_score": 38.88333333333333, "fully_correct": 9, "profile_assessed": 120, "mean_fully_met_claim_coverage": 0.33319444444444446, "answers_with_material_errors": 111, "material_error_assessed": 120, "pending_reviews": 1, "operationally_flagged_answers": 0, "output_ceiling_hits": 0, "thinking_marker_breaches": 0, "direct_failures": 0, "regenerated_answers": 0, "mean_output_tokens": 5508.375, "mean_client_seconds": 34.57511501556607 }, "timing": { "recorded_cases": 120, "complete_timing_cases": 120, "missing_or_partial_timing_cases": 0, "mean_seconds": 34.57511501556607, "median_seconds": 26.374946923504467, "p90_seconds": 65.69577317099902, "p95_seconds": 86.68572923299507, "p99_seconds": 116.41093123900646, "max_seconds": 204.30458438499772, "nonempty_finals": 120, "no_final_cases": 0, "output_ceiling_hits": 0, "retried_cases": 0, "request_failure_attempts": 0, "final_returned_median_seconds": 26.374946923504467, "final_returned_p95_seconds": 86.68572923299507, "first_token_timing_cases": 0, "first_token_p95_seconds": null, "observed_above_seconds": { "120": { "count": 1, "denominator": 120 }, "180": { "count": 1, "denominator": 120 }, "300": { "count": 0, "denominator": 120 }, "900": { "count": 0, "denominator": 120 }, "1800": { "count": 0, "denominator": 120 } }, "p25_seconds": 15.907932058005827, "p75_seconds": 44.13561461601057, "final_answer_completion": { "count": 120, "mean": 34.57511501556607, "median": 26.374946923504467, "p95": 86.68572923299507, "maximum": 204.30458438499772, "p25": 15.907932058005827, "p75": 44.13561461601057 }, "above_30_seconds": 54, "bins": [ 1, 11, 29, 25, 54 ] }, "output_tokens": { "count": 120, "mean": 5508.375, "median": 4377.0, "p95": 13574, "maximum": 28099 }, "input_tokens": { "count": 120, "mean": 123.20833333333333, "median": 115.0, "p95": 181, "maximum": 295 }, "thinking_tokens": { "count": 0, "mean": null, "median": null, "p95": null, "maximum": null }, "server_tokens_per_second": { "count": 0, "mean": null, "median": null, "p95": null, "maximum": null }, "first_token_seconds": { "count": 0, "mean": null, "median": null, "p95": null, "maximum": null }, "no_final": 0 }, { "configuration": "granite4.2-3b/on", "condition": "supplied_passage", "records": 120, "quality": { "planned": 120, "answers_returned": 120, "operational_failures": 0, "graded": 120, "exact_claim_scores": 120, "mean_claim_score": 90.58333333333333, "fully_correct": 76, "profile_assessed": 119, "mean_fully_met_claim_coverage": 0.8825, "answers_with_material_errors": 43, "material_error_assessed": 119, "pending_reviews": 1, "operationally_flagged_answers": 0, "output_ceiling_hits": 0, "thinking_marker_breaches": 0, "direct_failures": 0, "regenerated_answers": 0, "mean_output_tokens": 2508.1916666666666, "mean_client_seconds": 15.560693055508697 }, "timing": { "recorded_cases": 120, "complete_timing_cases": 120, "missing_or_partial_timing_cases": 0, "mean_seconds": 15.560693055508697, "median_seconds": 8.51492815700476, "p90_seconds": 29.18252176900569, "p95_seconds": 45.17118737300916, "p99_seconds": 94.05363320399192, "max_seconds": 124.34756956799538, "nonempty_finals": 120, "no_final_cases": 0, "output_ceiling_hits": 0, "retried_cases": 0, "request_failure_attempts": 0, "final_returned_median_seconds": 8.51492815700476, "final_returned_p95_seconds": 45.17118737300916, "first_token_timing_cases": 0, "first_token_p95_seconds": null, "observed_above_seconds": { "120": { "count": 1, "denominator": 120 }, "180": { "count": 0, "denominator": 120 }, "300": { "count": 0, "denominator": 120 }, "900": { "count": 0, "denominator": 120 }, "1800": { "count": 0, "denominator": 120 } }, "p25_seconds": 5.164923757998622, "p75_seconds": 18.353012742998544, "final_answer_completion": { "count": 120, "mean": 15.560693055508697, "median": 8.51492815700476, "p95": 45.17118737300916, "maximum": 124.34756956799538, "p25": 5.164923757998622, "p75": 18.353012742998544 }, "above_30_seconds": 12, "bins": [ 27, 43, 25, 13, 12 ] }, "output_tokens": { "count": 120, "mean": 2508.1916666666666, "median": 1443.5, "p95": 7364, "maximum": 18336 }, "input_tokens": { "count": 120, "mean": 358.39166666666665, "median": 346.5, "p95": 578, "maximum": 1005 }, "thinking_tokens": { "count": 0, "mean": null, "median": null, "p95": null, "maximum": null }, "server_tokens_per_second": { "count": 0, "mean": null, "median": null, "p95": null, "maximum": null }, "first_token_seconds": { "count": 0, "mean": null, "median": null, "p95": null, "maximum": null }, "no_final": 0 }, { "configuration": "qwen3.5-4b/off", "condition": "no_source", "records": 120, "quality": { "planned": 120, "answers_returned": 120, "operational_failures": 0, "graded": 120, "exact_claim_scores": 120, "mean_claim_score": 36.69166666666667, "fully_correct": 0, "profile_assessed": 120, "mean_fully_met_claim_coverage": 0.30569444444444444, "answers_with_material_errors": 120, "material_error_assessed": 120, "pending_reviews": 23, "operationally_flagged_answers": 29, "output_ceiling_hits": 1, "thinking_marker_breaches": 28, "direct_failures": 0, "regenerated_answers": 0, "mean_output_tokens": 2916.616666666667, "mean_client_seconds": 22.127485552891933 }, "timing": { "recorded_cases": 120, "complete_timing_cases": 120, "missing_or_partial_timing_cases": 0, "mean_seconds": 22.127485552891933, "median_seconds": 6.114862078502483, "p90_seconds": 66.43627836499945, "p95_seconds": 82.86190171299677, "p99_seconds": 107.71195344100124, "max_seconds": 300.5635435330041, "nonempty_finals": 120, "no_final_cases": 0, "output_ceiling_hits": 1, "retried_cases": 0, "request_failure_attempts": 0, "final_returned_median_seconds": 6.114862078502483, "final_returned_p95_seconds": 82.86190171299677, "first_token_timing_cases": 0, "first_token_p95_seconds": null, "observed_above_seconds": { "120": { "count": 1, "denominator": 120 }, "180": { "count": 1, "denominator": 120 }, "300": { "count": 1, "denominator": 120 }, "900": { "count": 0, "denominator": 120 }, "1800": { "count": 0, "denominator": 120 } }, "p25_seconds": 4.263538664003136, "p75_seconds": 23.148665054992307, "final_answer_completion": { "count": 120, "mean": 22.127485552891933, "median": 6.114862078502483, "p95": 82.86190171299677, "maximum": 300.5635435330041, "p25": 4.263538664003136, "p75": 23.148665054992307 }, "above_30_seconds": 30, "bins": [ 43, 37, 8, 2, 30 ] }, "output_tokens": { "count": 120, "mean": 2916.616666666667, "median": 834.0, "p95": 11057, "maximum": 32768 }, "input_tokens": { "count": 120, "mean": 124.375, "median": 116.0, "p95": 182, "maximum": 297 }, "thinking_tokens": { "count": 0, "mean": null, "median": null, "p95": null, "maximum": null }, "server_tokens_per_second": { "count": 0, "mean": null, "median": null, "p95": null, "maximum": null }, "first_token_seconds": { "count": 0, "mean": null, "median": null, "p95": null, "maximum": null }, "no_final": 0 }, { "configuration": "qwen3.5-4b/off", "condition": "supplied_passage", "records": 120, "quality": { "planned": 120, "answers_returned": 120, "operational_failures": 0, "graded": 120, "exact_claim_scores": 120, "mean_claim_score": 87.36666666666666, "fully_correct": 51, "profile_assessed": 120, "mean_fully_met_claim_coverage": 0.8476388888888889, "answers_with_material_errors": 68, "material_error_assessed": 120, "pending_reviews": 5, "operationally_flagged_answers": 6, "output_ceiling_hits": 0, "thinking_marker_breaches": 6, "direct_failures": 0, "regenerated_answers": 0, "mean_output_tokens": 986.5, "mean_client_seconds": 7.438834479966075 }, "timing": { "recorded_cases": 120, "complete_timing_cases": 120, "missing_or_partial_timing_cases": 0, "mean_seconds": 7.438834479966075, "median_seconds": 3.5740008395077894, "p90_seconds": 8.9167451300018, "p95_seconds": 16.167494409004576, "p99_seconds": 79.12350526599039, "max_seconds": 101.28867584100226, "nonempty_finals": 120, "no_final_cases": 0, "output_ceiling_hits": 0, "retried_cases": 0, "request_failure_attempts": 0, "final_returned_median_seconds": 3.5740008395077894, "final_returned_p95_seconds": 16.167494409004576, "first_token_timing_cases": 0, "first_token_p95_seconds": null, "observed_above_seconds": { "120": { "count": 0, "denominator": 120 }, "180": { "count": 0, "denominator": 120 }, "300": { "count": 0, "denominator": 120 }, "900": { "count": 0, "denominator": 120 }, "1800": { "count": 0, "denominator": 120 } }, "p25_seconds": 2.5554613540007267, "p75_seconds": 5.134720807996928, "final_answer_completion": { "count": 120, "mean": 7.438834479966075, "median": 3.5740008395077894, "p95": 16.167494409004576, "maximum": 101.28867584100226, "p25": 2.5554613540007267, "p75": 5.134720807996928 }, "above_30_seconds": 6, "bins": [ 88, 22, 4, 0, 6 ] }, "output_tokens": { "count": 120, "mean": 986.5, "median": 467.0, "p95": 2222, "maximum": 13376 }, "input_tokens": { "count": 120, "mean": 349.78333333333336, "median": 337.5, "p95": 556, "maximum": 964 }, "thinking_tokens": { "count": 0, "mean": null, "median": null, "p95": null, "maximum": null }, "server_tokens_per_second": { "count": 0, "mean": null, "median": null, "p95": null, "maximum": null }, "first_token_seconds": { "count": 0, "mean": null, "median": null, "p95": null, "maximum": null }, "no_final": 0 }, { "configuration": "qwen3.5-4b/on", "condition": "no_source", "records": 120, "quality": { "planned": 120, "answers_returned": 120, "operational_failures": 0, "graded": 120, "exact_claim_scores": 120, "mean_claim_score": 40.59166666666667, "fully_correct": 2, "profile_assessed": 119, "mean_fully_met_claim_coverage": 0.34555555555555556, "answers_with_material_errors": 116, "material_error_assessed": 119, "pending_reviews": 18, "operationally_flagged_answers": 1, "output_ceiling_hits": 1, "thinking_marker_breaches": 0, "direct_failures": 1, "regenerated_answers": 0, "mean_output_tokens": 11715.916666666666, "mean_client_seconds": 90.43474560724935 }, "timing": { "recorded_cases": 120, "complete_timing_cases": 120, "missing_or_partial_timing_cases": 0, "mean_seconds": 90.43474560724935, "median_seconds": 83.77937390448642, "p90_seconds": 138.2841383019986, "p95_seconds": 163.26857816400297, "p99_seconds": 193.32805091701448, "max_seconds": 301.8480701780063, "nonempty_finals": 119, "no_final_cases": 1, "output_ceiling_hits": 1, "retried_cases": 0, "request_failure_attempts": 0, "final_returned_median_seconds": 83.75592601599055, "final_returned_p95_seconds": 163.26857816400297, "first_token_timing_cases": 0, "first_token_p95_seconds": null, "observed_above_seconds": { "120": { "count": 22, "denominator": 120 }, "180": { "count": 3, "denominator": 120 }, "300": { "count": 1, "denominator": 120 }, "900": { "count": 0, "denominator": 120 }, "1800": { "count": 0, "denominator": 120 } }, "p25_seconds": 65.06239960300445, "p75_seconds": 105.07801196399669, "final_answer_completion": { "count": 119, "mean": 88.65816304783122, "median": 83.75592601599055, "p95": 163.26857816400297, "maximum": 193.32805091701448, "p25": 65.06239960300445, "p75": 105.07801196399669 }, "above_30_seconds": 117, "bins": [ 0, 0, 1, 2, 117 ] }, "output_tokens": { "count": 120, "mean": 11715.916666666666, "median": 10995.5, "p95": 19566, "maximum": 32768 }, "input_tokens": { "count": 120, "mean": 122.375, "median": 114.0, "p95": 180, "maximum": 295 }, "thinking_tokens": { "count": 0, "mean": null, "median": null, "p95": null, "maximum": null }, "server_tokens_per_second": { "count": 0, "mean": null, "median": null, "p95": null, "maximum": null }, "first_token_seconds": { "count": 0, "mean": null, "median": null, "p95": null, "maximum": null }, "no_final": 1 }, { "configuration": "qwen3.5-4b/on", "condition": "supplied_passage", "records": 120, "quality": { "planned": 120, "answers_returned": 120, "operational_failures": 0, "graded": 120, "exact_claim_scores": 120, "mean_claim_score": 95.38333333333334, "fully_correct": 90, "profile_assessed": 120, "mean_fully_met_claim_coverage": 0.9331944444444444, "answers_with_material_errors": 28, "material_error_assessed": 120, "pending_reviews": 1, "operationally_flagged_answers": 0, "output_ceiling_hits": 0, "thinking_marker_breaches": 0, "direct_failures": 0, "regenerated_answers": 0, "mean_output_tokens": 5193.15, "mean_client_seconds": 38.75811221433311 }, "timing": { "recorded_cases": 120, "complete_timing_cases": 120, "missing_or_partial_timing_cases": 0, "mean_seconds": 38.75811221433311, "median_seconds": 32.70316506900417, "p90_seconds": 74.03098855999997, "p95_seconds": 94.70089313600329, "p99_seconds": 148.31651260999206, "max_seconds": 166.62847473600414, "nonempty_finals": 120, "no_final_cases": 0, "output_ceiling_hits": 0, "retried_cases": 0, "request_failure_attempts": 0, "final_returned_median_seconds": 32.70316506900417, "final_returned_p95_seconds": 94.70089313600329, "first_token_timing_cases": 0, "first_token_p95_seconds": null, "observed_above_seconds": { "120": { "count": 3, "denominator": 120 }, "180": { "count": 0, "denominator": 120 }, "300": { "count": 0, "denominator": 120 }, "900": { "count": 0, "denominator": 120 }, "1800": { "count": 0, "denominator": 120 } }, "p25_seconds": 17.909592307987623, "p75_seconds": 47.11463881999953, "final_answer_completion": { "count": 120, "mean": 38.75811221433311, "median": 32.70316506900417, "p95": 94.70089313600329, "maximum": 166.62847473600414, "p25": 17.909592307987623, "p75": 47.11463881999953 }, "above_30_seconds": 67, "bins": [ 0, 8, 25, 20, 67 ] }, "output_tokens": { "count": 120, "mean": 5193.15, "median": 4501.0, "p95": 12534, "maximum": 20290 }, "input_tokens": { "count": 120, "mean": 347.78333333333336, "median": 335.5, "p95": 554, "maximum": 962 }, "thinking_tokens": { "count": 0, "mean": null, "median": null, "p95": null, "maximum": null }, "server_tokens_per_second": { "count": 0, "mean": null, "median": null, "p95": null, "maximum": null }, "first_token_seconds": { "count": 0, "mean": null, "median": null, "p95": null, "maximum": null }, "no_final": 0 }, { "configuration": "granite4.2-8b/on", "condition": "no_source", "records": 120, "quality": { "planned": 120, "answers_returned": 120, "operational_failures": 0, "graded": 120, "exact_claim_scores": 120, "mean_claim_score": 44.575, "fully_correct": 2, "profile_assessed": 119, "mean_fully_met_claim_coverage": 0.37958333333333333, "answers_with_material_errors": 117, "material_error_assessed": 119, "pending_reviews": 4, "operationally_flagged_answers": 0, "output_ceiling_hits": 0, "thinking_marker_breaches": 0, "direct_failures": 0, "regenerated_answers": 0, "mean_output_tokens": 4734.825, "mean_client_seconds": 48.04009623179081 }, "timing": { "recorded_cases": 120, "complete_timing_cases": 120, "missing_or_partial_timing_cases": 0, "mean_seconds": 48.04009623179081, "median_seconds": 37.739013958504074, "p90_seconds": 85.81496201298432, "p95_seconds": 96.41692897598841, "p99_seconds": 202.17382362298667, "max_seconds": 219.2593050459982, "nonempty_finals": 120, "no_final_cases": 0, "output_ceiling_hits": 0, "retried_cases": 0, "request_failure_attempts": 0, "final_returned_median_seconds": 37.739013958504074, "final_returned_p95_seconds": 96.41692897598841, "first_token_timing_cases": 0, "first_token_p95_seconds": null, "observed_above_seconds": { "120": { "count": 3, "denominator": 120 }, "180": { "count": 2, "denominator": 120 }, "300": { "count": 0, "denominator": 120 }, "900": { "count": 0, "denominator": 120 }, "1800": { "count": 0, "denominator": 120 } }, "p25_seconds": 25.79880037700059, "p75_seconds": 64.16362267101067, "final_answer_completion": { "count": 120, "mean": 48.04009623179081, "median": 37.739013958504074, "p95": 96.41692897598841, "maximum": 219.2593050459982, "p25": 25.79880037700059, "p75": 64.16362267101067 }, "above_30_seconds": 81, "bins": [ 0, 0, 21, 18, 81 ] }, "output_tokens": { "count": 120, "mean": 4734.825, "median": 3825.0, "p95": 9269, "maximum": 19375 }, "input_tokens": { "count": 120, "mean": 123.20833333333333, "median": 115.0, "p95": 181, "maximum": 295 }, "thinking_tokens": { "count": 0, "mean": null, "median": null, "p95": null, "maximum": null }, "server_tokens_per_second": { "count": 0, "mean": null, "median": null, "p95": null, "maximum": null }, "first_token_seconds": { "count": 0, "mean": null, "median": null, "p95": null, "maximum": null }, "no_final": 0 }, { "configuration": "granite4.2-8b/on", "condition": "supplied_passage", "records": 120, "quality": { "planned": 120, "answers_returned": 120, "operational_failures": 0, "graded": 120, "exact_claim_scores": 120, "mean_claim_score": 93.96666666666667, "fully_correct": 63, "profile_assessed": 117, "mean_fully_met_claim_coverage": 0.9215277777777777, "answers_with_material_errors": 54, "material_error_assessed": 117, "pending_reviews": 4, "operationally_flagged_answers": 0, "output_ceiling_hits": 0, "thinking_marker_breaches": 0, "direct_failures": 0, "regenerated_answers": 0, "mean_output_tokens": 2971.891666666667, "mean_client_seconds": 30.398619683539923 }, "timing": { "recorded_cases": 120, "complete_timing_cases": 120, "missing_or_partial_timing_cases": 0, "mean_seconds": 30.398619683539923, "median_seconds": 18.47347018450091, "p90_seconds": 57.82827217900194, "p95_seconds": 97.1606799820147, "p99_seconds": 153.6355670490011, "max_seconds": 304.3662539739744, "nonempty_finals": 120, "no_final_cases": 0, "output_ceiling_hits": 0, "retried_cases": 0, "request_failure_attempts": 0, "final_returned_median_seconds": 18.47347018450091, "final_returned_p95_seconds": 97.1606799820147, "first_token_timing_cases": 0, "first_token_p95_seconds": null, "observed_above_seconds": { "120": { "count": 4, "denominator": 120 }, "180": { "count": 1, "denominator": 120 }, "300": { "count": 1, "denominator": 120 }, "900": { "count": 0, "denominator": 120 }, "1800": { "count": 0, "denominator": 120 } }, "p25_seconds": 13.288219420996029, "p75_seconds": 30.699411610985408, "final_answer_completion": { "count": 120, "mean": 30.398619683539923, "median": 18.47347018450091, "p95": 97.1606799820147, "maximum": 304.3662539739744, "p25": 13.288219420996029, "p75": 30.699411610985408 }, "above_30_seconds": 31, "bins": [ 0, 13, 55, 21, 31 ] }, "output_tokens": { "count": 120, "mean": 2971.891666666667, "median": 1884.0, "p95": 9408, "maximum": 25497 }, "input_tokens": { "count": 120, "mean": 358.39166666666665, "median": 346.5, "p95": 578, "maximum": 1005 }, "thinking_tokens": { "count": 0, "mean": null, "median": null, "p95": null, "maximum": null }, "server_tokens_per_second": { "count": 0, "mean": null, "median": null, "p95": null, "maximum": null }, "first_token_seconds": { "count": 0, "mean": null, "median": null, "p95": null, "maximum": null }, "no_final": 0 }, { "configuration": "granite4.2-8b/off", "condition": "no_source", "records": 120, "quality": { "planned": 120, "answers_returned": 120, "operational_failures": 0, "graded": 120, "exact_claim_scores": 120, "mean_claim_score": 41.09166666666667, "fully_correct": 2, "profile_assessed": 120, "mean_fully_met_claim_coverage": 0.3473611111111111, "answers_with_material_errors": 118, "material_error_assessed": 120, "pending_reviews": 4, "operationally_flagged_answers": 78, "output_ceiling_hits": 0, "thinking_marker_breaches": 78, "direct_failures": 0, "regenerated_answers": 0, "mean_output_tokens": 3264.0833333333335, "mean_client_seconds": 33.18850703368516 }, "timing": { "recorded_cases": 120, "complete_timing_cases": 120, "missing_or_partial_timing_cases": 0, "mean_seconds": 33.18850703368516, "median_seconds": 22.379449365500477, "p90_seconds": 62.04956451698672, "p95_seconds": 86.87513911200222, "p99_seconds": 146.75476339098532, "max_seconds": 326.8716572630219, "nonempty_finals": 120, "no_final_cases": 0, "output_ceiling_hits": 0, "retried_cases": 0, "request_failure_attempts": 0, "final_returned_median_seconds": 22.379449365500477, "final_returned_p95_seconds": 86.87513911200222, "first_token_timing_cases": 0, "first_token_p95_seconds": null, "observed_above_seconds": { "120": { "count": 4, "denominator": 120 }, "180": { "count": 1, "denominator": 120 }, "300": { "count": 1, "denominator": 120 }, "900": { "count": 0, "denominator": 120 }, "1800": { "count": 0, "denominator": 120 } }, "p25_seconds": 13.03906223201193, "p75_seconds": 37.6958039989986, "final_answer_completion": { "count": 120, "mean": 33.18850703368516, "median": 22.379449365500477, "p95": 86.87513911200222, "maximum": 326.8716572630219, "p25": 13.03906223201193, "p75": 37.6958039989986 }, "above_30_seconds": 46, "bins": [ 0, 14, 40, 20, 46 ] }, "output_tokens": { "count": 120, "mean": 3264.0833333333335, "median": 2285.5, "p95": 8363, "maximum": 27161 }, "input_tokens": { "count": 120, "mean": 123.20833333333333, "median": 115.0, "p95": 181, "maximum": 295 }, "thinking_tokens": { "count": 0, "mean": null, "median": null, "p95": null, "maximum": null }, "server_tokens_per_second": { "count": 0, "mean": null, "median": null, "p95": null, "maximum": null }, "first_token_seconds": { "count": 0, "mean": null, "median": null, "p95": null, "maximum": null }, "no_final": 0 }, { "configuration": "granite4.2-8b/off", "condition": "supplied_passage", "records": 120, "quality": { "planned": 120, "answers_returned": 120, "operational_failures": 0, "graded": 120, "exact_claim_scores": 120, "mean_claim_score": 88.825, "fully_correct": 63, "profile_assessed": 120, "mean_fully_met_claim_coverage": 0.8708333333333333, "answers_with_material_errors": 57, "material_error_assessed": 120, "pending_reviews": 1, "operationally_flagged_answers": 61, "output_ceiling_hits": 0, "thinking_marker_breaches": 61, "direct_failures": 0, "regenerated_answers": 0, "mean_output_tokens": 1685.0833333333333, "mean_client_seconds": 16.949836852439088 }, "timing": { "recorded_cases": 120, "complete_timing_cases": 120, "missing_or_partial_timing_cases": 0, "mean_seconds": 16.949836852439088, "median_seconds": 9.41112702449027, "p90_seconds": 33.4469473589852, "p95_seconds": 46.68501166198985, "p99_seconds": 94.5617580129765, "max_seconds": 152.06399614401744, "nonempty_finals": 120, "no_final_cases": 0, "output_ceiling_hits": 0, "retried_cases": 0, "request_failure_attempts": 0, "final_returned_median_seconds": 9.41112702449027, "final_returned_p95_seconds": 46.68501166198985, "first_token_timing_cases": 0, "first_token_p95_seconds": null, "observed_above_seconds": { "120": { "count": 1, "denominator": 120 }, "180": { "count": 0, "denominator": 120 }, "300": { "count": 0, "denominator": 120 }, "900": { "count": 0, "denominator": 120 }, "1800": { "count": 0, "denominator": 120 } }, "p25_seconds": 6.145904293021886, "p75_seconds": 19.536974838003516, "final_answer_completion": { "count": 120, "mean": 16.949836852439088, "median": 9.41112702449027, "p95": 46.68501166198985, "maximum": 152.06399614401744, "p25": 6.145904293021886, "p75": 19.536974838003516 }, "above_30_seconds": 18, "bins": [ 17, 44, 31, 10, 18 ] }, "output_tokens": { "count": 120, "mean": 1685.0833333333333, "median": 950.0, "p95": 4691, "maximum": 13886 }, "input_tokens": { "count": 120, "mean": 358.39166666666665, "median": 346.5, "p95": 578, "maximum": 1005 }, "thinking_tokens": { "count": 0, "mean": null, "median": null, "p95": null, "maximum": null }, "server_tokens_per_second": { "count": 0, "mean": null, "median": null, "p95": null, "maximum": null }, "first_token_seconds": { "count": 0, "mean": null, "median": null, "p95": null, "maximum": null }, "no_final": 0 }, { "configuration": "lfm2.5-8b-a1b/native", "condition": "no_source", "records": 120, "quality": { "planned": 120, "answers_returned": 120, "operational_failures": 0, "graded": 120, "exact_claim_scores": 120, "mean_claim_score": 27.625, "fully_correct": 0, "profile_assessed": 120, "mean_fully_met_claim_coverage": 0.21611111111111111, "answers_with_material_errors": 119, "material_error_assessed": 120, "pending_reviews": 2, "operationally_flagged_answers": 1, "output_ceiling_hits": 1, "thinking_marker_breaches": 0, "direct_failures": 1, "regenerated_answers": 0, "mean_output_tokens": 4046.925, "mean_client_seconds": 13.336282462099673 }, "timing": { "recorded_cases": 120, "complete_timing_cases": 120, "missing_or_partial_timing_cases": 0, "mean_seconds": 13.336282462099673, "median_seconds": 9.907918013996095, "p90_seconds": 22.2773051999975, "p95_seconds": 29.351412558025913, "p99_seconds": 56.42283325601602, "max_seconds": 112.16383835699526, "nonempty_finals": 119, "no_final_cases": 1, "output_ceiling_hits": 1, "retried_cases": 0, "request_failure_attempts": 0, "final_returned_median_seconds": 9.788082423998276, "final_returned_p95_seconds": 29.351412558025913, "first_token_timing_cases": 120, "first_token_p95_seconds": 0.116819, "observed_above_seconds": { "120": { "count": 0, "denominator": 120 }, "180": { "count": 0, "denominator": 120 }, "300": { "count": 0, "denominator": 120 }, "900": { "count": 0, "denominator": 120 }, "1800": { "count": 0, "denominator": 120 } }, "p25_seconds": 6.383746783976676, "p75_seconds": 17.538372928014724, "final_answer_completion": { "count": 119, "mean": 12.505798799117358, "median": 9.788082423998276, "p95": 29.351412558025913, "maximum": 56.42283325601602, "p25": 6.383746783976676, "p75": 17.538372928014724 }, "above_30_seconds": 5, "bins": [ 14, 46, 40, 15, 5 ] }, "output_tokens": { "count": 120, "mean": 4046.925, "median": 3037.5, "p95": 8904, "maximum": 32768 }, "input_tokens": { "count": 120, "mean": 124.6, "median": 116.0, "p95": 181, "maximum": 301 }, "thinking_tokens": { "count": 120, "mean": 3656.1, "median": 2584.0, "p95": 8328, "maximum": 32766 }, "server_tokens_per_second": { "count": 120, "mean": 311.49877122227093, "median": 310.5096004097762, "p95": 321.0085772548855, "maximum": 328.9777420320027 }, "first_token_seconds": { "count": 120, "mean": 0.05459571666666667, "median": 0.042054499999999995, "p95": 0.116819, "maximum": 0.152178 }, "no_final": 1 }, { "configuration": "lfm2.5-8b-a1b/native", "condition": "supplied_passage", "records": 120, "quality": { "planned": 120, "answers_returned": 120, "operational_failures": 0, "graded": 120, "exact_claim_scores": 120, "mean_claim_score": 71.40833333333333, "fully_correct": 32, "profile_assessed": 120, "mean_fully_met_claim_coverage": 0.6590277777777778, "answers_with_material_errors": 86, "material_error_assessed": 120, "pending_reviews": 3, "operationally_flagged_answers": 0, "output_ceiling_hits": 0, "thinking_marker_breaches": 0, "direct_failures": 0, "regenerated_answers": 0, "mean_output_tokens": 1729.875, "mean_client_seconds": 5.723916339183051 }, "timing": { "recorded_cases": 120, "complete_timing_cases": 120, "missing_or_partial_timing_cases": 0, "mean_seconds": 5.723916339183051, "median_seconds": 4.069626017007977, "p90_seconds": 10.44233370697475, "p95_seconds": 14.343253933999222, "p99_seconds": 22.32033113099169, "max_seconds": 46.16163652201067, "nonempty_finals": 120, "no_final_cases": 0, "output_ceiling_hits": 0, "retried_cases": 0, "request_failure_attempts": 0, "final_returned_median_seconds": 4.069626017007977, "final_returned_p95_seconds": 14.343253933999222, "first_token_timing_cases": 120, "first_token_p95_seconds": 0.128366, "observed_above_seconds": { "120": { "count": 0, "denominator": 120 }, "180": { "count": 0, "denominator": 120 }, "300": { "count": 0, "denominator": 120 }, "900": { "count": 0, "denominator": 120 }, "1800": { "count": 0, "denominator": 120 } }, "p25_seconds": 2.97586185898399, "p75_seconds": 5.651144048984861, "final_answer_completion": { "count": 120, "mean": 5.723916339183051, "median": 4.069626017007977, "p95": 14.343253933999222, "maximum": 46.16163652201067, "p25": 2.97586185898399, "p75": 5.651144048984861 }, "above_30_seconds": 1, "bins": [ 73, 33, 10, 3, 1 ] }, "output_tokens": { "count": 120, "mean": 1729.875, "median": 1235.5, "p95": 4414, "maximum": 14076 }, "input_tokens": { "count": 120, "mean": 342.3833333333333, "median": 332.0, "p95": 546, "maximum": 961 }, "thinking_tokens": { "count": 120, "mean": 1368.6833333333334, "median": 852.5, "p95": 3867, "maximum": 13773 }, "server_tokens_per_second": { "count": 120, "mean": 313.52707575931635, "median": 314.4757126761283, "p95": 320.8534097212549, "maximum": 324.18276172159875 }, "first_token_seconds": { "count": 120, "mean": 0.06747885, "median": 0.053706000000000004, "p95": 0.128366, "maximum": 0.181173 }, "no_final": 0 }, { "configuration": "qwen3.5-9b/off", "condition": "no_source", "records": 120, "quality": { "planned": 120, "answers_returned": 120, "operational_failures": 0, "graded": 120, "exact_claim_scores": 120, "mean_claim_score": 51.88333333333333, "fully_correct": 5, "profile_assessed": 120, "mean_fully_met_claim_coverage": 0.45944444444444443, "answers_with_material_errors": 115, "material_error_assessed": 120, "pending_reviews": 10, "operationally_flagged_answers": 13, "output_ceiling_hits": 0, "thinking_marker_breaches": 13, "direct_failures": 0, "regenerated_answers": 0, "mean_output_tokens": 1701.375, "mean_client_seconds": 16.72947611001776 }, "timing": { "recorded_cases": 120, "complete_timing_cases": 120, "missing_or_partial_timing_cases": 0, "mean_seconds": 16.72947611001776, "median_seconds": 7.620644941518549, "p90_seconds": 52.43825192100485, "p95_seconds": 76.44514443399385, "p99_seconds": 107.81710012498661, "max_seconds": 125.00687698199181, "nonempty_finals": 120, "no_final_cases": 0, "output_ceiling_hits": 0, "retried_cases": 0, "request_failure_attempts": 0, "final_returned_median_seconds": 7.620644941518549, "final_returned_p95_seconds": 76.44514443399385, "first_token_timing_cases": 0, "first_token_p95_seconds": null, "observed_above_seconds": { "120": { "count": 1, "denominator": 120 }, "180": { "count": 0, "denominator": 120 }, "300": { "count": 0, "denominator": 120 }, "900": { "count": 0, "denominator": 120 }, "1800": { "count": 0, "denominator": 120 } }, "p25_seconds": 5.218570884026121, "p75_seconds": 11.776204310008325, "final_answer_completion": { "count": 120, "mean": 16.72947611001776, "median": 7.620644941518549, "p95": 76.44514443399385, "maximum": 125.00687698199181, "p25": 5.218570884026121, "p75": 11.776204310008325 }, "above_30_seconds": 17, "bins": [ 23, 62, 14, 4, 17 ] }, "output_tokens": { "count": 120, "mean": 1701.375, "median": 766.5, "p95": 7835, "maximum": 12512 }, "input_tokens": { "count": 120, "mean": 124.375, "median": 116.0, "p95": 182, "maximum": 297 }, "thinking_tokens": { "count": 0, "mean": null, "median": null, "p95": null, "maximum": null }, "server_tokens_per_second": { "count": 0, "mean": null, "median": null, "p95": null, "maximum": null }, "first_token_seconds": { "count": 0, "mean": null, "median": null, "p95": null, "maximum": null }, "no_final": 0 }, { "configuration": "qwen3.5-9b/off", "condition": "supplied_passage", "records": 120, "quality": { "planned": 120, "answers_returned": 120, "operational_failures": 0, "graded": 120, "exact_claim_scores": 120, "mean_claim_score": 89.24166666666666, "fully_correct": 49, "profile_assessed": 116, "mean_fully_met_claim_coverage": 0.8701388888888889, "answers_with_material_errors": 63, "material_error_assessed": 116, "pending_reviews": 9, "operationally_flagged_answers": 3, "output_ceiling_hits": 0, "thinking_marker_breaches": 3, "direct_failures": 0, "regenerated_answers": 0, "mean_output_tokens": 749.725, "mean_client_seconds": 7.4671080245427826 }, "timing": { "recorded_cases": 120, "complete_timing_cases": 120, "missing_or_partial_timing_cases": 0, "mean_seconds": 7.4671080245427826, "median_seconds": 4.855398231986328, "p90_seconds": 13.177598420996219, "p95_seconds": 19.39523829999962, "p99_seconds": 60.62675652699545, "max_seconds": 62.68317884899443, "nonempty_finals": 120, "no_final_cases": 0, "output_ceiling_hits": 0, "retried_cases": 0, "request_failure_attempts": 0, "final_returned_median_seconds": 4.855398231986328, "final_returned_p95_seconds": 19.39523829999962, "first_token_timing_cases": 0, "first_token_p95_seconds": null, "observed_above_seconds": { "120": { "count": 0, "denominator": 120 }, "180": { "count": 0, "denominator": 120 }, "300": { "count": 0, "denominator": 120 }, "900": { "count": 0, "denominator": 120 }, "1800": { "count": 0, "denominator": 120 } }, "p25_seconds": 3.7374641920032445, "p75_seconds": 6.772200307023013, "final_answer_completion": { "count": 120, "mean": 7.4671080245427826, "median": 4.855398231986328, "p95": 19.39523829999962, "maximum": 62.68317884899443, "p25": 3.7374641920032445, "p75": 6.772200307023013 }, "above_30_seconds": 4, "bins": [ 62, 43, 10, 1, 4 ] }, "output_tokens": { "count": 120, "mean": 749.725, "median": 486.0, "p95": 1979, "maximum": 6445 }, "input_tokens": { "count": 120, "mean": 349.78333333333336, "median": 337.5, "p95": 556, "maximum": 964 }, "thinking_tokens": { "count": 0, "mean": null, "median": null, "p95": null, "maximum": null }, "server_tokens_per_second": { "count": 0, "mean": null, "median": null, "p95": null, "maximum": null }, "first_token_seconds": { "count": 0, "mean": null, "median": null, "p95": null, "maximum": null }, "no_final": 0 }, { "configuration": "qwen3.5-9b/on", "condition": "no_source", "records": 120, "quality": { "planned": 120, "answers_returned": 120, "operational_failures": 0, "graded": 120, "exact_claim_scores": 120, "mean_claim_score": 45.3, "fully_correct": 8, "profile_assessed": 119, "mean_fully_met_claim_coverage": 0.4038888888888889, "answers_with_material_errors": 110, "material_error_assessed": 119, "pending_reviews": 22, "operationally_flagged_answers": 2, "output_ceiling_hits": 2, "thinking_marker_breaches": 0, "direct_failures": 1, "regenerated_answers": 0, "mean_output_tokens": 9077.375, "mean_client_seconds": 89.8962851640749 }, "timing": { "recorded_cases": 120, "complete_timing_cases": 120, "missing_or_partial_timing_cases": 0, "mean_seconds": 89.8962851640749, "median_seconds": 83.022500837993, "p90_seconds": 138.56031002098462, "p95_seconds": 158.42903493501944, "p99_seconds": 339.06330648000585, "max_seconds": 339.0751682180271, "nonempty_finals": 119, "no_final_cases": 1, "output_ceiling_hits": 2, "retried_cases": 0, "request_failure_attempts": 0, "final_returned_median_seconds": 82.89199672199902, "final_returned_p95_seconds": 158.42903493501944, "first_token_timing_cases": 0, "first_token_p95_seconds": null, "observed_above_seconds": { "120": { "count": 20, "denominator": 120 }, "180": { "count": 2, "denominator": 120 }, "300": { "count": 2, "denominator": 120 }, "900": { "count": 0, "denominator": 120 }, "1800": { "count": 0, "denominator": 120 } }, "p25_seconds": 67.03282110000146, "p75_seconds": 98.99658961599926, "final_answer_completion": { "count": 119, "mean": 87.80244464881497, "median": 82.89199672199902, "p95": 158.42903493501944, "maximum": 339.0751682180271, "p25": 67.03282110000146, "p75": 98.99658961599926 }, "above_30_seconds": 116, "bins": [ 0, 1, 1, 2, 116 ] }, "output_tokens": { "count": 120, "mean": 9077.375, "median": 8419.0, "p95": 15804, "maximum": 32768 }, "input_tokens": { "count": 120, "mean": 122.375, "median": 114.0, "p95": 180, "maximum": 295 }, "thinking_tokens": { "count": 0, "mean": null, "median": null, "p95": null, "maximum": null }, "server_tokens_per_second": { "count": 0, "mean": null, "median": null, "p95": null, "maximum": null }, "first_token_seconds": { "count": 0, "mean": null, "median": null, "p95": null, "maximum": null }, "no_final": 1 }, { "configuration": "qwen3.5-9b/on", "condition": "supplied_passage", "records": 120, "quality": { "planned": 120, "answers_returned": 120, "operational_failures": 0, "graded": 120, "exact_claim_scores": 120, "mean_claim_score": 96.625, "fully_correct": 99, "profile_assessed": 120, "mean_fully_met_claim_coverage": 0.9554166666666667, "answers_with_material_errors": 19, "material_error_assessed": 120, "pending_reviews": 0, "operationally_flagged_answers": 0, "output_ceiling_hits": 0, "thinking_marker_breaches": 0, "direct_failures": 0, "regenerated_answers": 0, "mean_output_tokens": 3577.925, "mean_client_seconds": 35.111306433840944 }, "timing": { "recorded_cases": 120, "complete_timing_cases": 120, "missing_or_partial_timing_cases": 0, "mean_seconds": 35.111306433840944, "median_seconds": 26.66372540150769, "p90_seconds": 66.45543938499759, "p95_seconds": 77.80049887401401, "p99_seconds": 138.38800971899764, "max_seconds": 145.64245316997403, "nonempty_finals": 120, "no_final_cases": 0, "output_ceiling_hits": 0, "retried_cases": 0, "request_failure_attempts": 0, "final_returned_median_seconds": 26.66372540150769, "final_returned_p95_seconds": 77.80049887401401, "first_token_timing_cases": 0, "first_token_p95_seconds": null, "observed_above_seconds": { "120": { "count": 2, "denominator": 120 }, "180": { "count": 0, "denominator": 120 }, "300": { "count": 0, "denominator": 120 }, "900": { "count": 0, "denominator": 120 }, "1800": { "count": 0, "denominator": 120 } }, "p25_seconds": 19.138037466997048, "p75_seconds": 40.515594926982885, "final_answer_completion": { "count": 120, "mean": 35.111306433840944, "median": 26.66372540150769, "p95": 77.80049887401401, "maximum": 145.64245316997403, "p25": 19.138037466997048, "p75": 40.515594926982885 }, "above_30_seconds": 53, "bins": [ 0, 2, 32, 33, 53 ] }, "output_tokens": { "count": 120, "mean": 3577.925, "median": 2712.5, "p95": 7852, "maximum": 14684 }, "input_tokens": { "count": 120, "mean": 347.78333333333336, "median": 335.5, "p95": 554, "maximum": 962 }, "thinking_tokens": { "count": 0, "mean": null, "median": null, "p95": null, "maximum": null }, "server_tokens_per_second": { "count": 0, "mean": null, "median": null, "p95": null, "maximum": null }, "first_token_seconds": { "count": 0, "mean": null, "median": null, "p95": null, "maximum": null }, "no_final": 0 }, { "configuration": "granite4.2-30b/on", "condition": "no_source", "records": 120, "quality": { "planned": 120, "answers_returned": 120, "operational_failures": 0, "graded": 120, "exact_claim_scores": 120, "mean_claim_score": 45.55, "fully_correct": 1, "profile_assessed": 120, "mean_fully_met_claim_coverage": 0.4122222222222222, "answers_with_material_errors": 119, "material_error_assessed": 120, "pending_reviews": 12, "operationally_flagged_answers": 0, "output_ceiling_hits": 0, "thinking_marker_breaches": 0, "direct_failures": 0, "regenerated_answers": 0, "mean_output_tokens": 2336.325, "mean_client_seconds": 102.97626134025 }, "timing": { "recorded_cases": 120, "complete_timing_cases": 120, "missing_or_partial_timing_cases": 0, "mean_seconds": 102.97626134025, "median_seconds": 71.84132410149323, "p90_seconds": 179.39653907099273, "p95_seconds": 239.7329822370084, "p99_seconds": 618.5921809279826, "max_seconds": 794.8344104519929, "nonempty_finals": 120, "no_final_cases": 0, "output_ceiling_hits": 0, "retried_cases": 0, "request_failure_attempts": 0, "final_returned_median_seconds": 71.84132410149323, "final_returned_p95_seconds": 239.7329822370084, "first_token_timing_cases": 0, "first_token_p95_seconds": null, "observed_above_seconds": { "120": { "count": 22, "denominator": 120 }, "180": { "count": 12, "denominator": 120 }, "300": { "count": 4, "denominator": 120 }, "900": { "count": 0, "denominator": 120 }, "1800": { "count": 0, "denominator": 120 } }, "p25_seconds": 55.852897441975074, "p75_seconds": 97.83876022600452, "final_answer_completion": { "count": 120, "mean": 102.97626134025, "median": 71.84132410149323, "p95": 239.7329822370084, "maximum": 794.8344104519929, "p25": 55.852897441975074, "p75": 97.83876022600452 }, "above_30_seconds": 119, "bins": [ 0, 0, 0, 1, 119 ] }, "output_tokens": { "count": 120, "mean": 2336.325, "median": 1860.5, "p95": 4890, "maximum": 11321 }, "input_tokens": { "count": 120, "mean": 123.20833333333333, "median": 115.0, "p95": 181, "maximum": 295 }, "thinking_tokens": { "count": 0, "mean": null, "median": null, "p95": null, "maximum": null }, "server_tokens_per_second": { "count": 0, "mean": null, "median": null, "p95": null, "maximum": null }, "first_token_seconds": { "count": 0, "mean": null, "median": null, "p95": null, "maximum": null }, "no_final": 0 }, { "configuration": "granite4.2-30b/on", "condition": "supplied_passage", "records": 120, "quality": { "planned": 120, "answers_returned": 120, "operational_failures": 0, "graded": 120, "exact_claim_scores": 120, "mean_claim_score": 95.91666666666667, "fully_correct": 69, "profile_assessed": 119, "mean_fully_met_claim_coverage": 0.9463888888888888, "answers_with_material_errors": 49, "material_error_assessed": 119, "pending_reviews": 3, "operationally_flagged_answers": 0, "output_ceiling_hits": 0, "thinking_marker_breaches": 0, "direct_failures": 0, "regenerated_answers": 0, "mean_output_tokens": 1493.2916666666667, "mean_client_seconds": 64.74146516768451 }, "timing": { "recorded_cases": 120, "complete_timing_cases": 120, "missing_or_partial_timing_cases": 0, "mean_seconds": 64.74146516768451, "median_seconds": 52.041306866987725, "p90_seconds": 80.43559403499239, "p95_seconds": 97.29196825801046, "p99_seconds": 559.1574552730017, "max_seconds": 666.3046439729806, "nonempty_finals": 120, "no_final_cases": 0, "output_ceiling_hits": 0, "retried_cases": 0, "request_failure_attempts": 0, "final_returned_median_seconds": 52.041306866987725, "final_returned_p95_seconds": 97.29196825801046, "first_token_timing_cases": 0, "first_token_p95_seconds": null, "observed_above_seconds": { "120": { "count": 3, "denominator": 120 }, "180": { "count": 2, "denominator": 120 }, "300": { "count": 2, "denominator": 120 }, "900": { "count": 0, "denominator": 120 }, "1800": { "count": 0, "denominator": 120 } }, "p25_seconds": 43.510723234008765, "p75_seconds": 62.60568271597731, "final_answer_completion": { "count": 120, "mean": 64.74146516768451, "median": 52.041306866987725, "p95": 97.29196825801046, "maximum": 666.3046439729806, "p25": 43.510723234008765, "p75": 62.60568271597731 }, "above_30_seconds": 118, "bins": [ 0, 0, 0, 2, 118 ] }, "output_tokens": { "count": 120, "mean": 1493.2916666666667, "median": 1310.5, "p95": 2243, "maximum": 9304 }, "input_tokens": { "count": 120, "mean": 358.39166666666665, "median": 346.5, "p95": 578, "maximum": 1005 }, "thinking_tokens": { "count": 0, "mean": null, "median": null, "p95": null, "maximum": null }, "server_tokens_per_second": { "count": 0, "mean": null, "median": null, "p95": null, "maximum": null }, "first_token_seconds": { "count": 0, "mean": null, "median": null, "p95": null, "maximum": null }, "no_final": 0 }, { "configuration": "granite4.2-30b/off", "condition": "no_source", "records": 120, "quality": { "planned": 120, "answers_returned": 120, "operational_failures": 0, "graded": 120, "exact_claim_scores": 120, "mean_claim_score": 44.06666666666667, "fully_correct": 0, "profile_assessed": 120, "mean_fully_met_claim_coverage": 0.3595833333333333, "answers_with_material_errors": 120, "material_error_assessed": 120, "pending_reviews": 7, "operationally_flagged_answers": 1, "output_ceiling_hits": 0, "thinking_marker_breaches": 1, "direct_failures": 0, "regenerated_answers": 0, "mean_output_tokens": 1406.5666666666666, "mean_client_seconds": 60.31475436160836 }, "timing": { "recorded_cases": 120, "complete_timing_cases": 120, "missing_or_partial_timing_cases": 0, "mean_seconds": 60.31475436160836, "median_seconds": 40.138928431493696, "p90_seconds": 99.73870880898903, "p95_seconds": 116.4921896349988, "p99_seconds": 505.9988825870096, "max_seconds": 515.3523483290046, "nonempty_finals": 120, "no_final_cases": 0, "output_ceiling_hits": 0, "retried_cases": 0, "request_failure_attempts": 0, "final_returned_median_seconds": 40.138928431493696, "final_returned_p95_seconds": 116.4921896349988, "first_token_timing_cases": 0, "first_token_p95_seconds": null, "observed_above_seconds": { "120": { "count": 6, "denominator": 120 }, "180": { "count": 4, "denominator": 120 }, "300": { "count": 4, "denominator": 120 }, "900": { "count": 0, "denominator": 120 }, "1800": { "count": 0, "denominator": 120 } }, "p25_seconds": 26.867685119999805, "p75_seconds": 57.83864623098634, "final_answer_completion": { "count": 120, "mean": 60.31475436160836, "median": 40.138928431493696, "p95": 116.4921896349988, "maximum": 515.3523483290046, "p25": 26.867685119999805, "p75": 57.83864623098634 }, "above_30_seconds": 80, "bins": [ 0, 0, 11, 29, 80 ] }, "output_tokens": { "count": 120, "mean": 1406.5666666666666, "median": 1069.5, "p95": 2710, "maximum": 8459 }, "input_tokens": { "count": 120, "mean": 123.20833333333333, "median": 115.0, "p95": 181, "maximum": 295 }, "thinking_tokens": { "count": 0, "mean": null, "median": null, "p95": null, "maximum": null }, "server_tokens_per_second": { "count": 0, "mean": null, "median": null, "p95": null, "maximum": null }, "first_token_seconds": { "count": 0, "mean": null, "median": null, "p95": null, "maximum": null }, "no_final": 0 }, { "configuration": "granite4.2-30b/off", "condition": "supplied_passage", "records": 120, "quality": { "planned": 120, "answers_returned": 120, "operational_failures": 0, "graded": 120, "exact_claim_scores": 120, "mean_claim_score": 95.24166666666666, "fully_correct": 62, "profile_assessed": 115, "mean_fully_met_claim_coverage": 0.9344444444444444, "answers_with_material_errors": 52, "material_error_assessed": 115, "pending_reviews": 8, "operationally_flagged_answers": 1, "output_ceiling_hits": 0, "thinking_marker_breaches": 1, "direct_failures": 0, "regenerated_answers": 0, "mean_output_tokens": 825.6416666666667, "mean_client_seconds": 33.9371785978173 }, "timing": { "recorded_cases": 120, "complete_timing_cases": 120, "missing_or_partial_timing_cases": 0, "mean_seconds": 33.9371785978173, "median_seconds": 27.0714332810021, "p90_seconds": 56.9543606749794, "p95_seconds": 72.32749069301644, "p99_seconds": 120.64004565699724, "max_seconds": 157.4918767880008, "nonempty_finals": 120, "no_final_cases": 0, "output_ceiling_hits": 0, "retried_cases": 0, "request_failure_attempts": 0, "final_returned_median_seconds": 27.0714332810021, "final_returned_p95_seconds": 72.32749069301644, "first_token_timing_cases": 0, "first_token_p95_seconds": null, "observed_above_seconds": { "120": { "count": 2, "denominator": 120 }, "180": { "count": 0, "denominator": 120 }, "300": { "count": 0, "denominator": 120 }, "900": { "count": 0, "denominator": 120 }, "1800": { "count": 0, "denominator": 120 } }, "p25_seconds": 20.237421446974622, "p75_seconds": 39.095746990991756, "final_answer_completion": { "count": 120, "mean": 33.9371785978173, "median": 27.0714332810021, "p95": 72.32749069301644, "maximum": 157.4918767880008, "p25": 20.237421446974622, "p75": 39.095746990991756 }, "above_30_seconds": 51, "bins": [ 0, 2, 26, 41, 51 ] }, "output_tokens": { "count": 120, "mean": 825.6416666666667, "median": 682.5, "p95": 1731, "maximum": 3095 }, "input_tokens": { "count": 120, "mean": 358.39166666666665, "median": 346.5, "p95": 578, "maximum": 1005 }, "thinking_tokens": { "count": 0, "mean": null, "median": null, "p95": null, "maximum": null }, "server_tokens_per_second": { "count": 0, "mean": null, "median": null, "p95": null, "maximum": null }, "first_token_seconds": { "count": 0, "mean": null, "median": null, "p95": null, "maximum": null }, "no_final": 0 }, { "configuration": "lfm2.5-24b-a2b/native", "condition": "no_source", "records": 120, "quality": { "planned": 120, "answers_returned": 120, "operational_failures": 0, "graded": 120, "exact_claim_scores": 120, "mean_claim_score": 41.925, "fully_correct": 7, "profile_assessed": 120, "mean_fully_met_claim_coverage": 0.3566666666666667, "answers_with_material_errors": 113, "material_error_assessed": 120, "pending_reviews": 2, "operationally_flagged_answers": 0, "output_ceiling_hits": 0, "thinking_marker_breaches": 0, "direct_failures": 0, "regenerated_answers": 0, "mean_output_tokens": 5959.275, "mean_client_seconds": 28.164994620691385 }, "timing": { "recorded_cases": 120, "complete_timing_cases": 120, "missing_or_partial_timing_cases": 0, "mean_seconds": 28.164994620691385, "median_seconds": 23.837354609480826, "p90_seconds": 50.87014818098396, "p95_seconds": 65.94923277699854, "p99_seconds": 94.2002344270004, "max_seconds": 102.41152738500386, "nonempty_finals": 120, "no_final_cases": 0, "output_ceiling_hits": 0, "retried_cases": 0, "request_failure_attempts": 0, "final_returned_median_seconds": 23.837354609480826, "final_returned_p95_seconds": 65.94923277699854, "first_token_timing_cases": 120, "first_token_p95_seconds": 0.112903, "observed_above_seconds": { "120": { "count": 0, "denominator": 120 }, "180": { "count": 0, "denominator": 120 }, "300": { "count": 0, "denominator": 120 }, "900": { "count": 0, "denominator": 120 }, "1800": { "count": 0, "denominator": 120 } }, "p25_seconds": 15.847320464003133, "p75_seconds": 34.345355907978956, "final_answer_completion": { "count": 120, "mean": 28.164994620691385, "median": 23.837354609480826, "p95": 65.94923277699854, "maximum": 102.41152738500386, "p25": 15.847320464003133, "p75": 34.345355907978956 }, "above_30_seconds": 40, "bins": [ 0, 5, 44, 31, 40 ] }, "output_tokens": { "count": 120, "mean": 5959.275, "median": 5078.0, "p95": 13817, "maximum": 20819 }, "input_tokens": { "count": 120, "mean": 125.6, "median": 117.0, "p95": 182, "maximum": 302 }, "thinking_tokens": { "count": 120, "mean": 5615.525, "median": 4799.5, "p95": 13256, "maximum": 20484 }, "server_tokens_per_second": { "count": 120, "mean": 214.9071057102925, "median": 215.82402574523343, "p95": 220.9844143580983, "maximum": 223.11112527689684 }, "first_token_seconds": { "count": 120, "mean": 0.09672036666666667, "median": 0.09411, "p95": 0.112903, "maximum": 0.444505 }, "no_final": 0 }, { "configuration": "lfm2.5-24b-a2b/native", "condition": "supplied_passage", "records": 120, "quality": { "planned": 120, "answers_returned": 120, "operational_failures": 0, "graded": 120, "exact_claim_scores": 120, "mean_claim_score": 95.075, "fully_correct": 73, "profile_assessed": 119, "mean_fully_met_claim_coverage": 0.9359722222222222, "answers_with_material_errors": 45, "material_error_assessed": 119, "pending_reviews": 1, "operationally_flagged_answers": 0, "output_ceiling_hits": 0, "thinking_marker_breaches": 0, "direct_failures": 0, "regenerated_answers": 0, "mean_output_tokens": 2868.525, "mean_client_seconds": 13.50058852727525 }, "timing": { "recorded_cases": 120, "complete_timing_cases": 120, "missing_or_partial_timing_cases": 0, "mean_seconds": 13.50058852727525, "median_seconds": 10.760558850510279, "p90_seconds": 22.904596676991787, "p95_seconds": 29.947083480015863, "p99_seconds": 44.834318874025485, "max_seconds": 83.07041241799016, "nonempty_finals": 120, "no_final_cases": 0, "output_ceiling_hits": 0, "retried_cases": 0, "request_failure_attempts": 0, "final_returned_median_seconds": 10.760558850510279, "final_returned_p95_seconds": 29.947083480015863, "first_token_timing_cases": 120, "first_token_p95_seconds": 0.165519, "observed_above_seconds": { "120": { "count": 0, "denominator": 120 }, "180": { "count": 0, "denominator": 120 }, "300": { "count": 0, "denominator": 120 }, "900": { "count": 0, "denominator": 120 }, "1800": { "count": 0, "denominator": 120 } }, "p25_seconds": 7.902464801009046, "p75_seconds": 14.9789040129981, "final_answer_completion": { "count": 120, "mean": 13.50058852727525, "median": 10.760558850510279, "p95": 29.947083480015863, "maximum": 83.07041241799016, "p25": 7.902464801009046, "p75": 14.9789040129981 }, "above_30_seconds": 6, "bins": [ 1, 54, 48, 11, 6 ] }, "output_tokens": { "count": 120, "mean": 2868.525, "median": 2335.0, "p95": 6361, "maximum": 17493 }, "input_tokens": { "count": 120, "mean": 343.3833333333333, "median": 333.0, "p95": 547, "maximum": 962 }, "thinking_tokens": { "count": 120, "mean": 2497.1583333333333, "median": 1901.0, "p95": 5920, "maximum": 16991 }, "server_tokens_per_second": { "count": 120, "mean": 216.69947031216475, "median": 217.2062374712121, "p95": 221.5937230035335, "maximum": 222.83188274353992 }, "first_token_seconds": { "count": 120, "mean": 0.12212041666666666, "median": 0.1168755, "p95": 0.165519, "maximum": 0.255921 }, "no_final": 0 }, { "configuration": "qwen3.8-27b/off", "condition": "no_source", "records": 120, "quality": { "planned": 120, "answers_returned": 120, "operational_failures": 0, "graded": 120, "exact_claim_scores": 120, "mean_claim_score": 53.666666666666664, "fully_correct": 6, "profile_assessed": 119, "mean_fully_met_claim_coverage": 0.47333333333333333, "answers_with_material_errors": 112, "material_error_assessed": 119, "pending_reviews": 14, "operationally_flagged_answers": 0, "output_ceiling_hits": 0, "thinking_marker_breaches": 0, "direct_failures": 0, "regenerated_answers": 0, "mean_output_tokens": 836.3, "mean_client_seconds": 16.345276489981188 }, "timing": { "recorded_cases": 120, "complete_timing_cases": 120, "missing_or_partial_timing_cases": 0, "mean_seconds": 16.345276489981188, "median_seconds": 14.209506271989085, "p90_seconds": 22.84523086398258, "p95_seconds": 28.924308763002045, "p99_seconds": 56.505335421999916, "max_seconds": 96.13749228202505, "nonempty_finals": 120, "no_final_cases": 0, "output_ceiling_hits": 0, "retried_cases": 0, "request_failure_attempts": 0, "final_returned_median_seconds": 14.209506271989085, "final_returned_p95_seconds": 28.924308763002045, "first_token_timing_cases": 0, "first_token_p95_seconds": null, "observed_above_seconds": { "120": { "count": 0, "denominator": 120 }, "180": { "count": 0, "denominator": 120 }, "300": { "count": 0, "denominator": 120 }, "900": { "count": 0, "denominator": 120 }, "1800": { "count": 0, "denominator": 120 } }, "p25_seconds": 11.339582551998319, "p75_seconds": 18.453588450996904, "final_answer_completion": { "count": 120, "mean": 16.345276489981188, "median": 14.209506271989085, "p95": 28.924308763002045, "maximum": 96.13749228202505, "p25": 11.339582551998319, "p75": 18.453588450996904 }, "above_30_seconds": 6, "bins": [ 0, 21, 74, 19, 6 ] }, "output_tokens": { "count": 120, "mean": 836.3, "median": 730.0, "p95": 1531, "maximum": 5050 }, "input_tokens": { "count": 120, "mean": 124.375, "median": 116.0, "p95": 182, "maximum": 297 }, "thinking_tokens": { "count": 0, "mean": null, "median": null, "p95": null, "maximum": null }, "server_tokens_per_second": { "count": 0, "mean": null, "median": null, "p95": null, "maximum": null }, "first_token_seconds": { "count": 0, "mean": null, "median": null, "p95": null, "maximum": null }, "no_final": 0 }, { "configuration": "qwen3.8-27b/off", "condition": "supplied_passage", "records": 120, "quality": { "planned": 120, "answers_returned": 120, "operational_failures": 0, "graded": 120, "exact_claim_scores": 120, "mean_claim_score": 95.15833333333333, "fully_correct": 90, "profile_assessed": 120, "mean_fully_met_claim_coverage": 0.9366666666666666, "answers_with_material_errors": 28, "material_error_assessed": 120, "pending_reviews": 1, "operationally_flagged_answers": 0, "output_ceiling_hits": 0, "thinking_marker_breaches": 0, "direct_failures": 0, "regenerated_answers": 0, "mean_output_tokens": 456.84166666666664, "mean_client_seconds": 8.58779860207481 }, "timing": { "recorded_cases": 120, "complete_timing_cases": 120, "missing_or_partial_timing_cases": 0, "mean_seconds": 8.58779860207481, "median_seconds": 7.705773909488926, "p90_seconds": 13.217989357013721, "p95_seconds": 14.14093909598887, "p99_seconds": 28.765928587992676, "max_seconds": 33.19555814098567, "nonempty_finals": 120, "no_final_cases": 0, "output_ceiling_hits": 0, "retried_cases": 0, "request_failure_attempts": 0, "final_returned_median_seconds": 7.705773909488926, "final_returned_p95_seconds": 14.14093909598887, "first_token_timing_cases": 0, "first_token_p95_seconds": null, "observed_above_seconds": { "120": { "count": 0, "denominator": 120 }, "180": { "count": 0, "denominator": 120 }, "300": { "count": 0, "denominator": 120 }, "900": { "count": 0, "denominator": 120 }, "1800": { "count": 0, "denominator": 120 } }, "p25_seconds": 5.67505301698111, "p75_seconds": 10.347955735982396, "final_answer_completion": { "count": 120, "mean": 8.58779860207481, "median": 7.705773909488926, "p95": 14.14093909598887, "maximum": 33.19555814098567, "p25": 5.67505301698111, "p75": 10.347955735982396 }, "above_30_seconds": 1, "bins": [ 20, 67, 30, 2, 1 ] }, "output_tokens": { "count": 120, "mean": 456.84166666666664, "median": 406.5, "p95": 817, "maximum": 1990 }, "input_tokens": { "count": 120, "mean": 349.78333333333336, "median": 337.5, "p95": 556, "maximum": 964 }, "thinking_tokens": { "count": 0, "mean": null, "median": null, "p95": null, "maximum": null }, "server_tokens_per_second": { "count": 0, "mean": null, "median": null, "p95": null, "maximum": null }, "first_token_seconds": { "count": 0, "mean": null, "median": null, "p95": null, "maximum": null }, "no_final": 0 }, { "configuration": "qwen3.8-27b/on", "condition": "no_source", "records": 120, "quality": { "planned": 120, "answers_returned": 120, "operational_failures": 0, "graded": 120, "exact_claim_scores": 120, "mean_claim_score": 53.875, "fully_correct": 12, "profile_assessed": 119, "mean_fully_met_claim_coverage": 0.4751388888888889, "answers_with_material_errors": 106, "material_error_assessed": 119, "pending_reviews": 11, "operationally_flagged_answers": 0, "output_ceiling_hits": 0, "thinking_marker_breaches": 0, "direct_failures": 0, "regenerated_answers": 0, "mean_output_tokens": 1735.025, "mean_client_seconds": 31.433135336017827 }, "timing": { "recorded_cases": 120, "complete_timing_cases": 120, "missing_or_partial_timing_cases": 0, "mean_seconds": 31.433135336017827, "median_seconds": 27.739497161994223, "p90_seconds": 49.86874774500029, "p95_seconds": 68.27020600097603, "p99_seconds": 104.80797892299597, "max_seconds": 105.94480921301874, "nonempty_finals": 120, "no_final_cases": 0, "output_ceiling_hits": 0, "retried_cases": 0, "request_failure_attempts": 0, "final_returned_median_seconds": 27.739497161994223, "final_returned_p95_seconds": 68.27020600097603, "first_token_timing_cases": 0, "first_token_p95_seconds": null, "observed_above_seconds": { "120": { "count": 0, "denominator": 120 }, "180": { "count": 0, "denominator": 120 }, "300": { "count": 0, "denominator": 120 }, "900": { "count": 0, "denominator": 120 }, "1800": { "count": 0, "denominator": 120 } }, "p25_seconds": 20.84977665601764, "p75_seconds": 37.69980688201031, "final_answer_completion": { "count": 120, "mean": 31.433135336017827, "median": 27.739497161994223, "p95": 68.27020600097603, "maximum": 105.94480921301874, "p25": 20.84977665601764, "p75": 37.69980688201031 }, "above_30_seconds": 52, "bins": [ 0, 1, 27, 40, 52 ] }, "output_tokens": { "count": 120, "mean": 1735.025, "median": 1513.0, "p95": 4129, "maximum": 6301 }, "input_tokens": { "count": 120, "mean": 148.375, "median": 140.0, "p95": 206, "maximum": 321 }, "thinking_tokens": { "count": 0, "mean": null, "median": null, "p95": null, "maximum": null }, "server_tokens_per_second": { "count": 0, "mean": null, "median": null, "p95": null, "maximum": null }, "first_token_seconds": { "count": 0, "mean": null, "median": null, "p95": null, "maximum": null }, "no_final": 0 }, { "configuration": "qwen3.8-27b/on", "condition": "supplied_passage", "records": 120, "quality": { "planned": 120, "answers_returned": 120, "operational_failures": 0, "graded": 120, "exact_claim_scores": 120, "mean_claim_score": 98.41666666666667, "fully_correct": 102, "profile_assessed": 119, "mean_fully_met_claim_coverage": 0.98125, "answers_with_material_errors": 16, "material_error_assessed": 119, "pending_reviews": 2, "operationally_flagged_answers": 0, "output_ceiling_hits": 0, "thinking_marker_breaches": 0, "direct_failures": 0, "regenerated_answers": 0, "mean_output_tokens": 709.6416666666667, "mean_client_seconds": 12.722110486893508 }, "timing": { "recorded_cases": 120, "complete_timing_cases": 120, "missing_or_partial_timing_cases": 0, "mean_seconds": 12.722110486893508, "median_seconds": 11.276302856014809, "p90_seconds": 18.20637133301352, "p95_seconds": 22.271306298003765, "p99_seconds": 32.96603372899699, "max_seconds": 44.50844476700877, "nonempty_finals": 120, "no_final_cases": 0, "output_ceiling_hits": 0, "retried_cases": 0, "request_failure_attempts": 0, "final_returned_median_seconds": 11.276302856014809, "final_returned_p95_seconds": 22.271306298003765, "first_token_timing_cases": 0, "first_token_p95_seconds": null, "observed_above_seconds": { "120": { "count": 0, "denominator": 120 }, "180": { "count": 0, "denominator": 120 }, "300": { "count": 0, "denominator": 120 }, "900": { "count": 0, "denominator": 120 }, "1800": { "count": 0, "denominator": 120 } }, "p25_seconds": 9.815797462011687, "p75_seconds": 14.364686550979968, "final_answer_completion": { "count": 120, "mean": 12.722110486893508, "median": 11.276302856014809, "p95": 22.271306298003765, "maximum": 44.50844476700877, "p25": 9.815797462011687, "p75": 14.364686550979968 }, "above_30_seconds": 2, "bins": [ 0, 33, 77, 8, 2 ] }, "output_tokens": { "count": 120, "mean": 709.6416666666667, "median": 632.0, "p95": 1247, "maximum": 2659 }, "input_tokens": { "count": 120, "mean": 373.78333333333336, "median": 361.5, "p95": 580, "maximum": 988 }, "thinking_tokens": { "count": 0, "mean": null, "median": null, "p95": null, "maximum": null }, "server_tokens_per_second": { "count": 0, "mean": null, "median": null, "p95": null, "maximum": null }, "first_token_seconds": { "count": 0, "mean": null, "median": null, "p95": null, "maximum": null }, "no_final": 0 }, { "configuration": "gemma4-26b-a4b/off", "condition": "no_source", "records": 120, "quality": { "planned": 120, "answers_returned": 120, "operational_failures": 0, "graded": 120, "exact_claim_scores": 120, "mean_claim_score": 48.71666666666667, "fully_correct": 6, "profile_assessed": 118, "mean_fully_met_claim_coverage": 0.4215277777777778, "answers_with_material_errors": 111, "material_error_assessed": 118, "pending_reviews": 5, "operationally_flagged_answers": 0, "output_ceiling_hits": 0, "thinking_marker_breaches": 0, "direct_failures": 0, "regenerated_answers": 0, "mean_output_tokens": 389.2083333333333, "mean_client_seconds": 3.2954716571415097 }, "timing": { "recorded_cases": 120, "complete_timing_cases": 120, "missing_or_partial_timing_cases": 0, "mean_seconds": 3.2954716571415097, "median_seconds": 3.1383249454956967, "p90_seconds": 5.036440906987991, "p95_seconds": 5.9794541420124006, "p99_seconds": 7.333905334002338, "max_seconds": 7.601430270005949, "nonempty_finals": 120, "no_final_cases": 0, "output_ceiling_hits": 0, "retried_cases": 0, "request_failure_attempts": 0, "final_returned_median_seconds": 3.1383249454956967, "final_returned_p95_seconds": 5.9794541420124006, "first_token_timing_cases": 120, "first_token_p95_seconds": 0.24919300000000003, "observed_above_seconds": { "120": { "count": 0, "denominator": 120 }, "180": { "count": 0, "denominator": 120 }, "300": { "count": 0, "denominator": 120 }, "900": { "count": 0, "denominator": 120 }, "1800": { "count": 0, "denominator": 120 } }, "p25_seconds": 2.3658776989905164, "p75_seconds": 3.8282598789955955, "final_answer_completion": { "count": 120, "mean": 3.2954716571415097, "median": 3.1383249454956967, "p95": 5.9794541420124006, "maximum": 7.601430270005949, "p25": 2.3658776989905164, "p75": 3.8282598789955955 }, "above_30_seconds": 0, "bins": [ 107, 13, 0, 0, 0 ] }, "output_tokens": { "count": 120, "mean": 389.2083333333333, "median": 371.5, "p95": 634, "maximum": 948 }, "input_tokens": { "count": 120, "mean": 126.33333333333333, "median": 118.0, "p95": 185, "maximum": 296 }, "thinking_tokens": { "count": 120, "mean": 0, "median": 0.0, "p95": 0, "maximum": 0 }, "server_tokens_per_second": { "count": 120, "mean": 133.8167892125082, "median": 135.06990219194302, "p95": 136.08178266424682, "maximum": 136.65570255277075 }, "first_token_seconds": { "count": 120, "mean": 0.11510863333333334, "median": 0.096411, "p95": 0.24919300000000003, "maximum": 0.43753 }, "no_final": 0 }, { "configuration": "gemma4-26b-a4b/off", "condition": "supplied_passage", "records": 120, "quality": { "planned": 120, "answers_returned": 120, "operational_failures": 0, "graded": 120, "exact_claim_scores": 120, "mean_claim_score": 94.59166666666667, "fully_correct": 96, "profile_assessed": 120, "mean_fully_met_claim_coverage": 0.9215277777777777, "answers_with_material_errors": 16, "material_error_assessed": 120, "pending_reviews": 0, "operationally_flagged_answers": 0, "output_ceiling_hits": 0, "thinking_marker_breaches": 0, "direct_failures": 0, "regenerated_answers": 0, "mean_output_tokens": 255.34166666666667, "mean_client_seconds": 2.396756961800323 }, "timing": { "recorded_cases": 120, "complete_timing_cases": 120, "missing_or_partial_timing_cases": 0, "mean_seconds": 2.396756961800323, "median_seconds": 2.1331635170063237, "p90_seconds": 3.473940933996346, "p95_seconds": 4.555981274024816, "p99_seconds": 6.249718392005889, "max_seconds": 11.144434504007222, "nonempty_finals": 120, "no_final_cases": 0, "output_ceiling_hits": 0, "retried_cases": 0, "request_failure_attempts": 0, "final_returned_median_seconds": 2.1331635170063237, "final_returned_p95_seconds": 4.555981274024816, "first_token_timing_cases": 120, "first_token_p95_seconds": 0.322423, "observed_above_seconds": { "120": { "count": 0, "denominator": 120 }, "180": { "count": 0, "denominator": 120 }, "300": { "count": 0, "denominator": 120 }, "900": { "count": 0, "denominator": 120 }, "1800": { "count": 0, "denominator": 120 } }, "p25_seconds": 1.6533574200002477, "p75_seconds": 2.731671781017212, "final_answer_completion": { "count": 120, "mean": 2.396756961800323, "median": 2.1331635170063237, "p95": 4.555981274024816, "maximum": 11.144434504007222, "p25": 1.6533574200002477, "p75": 2.731671781017212 }, "above_30_seconds": 0, "bins": [ 115, 4, 1, 0, 0 ] }, "output_tokens": { "count": 120, "mean": 255.34166666666667, "median": 235.5, "p95": 450, "maximum": 610 }, "input_tokens": { "count": 120, "mean": 361.1666666666667, "median": 347.5, "p95": 579, "maximum": 1003 }, "thinking_tokens": { "count": 120, "mean": 0, "median": 0.0, "p95": 0, "maximum": 0 }, "server_tokens_per_second": { "count": 120, "mean": 133.12435755724005, "median": 133.70717537918085, "p95": 135.77191950868942, "maximum": 136.66833140325662 }, "first_token_seconds": { "count": 120, "mean": 0.17529409166666668, "median": 0.15581650000000002, "p95": 0.322423, "maximum": 0.451398 }, "no_final": 0 }, { "configuration": "gemma4-26b-a4b/on", "condition": "no_source", "records": 120, "quality": { "planned": 120, "answers_returned": 120, "operational_failures": 0, "graded": 120, "exact_claim_scores": 120, "mean_claim_score": 51.24166666666667, "fully_correct": 16, "profile_assessed": 120, "mean_fully_met_claim_coverage": 0.4434722222222222, "answers_with_material_errors": 102, "material_error_assessed": 120, "pending_reviews": 2, "operationally_flagged_answers": 0, "output_ceiling_hits": 0, "thinking_marker_breaches": 0, "direct_failures": 0, "regenerated_answers": 0, "mean_output_tokens": 2754.9333333333334, "mean_client_seconds": 22.196005501001007 }, "timing": { "recorded_cases": 120, "complete_timing_cases": 120, "missing_or_partial_timing_cases": 0, "mean_seconds": 22.196005501001007, "median_seconds": 17.415960502505186, "p90_seconds": 45.56172523699934, "p95_seconds": 63.54964979100623, "p99_seconds": 71.41336383399903, "max_seconds": 72.20774586198968, "nonempty_finals": 120, "no_final_cases": 0, "output_ceiling_hits": 0, "retried_cases": 0, "request_failure_attempts": 0, "final_returned_median_seconds": 17.415960502505186, "final_returned_p95_seconds": 63.54964979100623, "first_token_timing_cases": 120, "first_token_p95_seconds": 0.126562, "observed_above_seconds": { "120": { "count": 0, "denominator": 120 }, "180": { "count": 0, "denominator": 120 }, "300": { "count": 0, "denominator": 120 }, "900": { "count": 0, "denominator": 120 }, "1800": { "count": 0, "denominator": 120 } }, "p25_seconds": 9.27217273999122, "p75_seconds": 30.603828694002004, "final_answer_completion": { "count": 120, "mean": 22.196005501001007, "median": 17.415960502505186, "p95": 63.54964979100623, "maximum": 72.20774586198968, "p25": 9.27217273999122, "p75": 30.603828694002004 }, "above_30_seconds": 32, "bins": [ 2, 32, 34, 20, 32 ] }, "output_tokens": { "count": 120, "mean": 2754.9333333333334, "median": 2197.5, "p95": 7589, "maximum": 8581 }, "input_tokens": { "count": 120, "mean": 124.33333333333333, "median": 116.0, "p95": 183, "maximum": 294 }, "thinking_tokens": { "count": 120, "mean": 2493.0333333333333, "median": 1952.0, "p95": 7206, "maximum": 8357 }, "server_tokens_per_second": { "count": 120, "mean": 128.79115875664795, "median": 129.4941155279476, "p95": 133.86983360069016, "maximum": 134.9780784215145 }, "first_token_seconds": { "count": 120, "mean": 0.09977154166666667, "median": 0.094256, "p95": 0.126562, "maximum": 0.342857 }, "no_final": 0 }, { "configuration": "gemma4-26b-a4b/on", "condition": "supplied_passage", "records": 120, "quality": { "planned": 120, "answers_returned": 120, "operational_failures": 0, "graded": 120, "exact_claim_scores": 120, "mean_claim_score": 99.075, "fully_correct": 113, "profile_assessed": 120, "mean_fully_met_claim_coverage": 0.9770833333333333, "answers_with_material_errors": 2, "material_error_assessed": 120, "pending_reviews": 0, "operationally_flagged_answers": 0, "output_ceiling_hits": 0, "thinking_marker_breaches": 0, "direct_failures": 0, "regenerated_answers": 0, "mean_output_tokens": 892.55, "mean_client_seconds": 7.248209042998981 }, "timing": { "recorded_cases": 120, "complete_timing_cases": 120, "missing_or_partial_timing_cases": 0, "mean_seconds": 7.248209042998981, "median_seconds": 6.0843440874887165, "p90_seconds": 10.22679254601826, "p95_seconds": 15.676330866990611, "p99_seconds": 26.857115785998758, "max_seconds": 32.93256563300383, "nonempty_finals": 120, "no_final_cases": 0, "output_ceiling_hits": 0, "retried_cases": 0, "request_failure_attempts": 0, "final_returned_median_seconds": 6.0843440874887165, "final_returned_p95_seconds": 15.676330866990611, "first_token_timing_cases": 120, "first_token_p95_seconds": 0.268415, "observed_above_seconds": { "120": { "count": 0, "denominator": 120 }, "180": { "count": 0, "denominator": 120 }, "300": { "count": 0, "denominator": 120 }, "900": { "count": 0, "denominator": 120 }, "1800": { "count": 0, "denominator": 120 } }, "p25_seconds": 4.822041962994263, "p75_seconds": 7.8823426209855825, "final_answer_completion": { "count": 120, "mean": 7.248209042998981, "median": 6.0843440874887165, "p95": 15.676330866990611, "maximum": 32.93256563300383, "p25": 4.822041962994263, "p75": 7.8823426209855825 }, "above_30_seconds": 1, "bins": [ 34, 73, 8, 4, 1 ] }, "output_tokens": { "count": 120, "mean": 892.55, "median": 750.0, "p95": 1960, "maximum": 3981 }, "input_tokens": { "count": 120, "mean": 359.1666666666667, "median": 345.5, "p95": 577, "maximum": 1001 }, "thinking_tokens": { "count": 120, "mean": 683.6583333333333, "median": 552.5, "p95": 1522, "maximum": 3706 }, "server_tokens_per_second": { "count": 120, "mean": 131.52440838515287, "median": 131.68561353107324, "p95": 134.4730938227515, "maximum": 135.85868933817724 }, "first_token_seconds": { "count": 120, "mean": 0.15902423333333332, "median": 0.1482705, "p95": 0.268415, "maximum": 0.346223 }, "no_final": 0 }, { "configuration": "glm4.7-flash/off", "condition": "no_source", "records": 120, "quality": { "planned": 120, "answers_returned": 120, "operational_failures": 0, "graded": 120, "exact_claim_scores": 120, "mean_claim_score": 35.86666666666667, "fully_correct": 0, "profile_assessed": 120, "mean_fully_met_claim_coverage": 0.27125, "answers_with_material_errors": 120, "material_error_assessed": 120, "pending_reviews": 21, "operationally_flagged_answers": 0, "output_ceiling_hits": 0, "thinking_marker_breaches": 0, "direct_failures": 0, "regenerated_answers": 0, "mean_output_tokens": 473.2916666666667, "mean_client_seconds": 4.3515448323822055 }, "timing": { "recorded_cases": 120, "complete_timing_cases": 120, "missing_or_partial_timing_cases": 0, "mean_seconds": 4.3515448323822055, "median_seconds": 4.110602660497534, "p90_seconds": 6.394233761006035, "p95_seconds": 7.721416376007255, "p99_seconds": 8.310929497005418, "max_seconds": 10.094457867002347, "nonempty_finals": 120, "no_final_cases": 0, "output_ceiling_hits": 0, "retried_cases": 0, "request_failure_attempts": 0, "final_returned_median_seconds": 4.110602660497534, "final_returned_p95_seconds": 7.721416376007255, "first_token_timing_cases": 0, "first_token_p95_seconds": null, "observed_above_seconds": { "120": { "count": 0, "denominator": 120 }, "180": { "count": 0, "denominator": 120 }, "300": { "count": 0, "denominator": 120 }, "900": { "count": 0, "denominator": 120 }, "1800": { "count": 0, "denominator": 120 } }, "p25_seconds": 3.2855270960135385, "p75_seconds": 5.091484102013055, "final_answer_completion": { "count": 120, "mean": 4.3515448323822055, "median": 4.110602660497534, "p95": 7.721416376007255, "maximum": 10.094457867002347, "p25": 3.2855270960135385, "p75": 5.091484102013055 }, "above_30_seconds": 0, "bins": [ 86, 33, 1, 0, 0 ] }, "output_tokens": { "count": 120, "mean": 473.2916666666667, "median": 441.5, "p95": 856, "maximum": 1112 }, "input_tokens": { "count": 120, "mean": 112.725, "median": 104.0, "p95": 171, "maximum": 285 }, "thinking_tokens": { "count": 0, "mean": null, "median": null, "p95": null, "maximum": null }, "server_tokens_per_second": { "count": 0, "mean": null, "median": null, "p95": null, "maximum": null }, "first_token_seconds": { "count": 0, "mean": null, "median": null, "p95": null, "maximum": null }, "no_final": 0 }, { "configuration": "glm4.7-flash/off", "condition": "supplied_passage", "records": 120, "quality": { "planned": 120, "answers_returned": 120, "operational_failures": 0, "graded": 120, "exact_claim_scores": 120, "mean_claim_score": 70.80833333333334, "fully_correct": 30, "profile_assessed": 120, "mean_fully_met_claim_coverage": 0.6493055555555556, "answers_with_material_errors": 89, "material_error_assessed": 120, "pending_reviews": 3, "operationally_flagged_answers": 0, "output_ceiling_hits": 0, "thinking_marker_breaches": 0, "direct_failures": 0, "regenerated_answers": 0, "mean_output_tokens": 356.1666666666667, "mean_client_seconds": 3.4040780980246685 }, "timing": { "recorded_cases": 120, "complete_timing_cases": 120, "missing_or_partial_timing_cases": 0, "mean_seconds": 3.4040780980246685, "median_seconds": 3.1147646919998806, "p90_seconds": 5.0356341089936905, "p95_seconds": 6.068288653012132, "p99_seconds": 8.094451915996615, "max_seconds": 8.749617649998982, "nonempty_finals": 120, "no_final_cases": 0, "output_ceiling_hits": 0, "retried_cases": 0, "request_failure_attempts": 0, "final_returned_median_seconds": 3.1147646919998806, "final_returned_p95_seconds": 6.068288653012132, "first_token_timing_cases": 0, "first_token_p95_seconds": null, "observed_above_seconds": { "120": { "count": 0, "denominator": 120 }, "180": { "count": 0, "denominator": 120 }, "300": { "count": 0, "denominator": 120 }, "900": { "count": 0, "denominator": 120 }, "1800": { "count": 0, "denominator": 120 } }, "p25_seconds": 2.410167993977666, "p75_seconds": 4.104408179991879, "final_answer_completion": { "count": 120, "mean": 3.4040780980246685, "median": 3.1147646919998806, "p95": 6.068288653012132, "maximum": 8.749617649998982, "p25": 2.410167993977666, "p75": 4.104408179991879 }, "above_30_seconds": 0, "bins": [ 107, 13, 0, 0, 0 ] }, "output_tokens": { "count": 120, "mean": 356.1666666666667, "median": 333.5, "p95": 643, "maximum": 1001 }, "input_tokens": { "count": 120, "mean": 323.84166666666664, "median": 313.0, "p95": 518, "maximum": 917 }, "thinking_tokens": { "count": 0, "mean": null, "median": null, "p95": null, "maximum": null }, "server_tokens_per_second": { "count": 0, "mean": null, "median": null, "p95": null, "maximum": null }, "first_token_seconds": { "count": 0, "mean": null, "median": null, "p95": null, "maximum": null }, "no_final": 0 }, { "configuration": "glm4.7-flash/on", "condition": "no_source", "records": 120, "quality": { "planned": 120, "answers_returned": 120, "operational_failures": 0, "graded": 120, "exact_claim_scores": 120, "mean_claim_score": 45.733333333333334, "fully_correct": 9, "profile_assessed": 120, "mean_fully_met_claim_coverage": 0.38930555555555557, "answers_with_material_errors": 111, "material_error_assessed": 120, "pending_reviews": 8, "operationally_flagged_answers": 0, "output_ceiling_hits": 0, "thinking_marker_breaches": 0, "direct_failures": 0, "regenerated_answers": 0, "mean_output_tokens": 3885.1666666666665, "mean_client_seconds": 34.20369698000771 }, "timing": { "recorded_cases": 120, "complete_timing_cases": 120, "missing_or_partial_timing_cases": 0, "mean_seconds": 34.37950779504099, "median_seconds": 28.832616621497436, "p90_seconds": 62.33365877100732, "p95_seconds": 73.58050612601801, "p99_seconds": 107.98939229600364, "max_seconds": 142.9352043629915, "nonempty_finals": 120, "no_final_cases": 0, "output_ceiling_hits": 0, "retried_cases": 1, "request_failure_attempts": 0, "final_returned_median_seconds": 28.832616621497436, "final_returned_p95_seconds": 73.58050612601801, "first_token_timing_cases": 0, "first_token_p95_seconds": null, "observed_above_seconds": { "120": { "count": 1, "denominator": 120 }, "180": { "count": 0, "denominator": 120 }, "300": { "count": 0, "denominator": 120 }, "900": { "count": 0, "denominator": 120 }, "1800": { "count": 0, "denominator": 120 } }, "p25_seconds": 20.784505132003687, "p75_seconds": 40.233277241990436, "final_answer_completion": { "count": 120, "mean": 34.37950779504099, "median": 28.832616621497436, "p95": 73.58050612601801, "maximum": 142.9352043629915, "p25": 20.784505132003687, "p75": 40.233277241990436 }, "above_30_seconds": 56, "bins": [ 0, 0, 28, 36, 56 ] }, "output_tokens": { "count": 120, "mean": 3885.1666666666665, "median": 3283.5, "p95": 8158, "maximum": 15014 }, "input_tokens": { "count": 120, "mean": 112.725, "median": 104.0, "p95": 171, "maximum": 285 }, "thinking_tokens": { "count": 0, "mean": null, "median": null, "p95": null, "maximum": null }, "server_tokens_per_second": { "count": 0, "mean": null, "median": null, "p95": null, "maximum": null }, "first_token_seconds": { "count": 0, "mean": null, "median": null, "p95": null, "maximum": null }, "no_final": 0 }, { "configuration": "glm4.7-flash/on", "condition": "supplied_passage", "records": 120, "quality": { "planned": 120, "answers_returned": 120, "operational_failures": 0, "graded": 120, "exact_claim_scores": 120, "mean_claim_score": 90.21666666666667, "fully_correct": 82, "profile_assessed": 120, "mean_fully_met_claim_coverage": 0.8623611111111111, "answers_with_material_errors": 26, "material_error_assessed": 120, "pending_reviews": 0, "operationally_flagged_answers": 0, "output_ceiling_hits": 0, "thinking_marker_breaches": 0, "direct_failures": 0, "regenerated_answers": 0, "mean_output_tokens": 2061.9166666666665, "mean_client_seconds": 17.89113451098286 }, "timing": { "recorded_cases": 120, "complete_timing_cases": 120, "missing_or_partial_timing_cases": 0, "mean_seconds": 18.117181959207908, "median_seconds": 13.765581615487463, "p90_seconds": 33.44953015400097, "p95_seconds": 44.24636647099396, "p99_seconds": 76.92090486900997, "max_seconds": 83.05583995699999, "nonempty_finals": 120, "no_final_cases": 0, "output_ceiling_hits": 0, "retried_cases": 1, "request_failure_attempts": 0, "final_returned_median_seconds": 13.765581615487463, "final_returned_p95_seconds": 44.24636647099396, "first_token_timing_cases": 0, "first_token_p95_seconds": null, "observed_above_seconds": { "120": { "count": 0, "denominator": 120 }, "180": { "count": 0, "denominator": 120 }, "300": { "count": 0, "denominator": 120 }, "900": { "count": 0, "denominator": 120 }, "1800": { "count": 0, "denominator": 120 } }, "p25_seconds": 11.029520962998504, "p75_seconds": 19.1848278180114, "final_answer_completion": { "count": 120, "mean": 18.117181959207908, "median": 13.765581615487463, "p95": 44.24636647099396, "maximum": 83.05583995699999, "p25": 11.029520962998504, "p75": 19.1848278180114 }, "above_30_seconds": 14, "bins": [ 0, 20, 71, 15, 14 ] }, "output_tokens": { "count": 120, "mean": 2061.9166666666665, "median": 1604.0, "p95": 4544, "maximum": 9095 }, "input_tokens": { "count": 120, "mean": 323.84166666666664, "median": 313.0, "p95": 518, "maximum": 917 }, "thinking_tokens": { "count": 0, "mean": null, "median": null, "p95": null, "maximum": null }, "server_tokens_per_second": { "count": 0, "mean": null, "median": null, "p95": null, "maximum": null }, "first_token_seconds": { "count": 0, "mean": null, "median": null, "p95": null, "maximum": null }, "no_final": 0 }, { "configuration": "muse-glimmer/low", "condition": "no_source", "records": 120, "quality": { "planned": 120, "answers_returned": 120, "operational_failures": 0, "graded": 120, "exact_claim_scores": 120, "mean_claim_score": 60.06666666666667, "fully_correct": 11, "profile_assessed": 120, "mean_fully_met_claim_coverage": 0.5441666666666667, "answers_with_material_errors": 108, "material_error_assessed": 120, "pending_reviews": 9, "operationally_flagged_answers": 0, "output_ceiling_hits": 0, "thinking_marker_breaches": 0, "direct_failures": 0, "regenerated_answers": 0, "mean_output_tokens": 912.7583333333333, "mean_client_seconds": 27.54944677187741 }, "timing": { "recorded_cases": 120, "complete_timing_cases": 120, "missing_or_partial_timing_cases": 0, "mean_seconds": 27.54944677187741, "median_seconds": 27.289599504016223, "p90_seconds": 39.13926862101653, "p95_seconds": 43.566442504990846, "p99_seconds": 46.7977286259993, "max_seconds": 51.1879825439828, "nonempty_finals": 120, "no_final_cases": 0, "output_ceiling_hits": 0, "retried_cases": 0, "request_failure_attempts": 0, "final_returned_median_seconds": 27.289599504016223, "final_returned_p95_seconds": 43.566442504990846, "first_token_timing_cases": 120, "first_token_p95_seconds": 0.265206, "observed_above_seconds": { "120": { "count": 0, "denominator": 120 }, "180": { "count": 0, "denominator": 120 }, "300": { "count": 0, "denominator": 120 }, "900": { "count": 0, "denominator": 120 }, "1800": { "count": 0, "denominator": 120 } }, "p25_seconds": 20.59537050401559, "p75_seconds": 31.945420611009467, "final_answer_completion": { "count": 120, "mean": 27.54944677187741, "median": 27.289599504016223, "p95": 43.566442504990846, "maximum": 51.1879825439828, "p25": 20.59537050401559, "p75": 31.945420611009467 }, "above_30_seconds": 42, "bins": [ 0, 0, 27, 51, 42 ] }, "output_tokens": { "count": 120, "mean": 912.7583333333333, "median": 904.0, "p95": 1446, "maximum": 1703 }, "input_tokens": { "count": 120, "mean": 134.85833333333332, "median": 126.5, "p95": 193, "maximum": 308 }, "thinking_tokens": { "count": 120, "mean": 458.40833333333336, "median": 418.0, "p95": 892, "maximum": 1098 }, "server_tokens_per_second": { "count": 120, "mean": 33.415634361740054, "median": 33.42487251709115, "p95": 33.520467597515044, "maximum": 33.573869695141326 }, "first_token_seconds": { "count": 120, "mean": 0.19606734166666667, "median": 0.187398, "p95": 0.265206, "maximum": 0.331065 }, "no_final": 0 }, { "configuration": "muse-glimmer/low", "condition": "supplied_passage", "records": 120, "quality": { "planned": 120, "answers_returned": 120, "operational_failures": 0, "graded": 120, "exact_claim_scores": 120, "mean_claim_score": 98.85833333333333, "fully_correct": 96, "profile_assessed": 120, "mean_fully_met_claim_coverage": 0.9858333333333333, "answers_with_material_errors": 24, "material_error_assessed": 120, "pending_reviews": 3, "operationally_flagged_answers": 0, "output_ceiling_hits": 0, "thinking_marker_breaches": 0, "direct_failures": 0, "regenerated_answers": 0, "mean_output_tokens": 581.0166666666667, "mean_client_seconds": 17.81076878534174 }, "timing": { "recorded_cases": 120, "complete_timing_cases": 120, "missing_or_partial_timing_cases": 0, "mean_seconds": 17.81076878534174, "median_seconds": 16.386069722517277, "p90_seconds": 24.8872484669846, "p95_seconds": 27.455511704989476, "p99_seconds": 40.25435937399743, "max_seconds": 47.98068379602046, "nonempty_finals": 120, "no_final_cases": 0, "output_ceiling_hits": 0, "retried_cases": 0, "request_failure_attempts": 0, "final_returned_median_seconds": 16.386069722517277, "final_returned_p95_seconds": 27.455511704989476, "first_token_timing_cases": 120, "first_token_p95_seconds": 0.536592, "observed_above_seconds": { "120": { "count": 0, "denominator": 120 }, "180": { "count": 0, "denominator": 120 }, "300": { "count": 0, "denominator": 120 }, "900": { "count": 0, "denominator": 120 }, "1800": { "count": 0, "denominator": 120 } }, "p25_seconds": 13.8378901600081, "p75_seconds": 20.646109254012117, "final_answer_completion": { "count": 120, "mean": 17.81076878534174, "median": 16.386069722517277, "p95": 27.455511704989476, "maximum": 47.98068379602046, "p25": 13.8378901600081, "p75": 20.646109254012117 }, "above_30_seconds": 4, "bins": [ 0, 3, 83, 30, 4 ] }, "output_tokens": { "count": 120, "mean": 581.0166666666667, "median": 534.5, "p95": 897, "maximum": 1559 }, "input_tokens": { "count": 120, "mean": 345.71666666666664, "median": 337.0, "p95": 542, "maximum": 941 }, "thinking_tokens": { "count": 120, "mean": 215.21666666666667, "median": 192.5, "p95": 439, "maximum": 724 }, "server_tokens_per_second": { "count": 120, "mean": 33.38882234206541, "median": 33.40672308043928, "p95": 33.50948652411969, "maximum": 33.57508405148112 }, "first_token_seconds": { "count": 120, "mean": 0.3745515, "median": 0.3600725, "p95": 0.536592, "maximum": 0.957218 }, "no_final": 0 } ], "sources": [ { "historical_pilot_sha256": "b3ad837fb975d9033455bd4dadc6d395d0644ab9f25f89064078a80ea7d172c4", "suite_id": "tiny", "selected_records": 1920, "observations_sha256": "1d07de0dd82a2ca520348e2225687220876260b9ceb212f679665159d0d151f4", "metrics_sha256": "2489cf08a2d7c23d17464f74dca512b7f352c8188ae59e4266cbccd20357e3a0", "runtime_profiles_sha256": "b05668f22eb7b4eae536200890e084dd49cd16ef6055192081d74497aec9f2ae", "source_selected_records": 2400 }, { "historical_pilot_sha256": null, "suite_id": "medium", "selected_records": 1680, "observations_sha256": "1e69f98b64b8893f2ff0770fd648e2f820032beabcd21c3ca3dba780331317ed", "metrics_sha256": "a3c50604e69754f261a4c088413ae9f80b5a2946d4b1fec82c0f9ef64524c1f3", "runtime_profiles_sha256": "4140e5a517327bed77f0452835bb44ce6739ad657bd1414d406c27d9b938415b", "source_selected_records": 1680 }, { "historical_pilot_sha256": null, "suite_id": "large", "selected_records": 2400, "observations_sha256": "b73550e0657dadb55f31fa62447b1ac56e33dda20403acaeda971c2c5cbb3154", "metrics_sha256": "47c7a197d15f3cc849e45cb15ded2c056c3c203c0e7bf1eed9a8ed1648edfa90", "runtime_profiles_sha256": "0ba5b1b95b1f5e65090bebc34d6fed90f73f7eace2851f2f9622002d896410e6", "source_selected_records": 2400 } ], "random_human_audit": "not performed; declined by operator", "grader_accuracy": "unmeasured" }