{"artifact_type":"public_benchmark_summary","schema_version":1,"dataset_access":"closed","runs":[{"model":"anthropic/claude-sonnet-5.5","provider":"openai-compatible","track":"answer-only","completed_at":"2026-10-01T18:04:03.406387Z","status":"success","protocol":{"track":"answer-only","name":"hse-multiselect-zero-shot-v10","selected_dataset_sha256":"f8d2c31d1095aed302e4a2c05ea795c9f897f061275265599fc1e5dba212c239","topics_sha256":"4897b9e828053e1c9ebde4729c8af75baffcd547d251e49a21006eab961d1fff","prompt_sha256":"d1ed6b553cb6997860d5eb7cb36e6399f091187c175b5872e568e40676ab5e68","implementation_sha256":"9b2c30085d7db94fdb18d9ebd839bf8340992e93211fad108162f4215644a69a","selected_questions":100,"repetitions":3,"temperature":null,"max_output_tokens":16384,"timeout_seconds":180,"sampling_method":"all","sampling_seed":42,"answer_order":"fixed","reasoning_settings":{"effort":"low"},"routing_settings":{"only":["anthropic"],"allow_fallbacks":false,"require_parameters":true},"question_concurrency":8,"request_concurrency":8,"batch_concurrency":1},"metrics":{"question_accuracy":0.7266666666666667,"valid_response_accuracy":0.7266666666666667,"coverage":1,"total_questions":300,"correct_questions":218,"valid_questions":300,"failed_questions":0,"provider_errors":0,"parse_errors":0,"incomplete_responses":0,"question_accuracy_ci":null},"repetition_metrics":[{"question_accuracy":0.71,"valid_response_accuracy":0.71,"coverage":1,"total_questions":100,"correct_questions":71,"valid_questions":100,"failed_questions":0,"provider_errors":0,"parse_errors":0,"incomplete_responses":0,"question_accuracy_ci":{"method":"wilson","confidence_level":0.95,"lower":0.6146110882442888,"upper":0.7898516425087772}},{"question_accuracy":0.74,"valid_response_accuracy":0.74,"coverage":1,"total_questions":100,"correct_questions":74,"valid_questions":100,"failed_questions":0,"provider_errors":0,"parse_errors":0,"incomplete_responses":0,"question_accuracy_ci":{"method":"wilson","confidence_level":0.95,"lower":0.6462901055081282,"upper":0.8159530153525186}},{"question_accuracy":0.73,"valid_response_accuracy":0.73,"coverage":1,"total_questions":100,"correct_questions":73,"valid_questions":100,"failed_questions":0,"provider_errors":0,"parse_errors":0,"incomplete_responses":0,"question_accuracy_ci":{"method":"wilson","confidence_level":0.95,"lower":0.6356788323205499,"upper":0.8073041585042365}}],"topic_results":[{"topic":"Environmental protection and waste","questions":20,"metrics":{"question_accuracy":0.6166666666666667,"valid_response_accuracy":0.6166666666666667,"coverage":1,"total_questions":60,"correct_questions":37,"valid_questions":60,"failed_questions":0,"provider_errors":0,"parse_errors":0,"incomplete_responses":0,"question_accuracy_ci":null}},{"topic":"Fire safety","questions":18,"metrics":{"question_accuracy":0.8703703703703703,"valid_response_accuracy":0.8703703703703703,"coverage":1,"total_questions":54,"correct_questions":47,"valid_questions":54,"failed_questions":0,"provider_errors":0,"parse_errors":0,"incomplete_responses":0,"question_accuracy_ci":null}},{"topic":"First aid","questions":20,"metrics":{"question_accuracy":0.4666666666666667,"valid_response_accuracy":0.4666666666666667,"coverage":1,"total_questions":60,"correct_questions":28,"valid_questions":60,"failed_questions":0,"provider_errors":0,"parse_errors":0,"incomplete_responses":0,"question_accuracy_ci":null}},{"topic":"Industrial and facility safety","questions":20,"metrics":{"question_accuracy":0.8333333333333334,"valid_response_accuracy":0.8333333333333334,"coverage":1,"total_questions":60,"correct_questions":50,"valid_questions":60,"failed_questions":0,"provider_errors":0,"parse_errors":0,"incomplete_responses":0,"question_accuracy_ci":null}},{"topic":"Occupational safety and health","questions":20,"metrics":{"question_accuracy":0.8333333333333334,"valid_response_accuracy":0.8333333333333334,"coverage":1,"total_questions":60,"correct_questions":50,"valid_questions":60,"failed_questions":0,"provider_errors":0,"parse_errors":0,"incomplete_responses":0,"question_accuracy_ci":null}},{"topic":"Regulatory administration","questions":2,"metrics":{"question_accuracy":1,"valid_response_accuracy":1,"coverage":1,"total_questions":6,"correct_questions":6,"valid_questions":6,"failed_questions":0,"provider_errors":0,"parse_errors":0,"incomplete_responses":0,"question_accuracy_ci":null}}],"baselines":{"all_false_question_accuracy":0,"all_true_question_accuracy":0},"operational_metrics":{"reported_cost_usd":"0.625028","responses_with_cost":300,"responses_without_cost":0,"prompt_tokens":87084,"completion_tokens":45086,"reasoning_tokens":0,"latency_median_seconds":2.250014146000467,"latency_p95_seconds":4.570256583996525,"providers":{"Anthropic":300},"format_compliant_responses":137,"format_checked_responses":300},"infrastructure_attempt_count":0},{"model":"anthropic/claude-sonnet-5.5","provider":"openai-compatible","track":"reasoning-assisted","completed_at":"2026-10-01T18:06:35.981424Z","status":"success","protocol":{"track":"reasoning-assisted","name":"hse-multiselect-zero-shot-v10","selected_dataset_sha256":"f8d2c31d1095aed302e4a2c05ea795c9f897f061275265599fc1e5dba212c239","topics_sha256":"4897b9e828053e1c9ebde4729c8af75baffcd547d251e49a21006eab961d1fff","prompt_sha256":"1e82d8ac2c47f18b33cf9cda4901214138aba1621d45b4d34123be7253a8a2f6","implementation_sha256":"9b2c30085d7db94fdb18d9ebd839bf8340992e93211fad108162f4215644a69a","selected_questions":100,"repetitions":3,"temperature":null,"max_output_tokens":16384,"timeout_seconds":180,"sampling_method":"all","sampling_seed":42,"answer_order":"fixed","reasoning_settings":{"effort":"low"},"routing_settings":{"only":["anthropic"],"allow_fallbacks":false,"require_parameters":true},"question_concurrency":8,"request_concurrency":8,"batch_concurrency":1},"metrics":{"question_accuracy":0.7666666666666667,"valid_response_accuracy":0.7666666666666667,"coverage":1,"total_questions":300,"correct_questions":230,"valid_questions":300,"failed_questions":0,"provider_errors":0,"parse_errors":0,"incomplete_responses":0,"question_accuracy_ci":null},"repetition_metrics":[{"question_accuracy":0.77,"valid_response_accuracy":0.77,"coverage":1,"total_questions":100,"correct_questions":77,"valid_questions":100,"failed_questions":0,"provider_errors":0,"parse_errors":0,"incomplete_responses":0,"question_accuracy_ci":{"method":"wilson","confidence_level":0.95,"lower":0.6784561697712622,"upper":0.8415673411969654}},{"question_accuracy":0.79,"valid_response_accuracy":0.79,"coverage":1,"total_questions":100,"correct_questions":79,"valid_questions":100,"failed_questions":0,"provider_errors":0,"parse_errors":0,"incomplete_responses":0,"question_accuracy_ci":{"method":"wilson","confidence_level":0.95,"lower":0.7002003116591013,"upper":0.858343459380847}},{"question_accuracy":0.74,"valid_response_accuracy":0.74,"coverage":1,"total_questions":100,"correct_questions":74,"valid_questions":100,"failed_questions":0,"provider_errors":0,"parse_errors":0,"incomplete_responses":0,"question_accuracy_ci":{"method":"wilson","confidence_level":0.95,"lower":0.6462901055081282,"upper":0.8159530153525186}}],"topic_results":[{"topic":"Environmental protection and waste","questions":20,"metrics":{"question_accuracy":0.7,"valid_response_accuracy":0.7,"coverage":1,"total_questions":60,"correct_questions":42,"valid_questions":60,"failed_questions":0,"provider_errors":0,"parse_errors":0,"incomplete_responses":0,"question_accuracy_ci":null}},{"topic":"Fire safety","questions":18,"metrics":{"question_accuracy":0.8888888888888888,"valid_response_accuracy":0.8888888888888888,"coverage":1,"total_questions":54,"correct_questions":48,"valid_questions":54,"failed_questions":0,"provider_errors":0,"parse_errors":0,"incomplete_responses":0,"question_accuracy_ci":null}},{"topic":"First aid","questions":20,"metrics":{"question_accuracy":0.4666666666666667,"valid_response_accuracy":0.4666666666666667,"coverage":1,"total_questions":60,"correct_questions":28,"valid_questions":60,"failed_questions":0,"provider_errors":0,"parse_errors":0,"incomplete_responses":0,"question_accuracy_ci":null}},{"topic":"Industrial and facility safety","questions":20,"metrics":{"question_accuracy":0.8833333333333333,"valid_response_accuracy":0.8833333333333333,"coverage":1,"total_questions":60,"correct_questions":53,"valid_questions":60,"failed_questions":0,"provider_errors":0,"parse_errors":0,"incomplete_responses":0,"question_accuracy_ci":null}},{"topic":"Occupational safety and health","questions":20,"metrics":{"question_accuracy":0.8833333333333333,"valid_response_accuracy":0.8833333333333333,"coverage":1,"total_questions":60,"correct_questions":53,"valid_questions":60,"failed_questions":0,"provider_errors":0,"parse_errors":0,"incomplete_responses":0,"question_accuracy_ci":null}},{"topic":"Regulatory administration","questions":2,"metrics":{"question_accuracy":1,"valid_response_accuracy":1,"coverage":1,"total_questions":6,"correct_questions":6,"valid_questions":6,"failed_questions":0,"provider_errors":0,"parse_errors":0,"incomplete_responses":0,"question_accuracy_ci":null}}],"baselines":{"all_false_question_accuracy":0,"all_true_question_accuracy":0},"operational_metrics":{"reported_cost_usd":"1.233448","responses_with_cost":300,"responses_without_cost":0,"prompt_tokens":77784,"completion_tokens":107788,"reasoning_tokens":0,"latency_median_seconds":3.8794086249981774,"latency_p95_seconds":5.71949937500176,"providers":{"Anthropic":300},"format_compliant_responses":294,"format_checked_responses":300},"infrastructure_attempt_count":0},{"model":"anthropic/claude-sonnet-5.5","provider":"openai-compatible","track":"structured-output","completed_at":"2026-10-01T18:07:57.246806Z","status":"success","protocol":{"track":"structured-output","name":"hse-multiselect-zero-shot-v10","selected_dataset_sha256":"f8d2c31d1095aed302e4a2c05ea795c9f897f061275265599fc1e5dba212c239","topics_sha256":"4897b9e828053e1c9ebde4729c8af75baffcd547d251e49a21006eab961d1fff","prompt_sha256":"8cf0f6b182fe981a596b4d48bc132286a16ebf2f25ffbb15df4e27c5a12a07c1","implementation_sha256":"9b2c30085d7db94fdb18d9ebd839bf8340992e93211fad108162f4215644a69a","selected_questions":100,"repetitions":3,"temperature":null,"max_output_tokens":16384,"timeout_seconds":180,"sampling_method":"all","sampling_seed":42,"answer_order":"fixed","reasoning_settings":{"effort":"low"},"routing_settings":{"only":["anthropic"],"allow_fallbacks":false,"require_parameters":true},"question_concurrency":8,"request_concurrency":8,"batch_concurrency":1},"metrics":{"question_accuracy":0.7066666666666667,"valid_response_accuracy":0.7066666666666667,"coverage":1,"total_questions":300,"correct_questions":212,"valid_questions":300,"failed_questions":0,"provider_errors":0,"parse_errors":0,"incomplete_responses":0,"question_accuracy_ci":null},"repetition_metrics":[{"question_accuracy":0.7,"valid_response_accuracy":0.7,"coverage":1,"total_questions":100,"correct_questions":70,"valid_questions":100,"failed_questions":0,"provider_errors":0,"parse_errors":0,"incomplete_responses":0,"question_accuracy_ci":{"method":"wilson","confidence_level":0.95,"lower":0.6041514536665332,"upper":0.7810511470506724}},{"question_accuracy":0.71,"valid_response_accuracy":0.71,"coverage":1,"total_questions":100,"correct_questions":71,"valid_questions":100,"failed_questions":0,"provider_errors":0,"parse_errors":0,"incomplete_responses":0,"question_accuracy_ci":{"method":"wilson","confidence_level":0.95,"lower":0.6146110882442888,"upper":0.7898516425087772}},{"question_accuracy":0.71,"valid_response_accuracy":0.71,"coverage":1,"total_questions":100,"correct_questions":71,"valid_questions":100,"failed_questions":0,"provider_errors":0,"parse_errors":0,"incomplete_responses":0,"question_accuracy_ci":{"method":"wilson","confidence_level":0.95,"lower":0.6146110882442888,"upper":0.7898516425087772}}],"topic_results":[{"topic":"Environmental protection and waste","questions":20,"metrics":{"question_accuracy":0.6833333333333333,"valid_response_accuracy":0.6833333333333333,"coverage":1,"total_questions":60,"correct_questions":41,"valid_questions":60,"failed_questions":0,"provider_errors":0,"parse_errors":0,"incomplete_responses":0,"question_accuracy_ci":null}},{"topic":"Fire safety","questions":18,"metrics":{"question_accuracy":0.7222222222222222,"valid_response_accuracy":0.7222222222222222,"coverage":1,"total_questions":54,"correct_questions":39,"valid_questions":54,"failed_questions":0,"provider_errors":0,"parse_errors":0,"incomplete_responses":0,"question_accuracy_ci":null}},{"topic":"First aid","questions":20,"metrics":{"question_accuracy":0.5166666666666667,"valid_response_accuracy":0.5166666666666667,"coverage":1,"total_questions":60,"correct_questions":31,"valid_questions":60,"failed_questions":0,"provider_errors":0,"parse_errors":0,"incomplete_responses":0,"question_accuracy_ci":null}},{"topic":"Industrial and facility safety","questions":20,"metrics":{"question_accuracy":0.8833333333333333,"valid_response_accuracy":0.8833333333333333,"coverage":1,"total_questions":60,"correct_questions":53,"valid_questions":60,"failed_questions":0,"provider_errors":0,"parse_errors":0,"incomplete_responses":0,"question_accuracy_ci":null}},{"topic":"Occupational safety and health","questions":20,"metrics":{"question_accuracy":0.75,"valid_response_accuracy":0.75,"coverage":1,"total_questions":60,"correct_questions":45,"valid_questions":60,"failed_questions":0,"provider_errors":0,"parse_errors":0,"incomplete_responses":0,"question_accuracy_ci":null}},{"topic":"Regulatory administration","questions":2,"metrics":{"question_accuracy":0.5,"valid_response_accuracy":0.5,"coverage":1,"total_questions":6,"correct_questions":3,"valid_questions":6,"failed_questions":0,"provider_errors":0,"parse_errors":0,"incomplete_responses":0,"question_accuracy_ci":null}}],"baselines":{"all_false_question_accuracy":0,"all_true_question_accuracy":0},"operational_metrics":{"reported_cost_usd":"0.428558","responses_with_cost":300,"responses_without_cost":0,"prompt_tokens":129384,"completion_tokens":16979,"reasoning_tokens":12070,"latency_median_seconds":1.6351342289999593,"latency_p95_seconds":6.688049584001419,"providers":{"Anthropic":300},"format_compliant_responses":300,"format_checked_responses":300},"infrastructure_attempt_count":0},{"model":"deepseek/deepseek-v4.1-flash","provider":"openai-compatible","track":"answer-only","completed_at":"2026-10-01T17:52:45.396059Z","status":"success","protocol":{"track":"answer-only","name":"hse-multiselect-zero-shot-v10","selected_dataset_sha256":"f8d2c31d1095aed302e4a2c05ea795c9f897f061275265599fc1e5dba212c239","topics_sha256":"4897b9e828053e1c9ebde4729c8af75baffcd547d251e49a21006eab961d1fff","prompt_sha256":"d1ed6b553cb6997860d5eb7cb36e6399f091187c175b5872e568e40676ab5e68","implementation_sha256":"9b2c30085d7db94fdb18d9ebd839bf8340992e93211fad108162f4215644a69a","selected_questions":100,"repetitions":3,"temperature":null,"max_output_tokens":16384,"timeout_seconds":180,"sampling_method":"all","sampling_seed":42,"answer_order":"fixed","reasoning_settings":{"effort":"low"},"routing_settings":{"only":["together"],"allow_fallbacks":false,"require_parameters":true},"question_concurrency":8,"request_concurrency":8,"batch_concurrency":1},"metrics":{"question_accuracy":0.6533333333333333,"valid_response_accuracy":0.6533333333333333,"coverage":1,"total_questions":300,"correct_questions":196,"valid_questions":300,"failed_questions":0,"provider_errors":0,"parse_errors":0,"incomplete_responses":0,"question_accuracy_ci":null},"repetition_metrics":[{"question_accuracy":0.62,"valid_response_accuracy":0.62,"coverage":1,"total_questions":100,"correct_questions":62,"valid_questions":100,"failed_questions":0,"provider_errors":0,"parse_errors":0,"incomplete_responses":0,"question_accuracy_ci":{"method":"wilson","confidence_level":0.95,"lower":0.5220975529551107,"upper":0.7090240074752127}},{"question_accuracy":0.69,"valid_response_accuracy":0.69,"coverage":1,"total_questions":100,"correct_questions":69,"valid_questions":100,"failed_questions":0,"provider_errors":0,"parse_errors":0,"incomplete_responses":0,"question_accuracy_ci":{"method":"wilson","confidence_level":0.95,"lower":0.5937394428051057,"upper":0.7722030278762395}},{"question_accuracy":0.65,"valid_response_accuracy":0.65,"coverage":1,"total_questions":100,"correct_questions":65,"valid_questions":100,"failed_questions":0,"provider_errors":0,"parse_errors":0,"incomplete_responses":0,"question_accuracy_ci":{"method":"wilson","confidence_level":0.95,"lower":0.5525444329688796,"upper":0.7363575175690247}}],"topic_results":[{"topic":"Environmental protection and waste","questions":20,"metrics":{"question_accuracy":0.6333333333333333,"valid_response_accuracy":0.6333333333333333,"coverage":1,"total_questions":60,"correct_questions":38,"valid_questions":60,"failed_questions":0,"provider_errors":0,"parse_errors":0,"incomplete_responses":0,"question_accuracy_ci":null}},{"topic":"Fire safety","questions":18,"metrics":{"question_accuracy":0.7222222222222222,"valid_response_accuracy":0.7222222222222222,"coverage":1,"total_questions":54,"correct_questions":39,"valid_questions":54,"failed_questions":0,"provider_errors":0,"parse_errors":0,"incomplete_responses":0,"question_accuracy_ci":null}},{"topic":"First aid","questions":20,"metrics":{"question_accuracy":0.43333333333333335,"valid_response_accuracy":0.43333333333333335,"coverage":1,"total_questions":60,"correct_questions":26,"valid_questions":60,"failed_questions":0,"provider_errors":0,"parse_errors":0,"incomplete_responses":0,"question_accuracy_ci":null}},{"topic":"Industrial and facility safety","questions":20,"metrics":{"question_accuracy":0.75,"valid_response_accuracy":0.75,"coverage":1,"total_questions":60,"correct_questions":45,"valid_questions":60,"failed_questions":0,"provider_errors":0,"parse_errors":0,"incomplete_responses":0,"question_accuracy_ci":null}},{"topic":"Occupational safety and health","questions":20,"metrics":{"question_accuracy":0.7333333333333333,"valid_response_accuracy":0.7333333333333333,"coverage":1,"total_questions":60,"correct_questions":44,"valid_questions":60,"failed_questions":0,"provider_errors":0,"parse_errors":0,"incomplete_responses":0,"question_accuracy_ci":null}},{"topic":"Regulatory administration","questions":2,"metrics":{"question_accuracy":0.6666666666666666,"valid_response_accuracy":0.6666666666666666,"coverage":1,"total_questions":6,"correct_questions":4,"valid_questions":6,"failed_questions":0,"provider_errors":0,"parse_errors":0,"incomplete_responses":0,"question_accuracy_ci":null}}],"baselines":{"all_false_question_accuracy":0,"all_true_question_accuracy":0},"operational_metrics":{"reported_cost_usd":"0.519128580","responses_with_cost":300,"responses_without_cost":0,"prompt_tokens":61407,"completion_tokens":417569,"reasoning_tokens":414292,"latency_median_seconds":3.300826083002903,"latency_p95_seconds":17.317834040994057,"providers":{"Together":300},"format_compliant_responses":300,"format_checked_responses":300},"infrastructure_attempt_count":0},{"model":"deepseek/deepseek-v4.1-flash","provider":"openai-compatible","track":"reasoning-assisted","completed_at":"2026-10-01T17:58:06.132245Z","status":"success","protocol":{"track":"reasoning-assisted","name":"hse-multiselect-zero-shot-v10","selected_dataset_sha256":"f8d2c31d1095aed302e4a2c05ea795c9f897f061275265599fc1e5dba212c239","topics_sha256":"4897b9e828053e1c9ebde4729c8af75baffcd547d251e49a21006eab961d1fff","prompt_sha256":"1e82d8ac2c47f18b33cf9cda4901214138aba1621d45b4d34123be7253a8a2f6","implementation_sha256":"9b2c30085d7db94fdb18d9ebd839bf8340992e93211fad108162f4215644a69a","selected_questions":100,"repetitions":3,"temperature":null,"max_output_tokens":16384,"timeout_seconds":180,"sampling_method":"all","sampling_seed":42,"answer_order":"fixed","reasoning_settings":{"effort":"low"},"routing_settings":{"only":["together"],"allow_fallbacks":false,"require_parameters":true},"question_concurrency":8,"request_concurrency":8,"batch_concurrency":1},"metrics":{"question_accuracy":0.6833333333333333,"valid_response_accuracy":0.6833333333333333,"coverage":1,"total_questions":300,"correct_questions":205,"valid_questions":300,"failed_questions":0,"provider_errors":0,"parse_errors":0,"incomplete_responses":0,"question_accuracy_ci":null},"repetition_metrics":[{"question_accuracy":0.7,"valid_response_accuracy":0.7,"coverage":1,"total_questions":100,"correct_questions":70,"valid_questions":100,"failed_questions":0,"provider_errors":0,"parse_errors":0,"incomplete_responses":0,"question_accuracy_ci":{"method":"wilson","confidence_level":0.95,"lower":0.6041514536665332,"upper":0.7810511470506724}},{"question_accuracy":0.69,"valid_response_accuracy":0.69,"coverage":1,"total_questions":100,"correct_questions":69,"valid_questions":100,"failed_questions":0,"provider_errors":0,"parse_errors":0,"incomplete_responses":0,"question_accuracy_ci":{"method":"wilson","confidence_level":0.95,"lower":0.5937394428051057,"upper":0.7722030278762395}},{"question_accuracy":0.66,"valid_response_accuracy":0.66,"coverage":1,"total_questions":100,"correct_questions":66,"valid_questions":100,"failed_questions":0,"provider_errors":0,"parse_errors":0,"incomplete_responses":0,"question_accuracy_ci":{"method":"wilson","confidence_level":0.95,"lower":0.5627772885472462,"upper":0.7453847920265184}}],"topic_results":[{"topic":"Environmental protection and waste","questions":20,"metrics":{"question_accuracy":0.5666666666666667,"valid_response_accuracy":0.5666666666666667,"coverage":1,"total_questions":60,"correct_questions":34,"valid_questions":60,"failed_questions":0,"provider_errors":0,"parse_errors":0,"incomplete_responses":0,"question_accuracy_ci":null}},{"topic":"Fire safety","questions":18,"metrics":{"question_accuracy":0.7222222222222222,"valid_response_accuracy":0.7222222222222222,"coverage":1,"total_questions":54,"correct_questions":39,"valid_questions":54,"failed_questions":0,"provider_errors":0,"parse_errors":0,"incomplete_responses":0,"question_accuracy_ci":null}},{"topic":"First aid","questions":20,"metrics":{"question_accuracy":0.6333333333333333,"valid_response_accuracy":0.6333333333333333,"coverage":1,"total_questions":60,"correct_questions":38,"valid_questions":60,"failed_questions":0,"provider_errors":0,"parse_errors":0,"incomplete_responses":0,"question_accuracy_ci":null}},{"topic":"Industrial and facility safety","questions":20,"metrics":{"question_accuracy":0.7666666666666667,"valid_response_accuracy":0.7666666666666667,"coverage":1,"total_questions":60,"correct_questions":46,"valid_questions":60,"failed_questions":0,"provider_errors":0,"parse_errors":0,"incomplete_responses":0,"question_accuracy_ci":null}},{"topic":"Occupational safety and health","questions":20,"metrics":{"question_accuracy":0.75,"valid_response_accuracy":0.75,"coverage":1,"total_questions":60,"correct_questions":45,"valid_questions":60,"failed_questions":0,"provider_errors":0,"parse_errors":0,"incomplete_responses":0,"question_accuracy_ci":null}},{"topic":"Regulatory administration","questions":2,"metrics":{"question_accuracy":0.5,"valid_response_accuracy":0.5,"coverage":1,"total_questions":6,"correct_questions":3,"valid_questions":6,"failed_questions":0,"provider_errors":0,"parse_errors":0,"incomplete_responses":0,"question_accuracy_ci":null}}],"baselines":{"all_false_question_accuracy":0,"all_true_question_accuracy":0},"operational_metrics":{"reported_cost_usd":"0.646958388","responses_with_cost":300,"responses_without_cost":0,"prompt_tokens":55107,"completion_tokens":525857,"reasoning_tokens":495296,"latency_median_seconds":4.60867618750126,"latency_p95_seconds":25.23727350000263,"providers":{"Together":300},"format_compliant_responses":155,"format_checked_responses":300},"infrastructure_attempt_count":0},{"model":"deepseek/deepseek-v4.1-flash","provider":"openai-compatible","track":"structured-output","completed_at":"2026-10-01T18:02:47.972355Z","status":"success","protocol":{"track":"structured-output","name":"hse-multiselect-zero-shot-v10","selected_dataset_sha256":"f8d2c31d1095aed302e4a2c05ea795c9f897f061275265599fc1e5dba212c239","topics_sha256":"4897b9e828053e1c9ebde4729c8af75baffcd547d251e49a21006eab961d1fff","prompt_sha256":"8cf0f6b182fe981a596b4d48bc132286a16ebf2f25ffbb15df4e27c5a12a07c1","implementation_sha256":"9b2c30085d7db94fdb18d9ebd839bf8340992e93211fad108162f4215644a69a","selected_questions":100,"repetitions":3,"temperature":null,"max_output_tokens":16384,"timeout_seconds":180,"sampling_method":"all","sampling_seed":42,"answer_order":"fixed","reasoning_settings":{"effort":"low"},"routing_settings":{"only":["together"],"allow_fallbacks":false,"require_parameters":true},"question_concurrency":8,"request_concurrency":8,"batch_concurrency":1},"metrics":{"question_accuracy":0.7266666666666667,"valid_response_accuracy":0.7266666666666667,"coverage":1,"total_questions":300,"correct_questions":218,"valid_questions":300,"failed_questions":0,"provider_errors":0,"parse_errors":0,"incomplete_responses":0,"question_accuracy_ci":null},"repetition_metrics":[{"question_accuracy":0.73,"valid_response_accuracy":0.73,"coverage":1,"total_questions":100,"correct_questions":73,"valid_questions":100,"failed_questions":0,"provider_errors":0,"parse_errors":0,"incomplete_responses":0,"question_accuracy_ci":{"method":"wilson","confidence_level":0.95,"lower":0.6356788323205499,"upper":0.8073041585042365}},{"question_accuracy":0.73,"valid_response_accuracy":0.73,"coverage":1,"total_questions":100,"correct_questions":73,"valid_questions":100,"failed_questions":0,"provider_errors":0,"parse_errors":0,"incomplete_responses":0,"question_accuracy_ci":{"method":"wilson","confidence_level":0.95,"lower":0.6356788323205499,"upper":0.8073041585042365}},{"question_accuracy":0.72,"valid_response_accuracy":0.72,"coverage":1,"total_questions":100,"correct_questions":72,"valid_questions":100,"failed_questions":0,"provider_errors":0,"parse_errors":0,"incomplete_responses":0,"question_accuracy_ci":{"method":"wilson","confidence_level":0.95,"lower":0.6251197129007884,"upper":0.7986031478881379}}],"topic_results":[{"topic":"Environmental protection and waste","questions":20,"metrics":{"question_accuracy":0.65,"valid_response_accuracy":0.65,"coverage":1,"total_questions":60,"correct_questions":39,"valid_questions":60,"failed_questions":0,"provider_errors":0,"parse_errors":0,"incomplete_responses":0,"question_accuracy_ci":null}},{"topic":"Fire safety","questions":18,"metrics":{"question_accuracy":0.7962962962962963,"valid_response_accuracy":0.7962962962962963,"coverage":1,"total_questions":54,"correct_questions":43,"valid_questions":54,"failed_questions":0,"provider_errors":0,"parse_errors":0,"incomplete_responses":0,"question_accuracy_ci":null}},{"topic":"First aid","questions":20,"metrics":{"question_accuracy":0.5833333333333334,"valid_response_accuracy":0.5833333333333334,"coverage":1,"total_questions":60,"correct_questions":35,"valid_questions":60,"failed_questions":0,"provider_errors":0,"parse_errors":0,"incomplete_responses":0,"question_accuracy_ci":null}},{"topic":"Industrial and facility safety","questions":20,"metrics":{"question_accuracy":0.75,"valid_response_accuracy":0.75,"coverage":1,"total_questions":60,"correct_questions":45,"valid_questions":60,"failed_questions":0,"provider_errors":0,"parse_errors":0,"incomplete_responses":0,"question_accuracy_ci":null}},{"topic":"Occupational safety and health","questions":20,"metrics":{"question_accuracy":0.8666666666666667,"valid_response_accuracy":0.8666666666666667,"coverage":1,"total_questions":60,"correct_questions":52,"valid_questions":60,"failed_questions":0,"provider_errors":0,"parse_errors":0,"incomplete_responses":0,"question_accuracy_ci":null}},{"topic":"Regulatory administration","questions":2,"metrics":{"question_accuracy":0.6666666666666666,"valid_response_accuracy":0.6666666666666666,"coverage":1,"total_questions":6,"correct_questions":4,"valid_questions":6,"failed_questions":0,"provider_errors":0,"parse_errors":0,"incomplete_responses":0,"question_accuracy_ci":null}}],"baselines":{"all_false_question_accuracy":0,"all_true_question_accuracy":0},"operational_metrics":{"reported_cost_usd":"0.497246772","responses_with_cost":300,"responses_without_cost":0,"prompt_tokens":45807,"completion_tokens":403046,"reasoning_tokens":399349,"latency_median_seconds":4.001577958497364,"latency_p95_seconds":20.451344458000676,"providers":{"Together":300},"format_compliant_responses":300,"format_checked_responses":300},"infrastructure_attempt_count":0},{"model":"google/gemini-3.8-flash","provider":"openai-compatible","track":"answer-only","completed_at":"2026-10-01T18:09:33.134965Z","status":"success","protocol":{"track":"answer-only","name":"hse-multiselect-zero-shot-v10","selected_dataset_sha256":"f8d2c31d1095aed302e4a2c05ea795c9f897f061275265599fc1e5dba212c239","topics_sha256":"4897b9e828053e1c9ebde4729c8af75baffcd547d251e49a21006eab961d1fff","prompt_sha256":"d1ed6b553cb6997860d5eb7cb36e6399f091187c175b5872e568e40676ab5e68","implementation_sha256":"9b2c30085d7db94fdb18d9ebd839bf8340992e93211fad108162f4215644a69a","selected_questions":100,"repetitions":3,"temperature":null,"max_output_tokens":16384,"timeout_seconds":180,"sampling_method":"all","sampling_seed":42,"answer_order":"fixed","reasoning_settings":{"effort":"low"},"routing_settings":{"only":["google-ai-studio"],"allow_fallbacks":false,"require_parameters":true},"question_concurrency":8,"request_concurrency":8,"batch_concurrency":1},"metrics":{"question_accuracy":0.8233333333333334,"valid_response_accuracy":0.8233333333333334,"coverage":1,"total_questions":300,"correct_questions":247,"valid_questions":300,"failed_questions":0,"provider_errors":0,"parse_errors":0,"incomplete_responses":0,"question_accuracy_ci":null},"repetition_metrics":[{"question_accuracy":0.82,"valid_response_accuracy":0.82,"coverage":1,"total_questions":100,"correct_questions":82,"valid_questions":100,"failed_questions":0,"provider_errors":0,"parse_errors":0,"incomplete_responses":0,"question_accuracy_ci":{"method":"wilson","confidence_level":0.95,"lower":0.7333264161369926,"upper":0.8829977450105364}},{"question_accuracy":0.83,"valid_response_accuracy":0.83,"coverage":1,"total_questions":100,"correct_questions":83,"valid_questions":100,"failed_questions":0,"provider_errors":0,"parse_errors":0,"incomplete_responses":0,"question_accuracy_ci":{"method":"wilson","confidence_level":0.95,"lower":0.7445199523239887,"upper":0.8910643388594006}},{"question_accuracy":0.82,"valid_response_accuracy":0.82,"coverage":1,"total_questions":100,"correct_questions":82,"valid_questions":100,"failed_questions":0,"provider_errors":0,"parse_errors":0,"incomplete_responses":0,"question_accuracy_ci":{"method":"wilson","confidence_level":0.95,"lower":0.7333264161369926,"upper":0.8829977450105364}}],"topic_results":[{"topic":"Environmental protection and waste","questions":20,"metrics":{"question_accuracy":0.7333333333333333,"valid_response_accuracy":0.7333333333333333,"coverage":1,"total_questions":60,"correct_questions":44,"valid_questions":60,"failed_questions":0,"provider_errors":0,"parse_errors":0,"incomplete_responses":0,"question_accuracy_ci":null}},{"topic":"Fire safety","questions":18,"metrics":{"question_accuracy":0.9444444444444444,"valid_response_accuracy":0.9444444444444444,"coverage":1,"total_questions":54,"correct_questions":51,"valid_questions":54,"failed_questions":0,"provider_errors":0,"parse_errors":0,"incomplete_responses":0,"question_accuracy_ci":null}},{"topic":"First aid","questions":20,"metrics":{"question_accuracy":0.5333333333333333,"valid_response_accuracy":0.5333333333333333,"coverage":1,"total_questions":60,"correct_questions":32,"valid_questions":60,"failed_questions":0,"provider_errors":0,"parse_errors":0,"incomplete_responses":0,"question_accuracy_ci":null}},{"topic":"Industrial and facility safety","questions":20,"metrics":{"question_accuracy":1,"valid_response_accuracy":1,"coverage":1,"total_questions":60,"correct_questions":60,"valid_questions":60,"failed_questions":0,"provider_errors":0,"parse_errors":0,"incomplete_responses":0,"question_accuracy_ci":null}},{"topic":"Occupational safety and health","questions":20,"metrics":{"question_accuracy":0.9,"valid_response_accuracy":0.9,"coverage":1,"total_questions":60,"correct_questions":54,"valid_questions":60,"failed_questions":0,"provider_errors":0,"parse_errors":0,"incomplete_responses":0,"question_accuracy_ci":null}},{"topic":"Regulatory administration","questions":2,"metrics":{"question_accuracy":1,"valid_response_accuracy":1,"coverage":1,"total_questions":6,"correct_questions":6,"valid_questions":6,"failed_questions":0,"provider_errors":0,"parse_errors":0,"incomplete_responses":0,"question_accuracy_ci":null}}],"baselines":{"all_false_question_accuracy":0,"all_true_question_accuracy":0},"operational_metrics":{"reported_cost_usd":"0.50241750","responses_with_cost":300,"responses_without_cost":0,"prompt_tokens":55950,"completion_tokens":122788,"reasoning_tokens":120689,"latency_median_seconds":2.7495261460026086,"latency_p95_seconds":6.208739292000246,"providers":{"Google AI Studio":300},"format_compliant_responses":300,"format_checked_responses":300},"infrastructure_attempt_count":0},{"model":"google/gemini-3.8-flash","provider":"openai-compatible","track":"reasoning-assisted","completed_at":"2026-10-01T18:12:10.221676Z","status":"success","protocol":{"track":"reasoning-assisted","name":"hse-multiselect-zero-shot-v10","selected_dataset_sha256":"f8d2c31d1095aed302e4a2c05ea795c9f897f061275265599fc1e5dba212c239","topics_sha256":"4897b9e828053e1c9ebde4729c8af75baffcd547d251e49a21006eab961d1fff","prompt_sha256":"1e82d8ac2c47f18b33cf9cda4901214138aba1621d45b4d34123be7253a8a2f6","implementation_sha256":"9b2c30085d7db94fdb18d9ebd839bf8340992e93211fad108162f4215644a69a","selected_questions":100,"repetitions":3,"temperature":null,"max_output_tokens":16384,"timeout_seconds":180,"sampling_method":"all","sampling_seed":42,"answer_order":"fixed","reasoning_settings":{"effort":"low"},"routing_settings":{"only":["google-ai-studio"],"allow_fallbacks":false,"require_parameters":true},"question_concurrency":8,"request_concurrency":8,"batch_concurrency":1},"metrics":{"question_accuracy":0.86,"valid_response_accuracy":0.86,"coverage":1,"total_questions":300,"correct_questions":258,"valid_questions":300,"failed_questions":0,"provider_errors":0,"parse_errors":0,"incomplete_responses":0,"question_accuracy_ci":null},"repetition_metrics":[{"question_accuracy":0.86,"valid_response_accuracy":0.86,"coverage":1,"total_questions":100,"correct_questions":86,"valid_questions":100,"failed_questions":0,"provider_errors":0,"parse_errors":0,"incomplete_responses":0,"question_accuracy_ci":{"method":"wilson","confidence_level":0.95,"lower":0.7786281178903616,"upper":0.9147365634006086}},{"question_accuracy":0.86,"valid_response_accuracy":0.86,"coverage":1,"total_questions":100,"correct_questions":86,"valid_questions":100,"failed_questions":0,"provider_errors":0,"parse_errors":0,"incomplete_responses":0,"question_accuracy_ci":{"method":"wilson","confidence_level":0.95,"lower":0.7786281178903616,"upper":0.9147365634006086}},{"question_accuracy":0.86,"valid_response_accuracy":0.86,"coverage":1,"total_questions":100,"correct_questions":86,"valid_questions":100,"failed_questions":0,"provider_errors":0,"parse_errors":0,"incomplete_responses":0,"question_accuracy_ci":{"method":"wilson","confidence_level":0.95,"lower":0.7786281178903616,"upper":0.9147365634006086}}],"topic_results":[{"topic":"Environmental protection and waste","questions":20,"metrics":{"question_accuracy":0.8166666666666667,"valid_response_accuracy":0.8166666666666667,"coverage":1,"total_questions":60,"correct_questions":49,"valid_questions":60,"failed_questions":0,"provider_errors":0,"parse_errors":0,"incomplete_responses":0,"question_accuracy_ci":null}},{"topic":"Fire safety","questions":18,"metrics":{"question_accuracy":0.9629629629629629,"valid_response_accuracy":0.9629629629629629,"coverage":1,"total_questions":54,"correct_questions":52,"valid_questions":54,"failed_questions":0,"provider_errors":0,"parse_errors":0,"incomplete_responses":0,"question_accuracy_ci":null}},{"topic":"First aid","questions":20,"metrics":{"question_accuracy":0.6166666666666667,"valid_response_accuracy":0.6166666666666667,"coverage":1,"total_questions":60,"correct_questions":37,"valid_questions":60,"failed_questions":0,"provider_errors":0,"parse_errors":0,"incomplete_responses":0,"question_accuracy_ci":null}},{"topic":"Industrial and facility safety","questions":20,"metrics":{"question_accuracy":1,"valid_response_accuracy":1,"coverage":1,"total_questions":60,"correct_questions":60,"valid_questions":60,"failed_questions":0,"provider_errors":0,"parse_errors":0,"incomplete_responses":0,"question_accuracy_ci":null}},{"topic":"Occupational safety and health","questions":20,"metrics":{"question_accuracy":0.9,"valid_response_accuracy":0.9,"coverage":1,"total_questions":60,"correct_questions":54,"valid_questions":60,"failed_questions":0,"provider_errors":0,"parse_errors":0,"incomplete_responses":0,"question_accuracy_ci":null}},{"topic":"Regulatory administration","questions":2,"metrics":{"question_accuracy":1,"valid_response_accuracy":1,"coverage":1,"total_questions":6,"correct_questions":6,"valid_questions":6,"failed_questions":0,"provider_errors":0,"parse_errors":0,"incomplete_responses":0,"question_accuracy_ci":null}}],"baselines":{"all_false_question_accuracy":0,"all_true_question_accuracy":0},"operational_metrics":{"reported_cost_usd":"0.71250000","responses_with_cost":300,"responses_without_cost":0,"prompt_tokens":48450,"completion_tokens":180310,"reasoning_tokens":125396,"latency_median_seconds":3.385880604000704,"latency_p95_seconds":8.060429332996137,"providers":{"Google AI Studio":300},"format_compliant_responses":300,"format_checked_responses":300},"infrastructure_attempt_count":0},{"model":"google/gemini-3.8-flash","provider":"openai-compatible","track":"structured-output","completed_at":"2026-10-01T18:14:02.946419Z","status":"success","protocol":{"track":"structured-output","name":"hse-multiselect-zero-shot-v10","selected_dataset_sha256":"f8d2c31d1095aed302e4a2c05ea795c9f897f061275265599fc1e5dba212c239","topics_sha256":"4897b9e828053e1c9ebde4729c8af75baffcd547d251e49a21006eab961d1fff","prompt_sha256":"8cf0f6b182fe981a596b4d48bc132286a16ebf2f25ffbb15df4e27c5a12a07c1","implementation_sha256":"9b2c30085d7db94fdb18d9ebd839bf8340992e93211fad108162f4215644a69a","selected_questions":100,"repetitions":3,"temperature":null,"max_output_tokens":16384,"timeout_seconds":180,"sampling_method":"all","sampling_seed":42,"answer_order":"fixed","reasoning_settings":{"effort":"low"},"routing_settings":{"only":["google-ai-studio"],"allow_fallbacks":false,"require_parameters":true},"question_concurrency":8,"request_concurrency":8,"batch_concurrency":1},"metrics":{"question_accuracy":0.8633333333333333,"valid_response_accuracy":0.8633333333333333,"coverage":1,"total_questions":300,"correct_questions":259,"valid_questions":300,"failed_questions":0,"provider_errors":0,"parse_errors":0,"incomplete_responses":0,"question_accuracy_ci":null},"repetition_metrics":[{"question_accuracy":0.89,"valid_response_accuracy":0.89,"coverage":1,"total_questions":100,"correct_questions":89,"valid_questions":100,"failed_questions":0,"provider_errors":0,"parse_errors":0,"incomplete_responses":0,"question_accuracy_ci":{"method":"wilson","confidence_level":0.95,"lower":0.8136870349691969,"upper":0.9374580364293543}},{"question_accuracy":0.85,"valid_response_accuracy":0.85,"coverage":1,"total_questions":100,"correct_questions":85,"valid_questions":100,"failed_questions":0,"provider_errors":0,"parse_errors":0,"incomplete_responses":0,"question_accuracy_ci":{"method":"wilson","confidence_level":0.95,"lower":0.7671644040916763,"upper":0.9069401471634337}},{"question_accuracy":0.85,"valid_response_accuracy":0.85,"coverage":1,"total_questions":100,"correct_questions":85,"valid_questions":100,"failed_questions":0,"provider_errors":0,"parse_errors":0,"incomplete_responses":0,"question_accuracy_ci":{"method":"wilson","confidence_level":0.95,"lower":0.7671644040916763,"upper":0.9069401471634337}}],"topic_results":[{"topic":"Environmental protection and waste","questions":20,"metrics":{"question_accuracy":0.8833333333333333,"valid_response_accuracy":0.8833333333333333,"coverage":1,"total_questions":60,"correct_questions":53,"valid_questions":60,"failed_questions":0,"provider_errors":0,"parse_errors":0,"incomplete_responses":0,"question_accuracy_ci":null}},{"topic":"Fire safety","questions":18,"metrics":{"question_accuracy":0.9444444444444444,"valid_response_accuracy":0.9444444444444444,"coverage":1,"total_questions":54,"correct_questions":51,"valid_questions":54,"failed_questions":0,"provider_errors":0,"parse_errors":0,"incomplete_responses":0,"question_accuracy_ci":null}},{"topic":"First aid","questions":20,"metrics":{"question_accuracy":0.5833333333333334,"valid_response_accuracy":0.5833333333333334,"coverage":1,"total_questions":60,"correct_questions":35,"valid_questions":60,"failed_questions":0,"provider_errors":0,"parse_errors":0,"incomplete_responses":0,"question_accuracy_ci":null}},{"topic":"Industrial and facility safety","questions":20,"metrics":{"question_accuracy":0.9833333333333333,"valid_response_accuracy":0.9833333333333333,"coverage":1,"total_questions":60,"correct_questions":59,"valid_questions":60,"failed_questions":0,"provider_errors":0,"parse_errors":0,"incomplete_responses":0,"question_accuracy_ci":null}},{"topic":"Occupational safety and health","questions":20,"metrics":{"question_accuracy":0.9166666666666666,"valid_response_accuracy":0.9166666666666666,"coverage":1,"total_questions":60,"correct_questions":55,"valid_questions":60,"failed_questions":0,"provider_errors":0,"parse_errors":0,"incomplete_responses":0,"question_accuracy_ci":null}},{"topic":"Regulatory administration","questions":2,"metrics":{"question_accuracy":1,"valid_response_accuracy":1,"coverage":1,"total_questions":6,"correct_questions":6,"valid_questions":6,"failed_questions":0,"provider_errors":0,"parse_errors":0,"incomplete_responses":0,"question_accuracy_ci":null}}],"baselines":{"all_false_question_accuracy":0,"all_true_question_accuracy":0},"operational_metrics":{"reported_cost_usd":"0.40917750","responses_with_cost":300,"responses_without_cost":0,"prompt_tokens":39450,"completion_tokens":101224,"reasoning_tokens":95900,"latency_median_seconds":2.537120604498341,"latency_p95_seconds":5.424734416003048,"providers":{"Google AI Studio":300},"format_compliant_responses":300,"format_checked_responses":300},"infrastructure_attempt_count":0},{"model":"z-ai/glm-5.3","provider":"openai-compatible","track":"answer-only","completed_at":"2026-10-01T18:56:23.569044Z","status":"success","protocol":{"track":"answer-only","name":"hse-multiselect-zero-shot-v10","selected_dataset_sha256":"f8d2c31d1095aed302e4a2c05ea795c9f897f061275265599fc1e5dba212c239","topics_sha256":"4897b9e828053e1c9ebde4729c8af75baffcd547d251e49a21006eab961d1fff","prompt_sha256":"d1ed6b553cb6997860d5eb7cb36e6399f091187c175b5872e568e40676ab5e68","implementation_sha256":"9b2c30085d7db94fdb18d9ebd839bf8340992e93211fad108162f4215644a69a","selected_questions":100,"repetitions":3,"temperature":null,"max_output_tokens":16384,"timeout_seconds":180,"sampling_method":"all","sampling_seed":42,"answer_order":"fixed","reasoning_settings":{"effort":"low"},"routing_settings":{"only":["morph/fp8"],"allow_fallbacks":false,"require_parameters":true},"question_concurrency":8,"request_concurrency":8,"batch_concurrency":1},"metrics":{"question_accuracy":0.6633333333333333,"valid_response_accuracy":0.6633333333333333,"coverage":1,"total_questions":300,"correct_questions":199,"valid_questions":300,"failed_questions":0,"provider_errors":0,"parse_errors":0,"incomplete_responses":0,"question_accuracy_ci":null},"repetition_metrics":[{"question_accuracy":0.66,"valid_response_accuracy":0.66,"coverage":1,"total_questions":100,"correct_questions":66,"valid_questions":100,"failed_questions":0,"provider_errors":0,"parse_errors":0,"incomplete_responses":0,"question_accuracy_ci":{"method":"wilson","confidence_level":0.95,"lower":0.5627772885472462,"upper":0.7453847920265184}},{"question_accuracy":0.64,"valid_response_accuracy":0.64,"coverage":1,"total_questions":100,"correct_questions":64,"valid_questions":100,"failed_questions":0,"provider_errors":0,"parse_errors":0,"incomplete_responses":0,"question_accuracy_ci":{"method":"wilson","confidence_level":0.95,"lower":0.5423540245160874,"upper":0.7272877959859567}},{"question_accuracy":0.69,"valid_response_accuracy":0.69,"coverage":1,"total_questions":100,"correct_questions":69,"valid_questions":100,"failed_questions":0,"provider_errors":0,"parse_errors":0,"incomplete_responses":0,"question_accuracy_ci":{"method":"wilson","confidence_level":0.95,"lower":0.5937394428051057,"upper":0.7722030278762395}}],"topic_results":[{"topic":"Environmental protection and waste","questions":20,"metrics":{"question_accuracy":0.5166666666666667,"valid_response_accuracy":0.5166666666666667,"coverage":1,"total_questions":60,"correct_questions":31,"valid_questions":60,"failed_questions":0,"provider_errors":0,"parse_errors":0,"incomplete_responses":0,"question_accuracy_ci":null}},{"topic":"Fire safety","questions":18,"metrics":{"question_accuracy":0.7222222222222222,"valid_response_accuracy":0.7222222222222222,"coverage":1,"total_questions":54,"correct_questions":39,"valid_questions":54,"failed_questions":0,"provider_errors":0,"parse_errors":0,"incomplete_responses":0,"question_accuracy_ci":null}},{"topic":"First aid","questions":20,"metrics":{"question_accuracy":0.5,"valid_response_accuracy":0.5,"coverage":1,"total_questions":60,"correct_questions":30,"valid_questions":60,"failed_questions":0,"provider_errors":0,"parse_errors":0,"incomplete_responses":0,"question_accuracy_ci":null}},{"topic":"Industrial and facility safety","questions":20,"metrics":{"question_accuracy":0.8,"valid_response_accuracy":0.8,"coverage":1,"total_questions":60,"correct_questions":48,"valid_questions":60,"failed_questions":0,"provider_errors":0,"parse_errors":0,"incomplete_responses":0,"question_accuracy_ci":null}},{"topic":"Occupational safety and health","questions":20,"metrics":{"question_accuracy":0.7666666666666667,"valid_response_accuracy":0.7666666666666667,"coverage":1,"total_questions":60,"correct_questions":46,"valid_questions":60,"failed_questions":0,"provider_errors":0,"parse_errors":0,"incomplete_responses":0,"question_accuracy_ci":null}},{"topic":"Regulatory administration","questions":2,"metrics":{"question_accuracy":0.8333333333333334,"valid_response_accuracy":0.8333333333333334,"coverage":1,"total_questions":6,"correct_questions":5,"valid_questions":6,"failed_questions":0,"provider_errors":0,"parse_errors":0,"incomplete_responses":0,"question_accuracy_ci":null}}],"baselines":{"all_false_question_accuracy":0,"all_true_question_accuracy":0},"operational_metrics":{"reported_cost_usd":"0.199945515","responses_with_cost":300,"responses_without_cost":0,"prompt_tokens":56145,"completion_tokens":71426,"reasoning_tokens":68761,"latency_median_seconds":6.900185500002408,"latency_p95_seconds":29.675432458003343,"providers":{"Morph":300},"format_compliant_responses":300,"format_checked_responses":300},"infrastructure_attempt_count":3},{"model":"z-ai/glm-5.3","provider":"openai-compatible","track":"reasoning-assisted","completed_at":"2026-10-01T19:14:15.105748Z","status":"success","protocol":{"track":"reasoning-assisted","name":"hse-multiselect-zero-shot-v10","selected_dataset_sha256":"f8d2c31d1095aed302e4a2c05ea795c9f897f061275265599fc1e5dba212c239","topics_sha256":"4897b9e828053e1c9ebde4729c8af75baffcd547d251e49a21006eab961d1fff","prompt_sha256":"1e82d8ac2c47f18b33cf9cda4901214138aba1621d45b4d34123be7253a8a2f6","implementation_sha256":"9b2c30085d7db94fdb18d9ebd839bf8340992e93211fad108162f4215644a69a","selected_questions":100,"repetitions":3,"temperature":null,"max_output_tokens":16384,"timeout_seconds":180,"sampling_method":"all","sampling_seed":42,"answer_order":"fixed","reasoning_settings":{"effort":"low"},"routing_settings":{"only":["morph/fp8"],"allow_fallbacks":false,"require_parameters":true},"question_concurrency":8,"request_concurrency":8,"batch_concurrency":1},"metrics":{"question_accuracy":0.6933333333333334,"valid_response_accuracy":0.6933333333333334,"coverage":1,"total_questions":300,"correct_questions":208,"valid_questions":300,"failed_questions":0,"provider_errors":0,"parse_errors":0,"incomplete_responses":0,"question_accuracy_ci":null},"repetition_metrics":[{"question_accuracy":0.7,"valid_response_accuracy":0.7,"coverage":1,"total_questions":100,"correct_questions":70,"valid_questions":100,"failed_questions":0,"provider_errors":0,"parse_errors":0,"incomplete_responses":0,"question_accuracy_ci":{"method":"wilson","confidence_level":0.95,"lower":0.6041514536665332,"upper":0.7810511470506724}},{"question_accuracy":0.69,"valid_response_accuracy":0.69,"coverage":1,"total_questions":100,"correct_questions":69,"valid_questions":100,"failed_questions":0,"provider_errors":0,"parse_errors":0,"incomplete_responses":0,"question_accuracy_ci":{"method":"wilson","confidence_level":0.95,"lower":0.5937394428051057,"upper":0.7722030278762395}},{"question_accuracy":0.69,"valid_response_accuracy":0.69,"coverage":1,"total_questions":100,"correct_questions":69,"valid_questions":100,"failed_questions":0,"provider_errors":0,"parse_errors":0,"incomplete_responses":0,"question_accuracy_ci":{"method":"wilson","confidence_level":0.95,"lower":0.5937394428051057,"upper":0.7722030278762395}}],"topic_results":[{"topic":"Environmental protection and waste","questions":20,"metrics":{"question_accuracy":0.6166666666666667,"valid_response_accuracy":0.6166666666666667,"coverage":1,"total_questions":60,"correct_questions":37,"valid_questions":60,"failed_questions":0,"provider_errors":0,"parse_errors":0,"incomplete_responses":0,"question_accuracy_ci":null}},{"topic":"Fire safety","questions":18,"metrics":{"question_accuracy":0.6851851851851852,"valid_response_accuracy":0.6851851851851852,"coverage":1,"total_questions":54,"correct_questions":37,"valid_questions":54,"failed_questions":0,"provider_errors":0,"parse_errors":0,"incomplete_responses":0,"question_accuracy_ci":null}},{"topic":"First aid","questions":20,"metrics":{"question_accuracy":0.6333333333333333,"valid_response_accuracy":0.6333333333333333,"coverage":1,"total_questions":60,"correct_questions":38,"valid_questions":60,"failed_questions":0,"provider_errors":0,"parse_errors":0,"incomplete_responses":0,"question_accuracy_ci":null}},{"topic":"Industrial and facility safety","questions":20,"metrics":{"question_accuracy":0.8166666666666667,"valid_response_accuracy":0.8166666666666667,"coverage":1,"total_questions":60,"correct_questions":49,"valid_questions":60,"failed_questions":0,"provider_errors":0,"parse_errors":0,"incomplete_responses":0,"question_accuracy_ci":null}},{"topic":"Occupational safety and health","questions":20,"metrics":{"question_accuracy":0.7,"valid_response_accuracy":0.7,"coverage":1,"total_questions":60,"correct_questions":42,"valid_questions":60,"failed_questions":0,"provider_errors":0,"parse_errors":0,"incomplete_responses":0,"question_accuracy_ci":null}},{"topic":"Regulatory administration","questions":2,"metrics":{"question_accuracy":0.8333333333333334,"valid_response_accuracy":0.8333333333333334,"coverage":1,"total_questions":6,"correct_questions":5,"valid_questions":6,"failed_questions":0,"provider_errors":0,"parse_errors":0,"incomplete_responses":0,"question_accuracy_ci":null}}],"baselines":{"all_false_question_accuracy":0,"all_true_question_accuracy":0},"operational_metrics":{"reported_cost_usd":"0.401283261","responses_with_cost":300,"responses_without_cost":0,"prompt_tokens":48645,"completion_tokens":146589,"reasoning_tokens":69345,"latency_median_seconds":16.22496835450147,"latency_p95_seconds":51.97404800000368,"providers":{"Morph":300},"format_compliant_responses":6,"format_checked_responses":300},"infrastructure_attempt_count":2},{"model":"z-ai/glm-5.3","provider":"openai-compatible","track":"structured-output","completed_at":"2026-10-01T19:16:04.260219Z","status":"success","protocol":{"track":"structured-output","name":"hse-multiselect-zero-shot-v10","selected_dataset_sha256":"f8d2c31d1095aed302e4a2c05ea795c9f897f061275265599fc1e5dba212c239","topics_sha256":"4897b9e828053e1c9ebde4729c8af75baffcd547d251e49a21006eab961d1fff","prompt_sha256":"8cf0f6b182fe981a596b4d48bc132286a16ebf2f25ffbb15df4e27c5a12a07c1","implementation_sha256":"9b2c30085d7db94fdb18d9ebd839bf8340992e93211fad108162f4215644a69a","selected_questions":100,"repetitions":3,"temperature":null,"max_output_tokens":16384,"timeout_seconds":180,"sampling_method":"all","sampling_seed":42,"answer_order":"fixed","reasoning_settings":{"effort":"low"},"routing_settings":{"only":["morph/fp8"],"allow_fallbacks":false,"require_parameters":true},"question_concurrency":8,"request_concurrency":8,"batch_concurrency":1},"metrics":{"question_accuracy":0.51,"valid_response_accuracy":0.51,"coverage":1,"total_questions":300,"correct_questions":153,"valid_questions":300,"failed_questions":0,"provider_errors":0,"parse_errors":0,"incomplete_responses":0,"question_accuracy_ci":null},"repetition_metrics":[{"question_accuracy":0.51,"valid_response_accuracy":0.51,"coverage":1,"total_questions":100,"correct_questions":51,"valid_questions":100,"failed_questions":0,"provider_errors":0,"parse_errors":0,"incomplete_responses":0,"question_accuracy_ci":{"method":"wilson","confidence_level":0.95,"lower":0.4134801193402717,"upper":0.6057800106955886}},{"question_accuracy":0.53,"valid_response_accuracy":0.53,"coverage":1,"total_questions":100,"correct_questions":53,"valid_questions":100,"failed_questions":0,"provider_errors":0,"parse_errors":0,"incomplete_responses":0,"question_accuracy_ci":{"method":"wilson","confidence_level":0.95,"lower":0.4328885697009936,"upper":0.6248918204065872}},{"question_accuracy":0.49,"valid_response_accuracy":0.49,"coverage":1,"total_questions":100,"correct_questions":49,"valid_questions":100,"failed_questions":0,"provider_errors":0,"parse_errors":0,"incomplete_responses":0,"question_accuracy_ci":{"method":"wilson","confidence_level":0.95,"lower":0.3942199893044113,"upper":0.5865198806597283}}],"topic_results":[{"topic":"Environmental protection and waste","questions":20,"metrics":{"question_accuracy":0.3333333333333333,"valid_response_accuracy":0.3333333333333333,"coverage":1,"total_questions":60,"correct_questions":20,"valid_questions":60,"failed_questions":0,"provider_errors":0,"parse_errors":0,"incomplete_responses":0,"question_accuracy_ci":null}},{"topic":"Fire safety","questions":18,"metrics":{"question_accuracy":0.6296296296296297,"valid_response_accuracy":0.6296296296296297,"coverage":1,"total_questions":54,"correct_questions":34,"valid_questions":54,"failed_questions":0,"provider_errors":0,"parse_errors":0,"incomplete_responses":0,"question_accuracy_ci":null}},{"topic":"First aid","questions":20,"metrics":{"question_accuracy":0.4166666666666667,"valid_response_accuracy":0.4166666666666667,"coverage":1,"total_questions":60,"correct_questions":25,"valid_questions":60,"failed_questions":0,"provider_errors":0,"parse_errors":0,"incomplete_responses":0,"question_accuracy_ci":null}},{"topic":"Industrial and facility safety","questions":20,"metrics":{"question_accuracy":0.6333333333333333,"valid_response_accuracy":0.6333333333333333,"coverage":1,"total_questions":60,"correct_questions":38,"valid_questions":60,"failed_questions":0,"provider_errors":0,"parse_errors":0,"incomplete_responses":0,"question_accuracy_ci":null}},{"topic":"Occupational safety and health","questions":20,"metrics":{"question_accuracy":0.5833333333333334,"valid_response_accuracy":0.5833333333333334,"coverage":1,"total_questions":60,"correct_questions":35,"valid_questions":60,"failed_questions":0,"provider_errors":0,"parse_errors":0,"incomplete_responses":0,"question_accuracy_ci":null}},{"topic":"Regulatory administration","questions":2,"metrics":{"question_accuracy":0.16666666666666666,"valid_response_accuracy":0.16666666666666666,"coverage":1,"total_questions":6,"correct_questions":1,"valid_questions":6,"failed_questions":0,"provider_errors":0,"parse_errors":0,"incomplete_responses":0,"question_accuracy_ci":null}}],"baselines":{"all_false_question_accuracy":0,"all_true_question_accuracy":0},"operational_metrics":{"reported_cost_usd":"0.015024537","responses_with_cost":300,"responses_without_cost":0,"prompt_tokens":38745,"completion_tokens":3693,"reasoning_tokens":0,"latency_median_seconds":2.3234996874998615,"latency_p95_seconds":6.715239625002141,"providers":{"Morph":300},"format_compliant_responses":300,"format_checked_responses":300},"infrastructure_attempt_count":0},{"model":"openai/gpt-5.6-luna","provider":"openai-compatible","track":"answer-only","completed_at":"2026-10-01T17:36:00.005190Z","status":"success","protocol":{"track":"answer-only","name":"hse-multiselect-zero-shot-v10","selected_dataset_sha256":"f8d2c31d1095aed302e4a2c05ea795c9f897f061275265599fc1e5dba212c239","topics_sha256":"4897b9e828053e1c9ebde4729c8af75baffcd547d251e49a21006eab961d1fff","prompt_sha256":"d1ed6b553cb6997860d5eb7cb36e6399f091187c175b5872e568e40676ab5e68","implementation_sha256":"9b2c30085d7db94fdb18d9ebd839bf8340992e93211fad108162f4215644a69a","selected_questions":100,"repetitions":3,"temperature":null,"max_output_tokens":16384,"timeout_seconds":180,"sampling_method":"all","sampling_seed":42,"answer_order":"fixed","reasoning_settings":{"effort":"low"},"routing_settings":{"only":["openai"],"allow_fallbacks":false,"require_parameters":true},"question_concurrency":8,"request_concurrency":8,"batch_concurrency":1},"metrics":{"question_accuracy":0.6666666666666666,"valid_response_accuracy":0.6666666666666666,"coverage":1,"total_questions":300,"correct_questions":200,"valid_questions":300,"failed_questions":0,"provider_errors":0,"parse_errors":0,"incomplete_responses":0,"question_accuracy_ci":null},"repetition_metrics":[{"question_accuracy":0.7,"valid_response_accuracy":0.7,"coverage":1,"total_questions":100,"correct_questions":70,"valid_questions":100,"failed_questions":0,"provider_errors":0,"parse_errors":0,"incomplete_responses":0,"question_accuracy_ci":{"method":"wilson","confidence_level":0.95,"lower":0.6041514536665332,"upper":0.7810511470506724}},{"question_accuracy":0.66,"valid_response_accuracy":0.66,"coverage":1,"total_questions":100,"correct_questions":66,"valid_questions":100,"failed_questions":0,"provider_errors":0,"parse_errors":0,"incomplete_responses":0,"question_accuracy_ci":{"method":"wilson","confidence_level":0.95,"lower":0.5627772885472462,"upper":0.7453847920265184}},{"question_accuracy":0.64,"valid_response_accuracy":0.64,"coverage":1,"total_questions":100,"correct_questions":64,"valid_questions":100,"failed_questions":0,"provider_errors":0,"parse_errors":0,"incomplete_responses":0,"question_accuracy_ci":{"method":"wilson","confidence_level":0.95,"lower":0.5423540245160874,"upper":0.7272877959859567}}],"topic_results":[{"topic":"Environmental protection and waste","questions":20,"metrics":{"question_accuracy":0.5666666666666667,"valid_response_accuracy":0.5666666666666667,"coverage":1,"total_questions":60,"correct_questions":34,"valid_questions":60,"failed_questions":0,"provider_errors":0,"parse_errors":0,"incomplete_responses":0,"question_accuracy_ci":null}},{"topic":"Fire safety","questions":18,"metrics":{"question_accuracy":0.7222222222222222,"valid_response_accuracy":0.7222222222222222,"coverage":1,"total_questions":54,"correct_questions":39,"valid_questions":54,"failed_questions":0,"provider_errors":0,"parse_errors":0,"incomplete_responses":0,"question_accuracy_ci":null}},{"topic":"First aid","questions":20,"metrics":{"question_accuracy":0.4166666666666667,"valid_response_accuracy":0.4166666666666667,"coverage":1,"total_questions":60,"correct_questions":25,"valid_questions":60,"failed_questions":0,"provider_errors":0,"parse_errors":0,"incomplete_responses":0,"question_accuracy_ci":null}},{"topic":"Industrial and facility safety","questions":20,"metrics":{"question_accuracy":0.85,"valid_response_accuracy":0.85,"coverage":1,"total_questions":60,"correct_questions":51,"valid_questions":60,"failed_questions":0,"provider_errors":0,"parse_errors":0,"incomplete_responses":0,"question_accuracy_ci":null}},{"topic":"Occupational safety and health","questions":20,"metrics":{"question_accuracy":0.75,"valid_response_accuracy":0.75,"coverage":1,"total_questions":60,"correct_questions":45,"valid_questions":60,"failed_questions":0,"provider_errors":0,"parse_errors":0,"incomplete_responses":0,"question_accuracy_ci":null}},{"topic":"Regulatory administration","questions":2,"metrics":{"question_accuracy":1,"valid_response_accuracy":1,"coverage":1,"total_questions":6,"correct_questions":6,"valid_questions":6,"failed_questions":0,"provider_errors":0,"parse_errors":0,"incomplete_responses":0,"question_accuracy_ci":null}}],"baselines":{"all_false_question_accuracy":0,"all_true_question_accuracy":0},"operational_metrics":{"reported_cost_usd":"0.0613668","responses_with_cost":300,"responses_without_cost":0,"prompt_tokens":55158,"completion_tokens":41946,"reasoning_tokens":37722,"latency_median_seconds":3.3388899999990826,"latency_p95_seconds":7.194955999999365,"providers":{"OpenAI":300},"format_compliant_responses":300,"format_checked_responses":300},"infrastructure_attempt_count":9},{"model":"openai/gpt-5.6-luna","provider":"openai-compatible","track":"reasoning-assisted","completed_at":"2026-10-01T17:39:03.441153Z","status":"success","protocol":{"track":"reasoning-assisted","name":"hse-multiselect-zero-shot-v10","selected_dataset_sha256":"f8d2c31d1095aed302e4a2c05ea795c9f897f061275265599fc1e5dba212c239","topics_sha256":"4897b9e828053e1c9ebde4729c8af75baffcd547d251e49a21006eab961d1fff","prompt_sha256":"1e82d8ac2c47f18b33cf9cda4901214138aba1621d45b4d34123be7253a8a2f6","implementation_sha256":"9b2c30085d7db94fdb18d9ebd839bf8340992e93211fad108162f4215644a69a","selected_questions":100,"repetitions":3,"temperature":null,"max_output_tokens":16384,"timeout_seconds":180,"sampling_method":"all","sampling_seed":42,"answer_order":"fixed","reasoning_settings":{"effort":"low"},"routing_settings":{"only":["openai"],"allow_fallbacks":false,"require_parameters":true},"question_concurrency":8,"request_concurrency":8,"batch_concurrency":1},"metrics":{"question_accuracy":0.6866666666666666,"valid_response_accuracy":0.6866666666666666,"coverage":1,"total_questions":300,"correct_questions":206,"valid_questions":300,"failed_questions":0,"provider_errors":0,"parse_errors":0,"incomplete_responses":0,"question_accuracy_ci":null},"repetition_metrics":[{"question_accuracy":0.66,"valid_response_accuracy":0.66,"coverage":1,"total_questions":100,"correct_questions":66,"valid_questions":100,"failed_questions":0,"provider_errors":0,"parse_errors":0,"incomplete_responses":0,"question_accuracy_ci":{"method":"wilson","confidence_level":0.95,"lower":0.5627772885472462,"upper":0.7453847920265184}},{"question_accuracy":0.7,"valid_response_accuracy":0.7,"coverage":1,"total_questions":100,"correct_questions":70,"valid_questions":100,"failed_questions":0,"provider_errors":0,"parse_errors":0,"incomplete_responses":0,"question_accuracy_ci":{"method":"wilson","confidence_level":0.95,"lower":0.6041514536665332,"upper":0.7810511470506724}},{"question_accuracy":0.7,"valid_response_accuracy":0.7,"coverage":1,"total_questions":100,"correct_questions":70,"valid_questions":100,"failed_questions":0,"provider_errors":0,"parse_errors":0,"incomplete_responses":0,"question_accuracy_ci":{"method":"wilson","confidence_level":0.95,"lower":0.6041514536665332,"upper":0.7810511470506724}}],"topic_results":[{"topic":"Environmental protection and waste","questions":20,"metrics":{"question_accuracy":0.5666666666666667,"valid_response_accuracy":0.5666666666666667,"coverage":1,"total_questions":60,"correct_questions":34,"valid_questions":60,"failed_questions":0,"provider_errors":0,"parse_errors":0,"incomplete_responses":0,"question_accuracy_ci":null}},{"topic":"Fire safety","questions":18,"metrics":{"question_accuracy":0.6851851851851852,"valid_response_accuracy":0.6851851851851852,"coverage":1,"total_questions":54,"correct_questions":37,"valid_questions":54,"failed_questions":0,"provider_errors":0,"parse_errors":0,"incomplete_responses":0,"question_accuracy_ci":null}},{"topic":"First aid","questions":20,"metrics":{"question_accuracy":0.4666666666666667,"valid_response_accuracy":0.4666666666666667,"coverage":1,"total_questions":60,"correct_questions":28,"valid_questions":60,"failed_questions":0,"provider_errors":0,"parse_errors":0,"incomplete_responses":0,"question_accuracy_ci":null}},{"topic":"Industrial and facility safety","questions":20,"metrics":{"question_accuracy":0.9166666666666666,"valid_response_accuracy":0.9166666666666666,"coverage":1,"total_questions":60,"correct_questions":55,"valid_questions":60,"failed_questions":0,"provider_errors":0,"parse_errors":0,"incomplete_responses":0,"question_accuracy_ci":null}},{"topic":"Occupational safety and health","questions":20,"metrics":{"question_accuracy":0.7666666666666667,"valid_response_accuracy":0.7666666666666667,"coverage":1,"total_questions":60,"correct_questions":46,"valid_questions":60,"failed_questions":0,"provider_errors":0,"parse_errors":0,"incomplete_responses":0,"question_accuracy_ci":null}},{"topic":"Regulatory administration","questions":2,"metrics":{"question_accuracy":1,"valid_response_accuracy":1,"coverage":1,"total_questions":6,"correct_questions":6,"valid_questions":6,"failed_questions":0,"provider_errors":0,"parse_errors":0,"incomplete_responses":0,"question_accuracy_ci":null}}],"baselines":{"all_false_question_accuracy":0,"all_true_question_accuracy":0},"operational_metrics":{"reported_cost_usd":"0.1194768","responses_with_cost":300,"responses_without_cost":0,"prompt_tokens":46758,"completion_tokens":91771,"reasoning_tokens":54708,"latency_median_seconds":3.981077833501331,"latency_p95_seconds":9.596745875001943,"providers":{"OpenAI":300},"format_compliant_responses":226,"format_checked_responses":300},"infrastructure_attempt_count":0},{"model":"openai/gpt-5.6-luna","provider":"openai-compatible","track":"structured-output","completed_at":"2026-10-01T17:41:34.492405Z","status":"success","protocol":{"track":"structured-output","name":"hse-multiselect-zero-shot-v10","selected_dataset_sha256":"f8d2c31d1095aed302e4a2c05ea795c9f897f061275265599fc1e5dba212c239","topics_sha256":"4897b9e828053e1c9ebde4729c8af75baffcd547d251e49a21006eab961d1fff","prompt_sha256":"8cf0f6b182fe981a596b4d48bc132286a16ebf2f25ffbb15df4e27c5a12a07c1","implementation_sha256":"9b2c30085d7db94fdb18d9ebd839bf8340992e93211fad108162f4215644a69a","selected_questions":100,"repetitions":3,"temperature":null,"max_output_tokens":16384,"timeout_seconds":180,"sampling_method":"all","sampling_seed":42,"answer_order":"fixed","reasoning_settings":{"effort":"low"},"routing_settings":{"only":["openai"],"allow_fallbacks":false,"require_parameters":true},"question_concurrency":8,"request_concurrency":8,"batch_concurrency":1},"metrics":{"question_accuracy":0.6733333333333333,"valid_response_accuracy":0.6733333333333333,"coverage":1,"total_questions":300,"correct_questions":202,"valid_questions":300,"failed_questions":0,"provider_errors":0,"parse_errors":0,"incomplete_responses":0,"question_accuracy_ci":null},"repetition_metrics":[{"question_accuracy":0.69,"valid_response_accuracy":0.69,"coverage":1,"total_questions":100,"correct_questions":69,"valid_questions":100,"failed_questions":0,"provider_errors":0,"parse_errors":0,"incomplete_responses":0,"question_accuracy_ci":{"method":"wilson","confidence_level":0.95,"lower":0.5937394428051057,"upper":0.7722030278762395}},{"question_accuracy":0.66,"valid_response_accuracy":0.66,"coverage":1,"total_questions":100,"correct_questions":66,"valid_questions":100,"failed_questions":0,"provider_errors":0,"parse_errors":0,"incomplete_responses":0,"question_accuracy_ci":{"method":"wilson","confidence_level":0.95,"lower":0.5627772885472462,"upper":0.7453847920265184}},{"question_accuracy":0.67,"valid_response_accuracy":0.67,"coverage":1,"total_questions":100,"correct_questions":67,"valid_questions":100,"failed_questions":0,"provider_errors":0,"parse_errors":0,"incomplete_responses":0,"question_accuracy_ci":{"method":"wilson","confidence_level":0.95,"lower":0.5730534379509052,"upper":0.7543687726587197}}],"topic_results":[{"topic":"Environmental protection and waste","questions":20,"metrics":{"question_accuracy":0.6666666666666666,"valid_response_accuracy":0.6666666666666666,"coverage":1,"total_questions":60,"correct_questions":40,"valid_questions":60,"failed_questions":0,"provider_errors":0,"parse_errors":0,"incomplete_responses":0,"question_accuracy_ci":null}},{"topic":"Fire safety","questions":18,"metrics":{"question_accuracy":0.6666666666666666,"valid_response_accuracy":0.6666666666666666,"coverage":1,"total_questions":54,"correct_questions":36,"valid_questions":54,"failed_questions":0,"provider_errors":0,"parse_errors":0,"incomplete_responses":0,"question_accuracy_ci":null}},{"topic":"First aid","questions":20,"metrics":{"question_accuracy":0.4,"valid_response_accuracy":0.4,"coverage":1,"total_questions":60,"correct_questions":24,"valid_questions":60,"failed_questions":0,"provider_errors":0,"parse_errors":0,"incomplete_responses":0,"question_accuracy_ci":null}},{"topic":"Industrial and facility safety","questions":20,"metrics":{"question_accuracy":0.8166666666666667,"valid_response_accuracy":0.8166666666666667,"coverage":1,"total_questions":60,"correct_questions":49,"valid_questions":60,"failed_questions":0,"provider_errors":0,"parse_errors":0,"incomplete_responses":0,"question_accuracy_ci":null}},{"topic":"Occupational safety and health","questions":20,"metrics":{"question_accuracy":0.7833333333333333,"valid_response_accuracy":0.7833333333333333,"coverage":1,"total_questions":60,"correct_questions":47,"valid_questions":60,"failed_questions":0,"provider_errors":0,"parse_errors":0,"incomplete_responses":0,"question_accuracy_ci":null}},{"topic":"Regulatory administration","questions":2,"metrics":{"question_accuracy":1,"valid_response_accuracy":1,"coverage":1,"total_questions":6,"correct_questions":6,"valid_questions":6,"failed_questions":0,"provider_errors":0,"parse_errors":0,"incomplete_responses":0,"question_accuracy_ci":null}}],"baselines":{"all_false_question_accuracy":0,"all_true_question_accuracy":0},"operational_metrics":{"reported_cost_usd":"0.0689832","responses_with_cost":300,"responses_without_cost":0,"prompt_tokens":50058,"completion_tokens":49143,"reasoning_tokens":43703,"latency_median_seconds":3.2505062710006314,"latency_p95_seconds":8.564972832995409,"providers":{"OpenAI":300},"format_compliant_responses":300,"format_checked_responses":300},"infrastructure_attempt_count":0},{"model":"openai/gpt-6-luna","provider":"openai-compatible","track":"answer-only","completed_at":"2026-10-01T19:40:56.120331Z","status":"success","protocol":{"track":"answer-only","name":"hse-multiselect-zero-shot-v10","selected_dataset_sha256":"f8d2c31d1095aed302e4a2c05ea795c9f897f061275265599fc1e5dba212c239","topics_sha256":"4897b9e828053e1c9ebde4729c8af75baffcd547d251e49a21006eab961d1fff","prompt_sha256":"d1ed6b553cb6997860d5eb7cb36e6399f091187c175b5872e568e40676ab5e68","implementation_sha256":"9b2c30085d7db94fdb18d9ebd839bf8340992e93211fad108162f4215644a69a","selected_questions":100,"repetitions":3,"temperature":null,"max_output_tokens":16384,"timeout_seconds":180,"sampling_method":"all","sampling_seed":42,"answer_order":"fixed","reasoning_settings":{"effort":"low"},"routing_settings":{"only":["openai"],"allow_fallbacks":false,"require_parameters":true},"question_concurrency":8,"request_concurrency":8,"batch_concurrency":1},"metrics":{"question_accuracy":0.7233333333333334,"valid_response_accuracy":0.7233333333333334,"coverage":1,"total_questions":300,"correct_questions":217,"valid_questions":300,"failed_questions":0,"provider_errors":0,"parse_errors":0,"incomplete_responses":0,"question_accuracy_ci":null},"repetition_metrics":[{"question_accuracy":0.74,"valid_response_accuracy":0.74,"coverage":1,"total_questions":100,"correct_questions":74,"valid_questions":100,"failed_questions":0,"provider_errors":0,"parse_errors":0,"incomplete_responses":0,"question_accuracy_ci":{"method":"wilson","confidence_level":0.95,"lower":0.6462901055081282,"upper":0.8159530153525186}},{"question_accuracy":0.69,"valid_response_accuracy":0.69,"coverage":1,"total_questions":100,"correct_questions":69,"valid_questions":100,"failed_questions":0,"provider_errors":0,"parse_errors":0,"incomplete_responses":0,"question_accuracy_ci":{"method":"wilson","confidence_level":0.95,"lower":0.5937394428051057,"upper":0.7722030278762395}},{"question_accuracy":0.74,"valid_response_accuracy":0.74,"coverage":1,"total_questions":100,"correct_questions":74,"valid_questions":100,"failed_questions":0,"provider_errors":0,"parse_errors":0,"incomplete_responses":0,"question_accuracy_ci":{"method":"wilson","confidence_level":0.95,"lower":0.6462901055081282,"upper":0.8159530153525186}}],"topic_results":[{"topic":"Environmental protection and waste","questions":20,"metrics":{"question_accuracy":0.6166666666666667,"valid_response_accuracy":0.6166666666666667,"coverage":1,"total_questions":60,"correct_questions":37,"valid_questions":60,"failed_questions":0,"provider_errors":0,"parse_errors":0,"incomplete_responses":0,"question_accuracy_ci":null}},{"topic":"Fire safety","questions":18,"metrics":{"question_accuracy":0.8148148148148148,"valid_response_accuracy":0.8148148148148148,"coverage":1,"total_questions":54,"correct_questions":44,"valid_questions":54,"failed_questions":0,"provider_errors":0,"parse_errors":0,"incomplete_responses":0,"question_accuracy_ci":null}},{"topic":"First aid","questions":20,"metrics":{"question_accuracy":0.43333333333333335,"valid_response_accuracy":0.43333333333333335,"coverage":1,"total_questions":60,"correct_questions":26,"valid_questions":60,"failed_questions":0,"provider_errors":0,"parse_errors":0,"incomplete_responses":0,"question_accuracy_ci":null}},{"topic":"Industrial and facility safety","questions":20,"metrics":{"question_accuracy":0.9333333333333333,"valid_response_accuracy":0.9333333333333333,"coverage":1,"total_questions":60,"correct_questions":56,"valid_questions":60,"failed_questions":0,"provider_errors":0,"parse_errors":0,"incomplete_responses":0,"question_accuracy_ci":null}},{"topic":"Occupational safety and health","questions":20,"metrics":{"question_accuracy":0.8,"valid_response_accuracy":0.8,"coverage":1,"total_questions":60,"correct_questions":48,"valid_questions":60,"failed_questions":0,"provider_errors":0,"parse_errors":0,"incomplete_responses":0,"question_accuracy_ci":null}},{"topic":"Regulatory administration","questions":2,"metrics":{"question_accuracy":1,"valid_response_accuracy":1,"coverage":1,"total_questions":6,"correct_questions":6,"valid_questions":6,"failed_questions":0,"provider_errors":0,"parse_errors":0,"incomplete_responses":0,"question_accuracy_ci":null}}],"baselines":{"all_false_question_accuracy":0,"all_true_question_accuracy":0},"operational_metrics":{"reported_cost_usd":"0.0318668","responses_with_cost":300,"responses_without_cost":0,"prompt_tokens":55158,"completion_tokens":52702,"reasoning_tokens":48406,"latency_median_seconds":3.323313832999702,"latency_p95_seconds":7.00626812500559,"providers":{"OpenAI":300},"format_compliant_responses":300,"format_checked_responses":300},"infrastructure_attempt_count":0},{"model":"openai/gpt-6-luna","provider":"openai-compatible","track":"reasoning-assisted","completed_at":"2026-10-01T19:43:38.005949Z","status":"success","protocol":{"track":"reasoning-assisted","name":"hse-multiselect-zero-shot-v10","selected_dataset_sha256":"f8d2c31d1095aed302e4a2c05ea795c9f897f061275265599fc1e5dba212c239","topics_sha256":"4897b9e828053e1c9ebde4729c8af75baffcd547d251e49a21006eab961d1fff","prompt_sha256":"1e82d8ac2c47f18b33cf9cda4901214138aba1621d45b4d34123be7253a8a2f6","implementation_sha256":"9b2c30085d7db94fdb18d9ebd839bf8340992e93211fad108162f4215644a69a","selected_questions":100,"repetitions":3,"temperature":null,"max_output_tokens":16384,"timeout_seconds":180,"sampling_method":"all","sampling_seed":42,"answer_order":"fixed","reasoning_settings":{"effort":"low"},"routing_settings":{"only":["openai"],"allow_fallbacks":false,"require_parameters":true},"question_concurrency":8,"request_concurrency":8,"batch_concurrency":1},"metrics":{"question_accuracy":0.75,"valid_response_accuracy":0.75,"coverage":1,"total_questions":300,"correct_questions":225,"valid_questions":300,"failed_questions":0,"provider_errors":0,"parse_errors":0,"incomplete_responses":0,"question_accuracy_ci":null},"repetition_metrics":[{"question_accuracy":0.73,"valid_response_accuracy":0.73,"coverage":1,"total_questions":100,"correct_questions":73,"valid_questions":100,"failed_questions":0,"provider_errors":0,"parse_errors":0,"incomplete_responses":0,"question_accuracy_ci":{"method":"wilson","confidence_level":0.95,"lower":0.6356788323205499,"upper":0.8073041585042365}},{"question_accuracy":0.77,"valid_response_accuracy":0.77,"coverage":1,"total_questions":100,"correct_questions":77,"valid_questions":100,"failed_questions":0,"provider_errors":0,"parse_errors":0,"incomplete_responses":0,"question_accuracy_ci":{"method":"wilson","confidence_level":0.95,"lower":0.6784561697712622,"upper":0.8415673411969654}},{"question_accuracy":0.75,"valid_response_accuracy":0.75,"coverage":1,"total_questions":100,"correct_questions":75,"valid_questions":100,"failed_questions":0,"provider_errors":0,"parse_errors":0,"incomplete_responses":0,"question_accuracy_ci":{"method":"wilson","confidence_level":0.95,"lower":0.656955364519384,"upper":0.8245478863771232}}],"topic_results":[{"topic":"Environmental protection and waste","questions":20,"metrics":{"question_accuracy":0.7,"valid_response_accuracy":0.7,"coverage":1,"total_questions":60,"correct_questions":42,"valid_questions":60,"failed_questions":0,"provider_errors":0,"parse_errors":0,"incomplete_responses":0,"question_accuracy_ci":null}},{"topic":"Fire safety","questions":18,"metrics":{"question_accuracy":0.7592592592592593,"valid_response_accuracy":0.7592592592592593,"coverage":1,"total_questions":54,"correct_questions":41,"valid_questions":54,"failed_questions":0,"provider_errors":0,"parse_errors":0,"incomplete_responses":0,"question_accuracy_ci":null}},{"topic":"First aid","questions":20,"metrics":{"question_accuracy":0.4666666666666667,"valid_response_accuracy":0.4666666666666667,"coverage":1,"total_questions":60,"correct_questions":28,"valid_questions":60,"failed_questions":0,"provider_errors":0,"parse_errors":0,"incomplete_responses":0,"question_accuracy_ci":null}},{"topic":"Industrial and facility safety","questions":20,"metrics":{"question_accuracy":0.95,"valid_response_accuracy":0.95,"coverage":1,"total_questions":60,"correct_questions":57,"valid_questions":60,"failed_questions":0,"provider_errors":0,"parse_errors":0,"incomplete_responses":0,"question_accuracy_ci":null}},{"topic":"Occupational safety and health","questions":20,"metrics":{"question_accuracy":0.85,"valid_response_accuracy":0.85,"coverage":1,"total_questions":60,"correct_questions":51,"valid_questions":60,"failed_questions":0,"provider_errors":0,"parse_errors":0,"incomplete_responses":0,"question_accuracy_ci":null}},{"topic":"Regulatory administration","questions":2,"metrics":{"question_accuracy":1,"valid_response_accuracy":1,"coverage":1,"total_questions":6,"correct_questions":6,"valid_questions":6,"failed_questions":0,"provider_errors":0,"parse_errors":0,"incomplete_responses":0,"question_accuracy_ci":null}}],"baselines":{"all_false_question_accuracy":0,"all_true_question_accuracy":0},"operational_metrics":{"reported_cost_usd":"0.0479873","responses_with_cost":300,"responses_without_cost":0,"prompt_tokens":46758,"completion_tokens":86623,"reasoning_tokens":56259,"latency_median_seconds":3.6067065834977257,"latency_p95_seconds":7.769650666996313,"providers":{"OpenAI":300},"format_compliant_responses":225,"format_checked_responses":300},"infrastructure_attempt_count":0},{"model":"openai/gpt-6-luna","provider":"openai-compatible","track":"structured-output","completed_at":"2026-10-01T19:45:59.632382Z","status":"success","protocol":{"track":"structured-output","name":"hse-multiselect-zero-shot-v10","selected_dataset_sha256":"f8d2c31d1095aed302e4a2c05ea795c9f897f061275265599fc1e5dba212c239","topics_sha256":"4897b9e828053e1c9ebde4729c8af75baffcd547d251e49a21006eab961d1fff","prompt_sha256":"8cf0f6b182fe981a596b4d48bc132286a16ebf2f25ffbb15df4e27c5a12a07c1","implementation_sha256":"9b2c30085d7db94fdb18d9ebd839bf8340992e93211fad108162f4215644a69a","selected_questions":100,"repetitions":3,"temperature":null,"max_output_tokens":16384,"timeout_seconds":180,"sampling_method":"all","sampling_seed":42,"answer_order":"fixed","reasoning_settings":{"effort":"low"},"routing_settings":{"only":["openai"],"allow_fallbacks":false,"require_parameters":true},"question_concurrency":8,"request_concurrency":8,"batch_concurrency":1},"metrics":{"question_accuracy":0.7533333333333333,"valid_response_accuracy":0.7533333333333333,"coverage":1,"total_questions":300,"correct_questions":226,"valid_questions":300,"failed_questions":0,"provider_errors":0,"parse_errors":0,"incomplete_responses":0,"question_accuracy_ci":null},"repetition_metrics":[{"question_accuracy":0.77,"valid_response_accuracy":0.77,"coverage":1,"total_questions":100,"correct_questions":77,"valid_questions":100,"failed_questions":0,"provider_errors":0,"parse_errors":0,"incomplete_responses":0,"question_accuracy_ci":{"method":"wilson","confidence_level":0.95,"lower":0.6784561697712622,"upper":0.8415673411969654}},{"question_accuracy":0.76,"valid_response_accuracy":0.76,"coverage":1,"total_questions":100,"correct_questions":76,"valid_questions":100,"failed_questions":0,"provider_errors":0,"parse_errors":0,"incomplete_responses":0,"question_accuracy_ci":{"method":"wilson","confidence_level":0.95,"lower":0.6676766365018553,"upper":0.8330867444305122}},{"question_accuracy":0.73,"valid_response_accuracy":0.73,"coverage":1,"total_questions":100,"correct_questions":73,"valid_questions":100,"failed_questions":0,"provider_errors":0,"parse_errors":0,"incomplete_responses":0,"question_accuracy_ci":{"method":"wilson","confidence_level":0.95,"lower":0.6356788323205499,"upper":0.8073041585042365}}],"topic_results":[{"topic":"Environmental protection and waste","questions":20,"metrics":{"question_accuracy":0.6833333333333333,"valid_response_accuracy":0.6833333333333333,"coverage":1,"total_questions":60,"correct_questions":41,"valid_questions":60,"failed_questions":0,"provider_errors":0,"parse_errors":0,"incomplete_responses":0,"question_accuracy_ci":null}},{"topic":"Fire safety","questions":18,"metrics":{"question_accuracy":0.8333333333333334,"valid_response_accuracy":0.8333333333333334,"coverage":1,"total_questions":54,"correct_questions":45,"valid_questions":54,"failed_questions":0,"provider_errors":0,"parse_errors":0,"incomplete_responses":0,"question_accuracy_ci":null}},{"topic":"First aid","questions":20,"metrics":{"question_accuracy":0.48333333333333334,"valid_response_accuracy":0.48333333333333334,"coverage":1,"total_questions":60,"correct_questions":29,"valid_questions":60,"failed_questions":0,"provider_errors":0,"parse_errors":0,"incomplete_responses":0,"question_accuracy_ci":null}},{"topic":"Industrial and facility safety","questions":20,"metrics":{"question_accuracy":0.95,"valid_response_accuracy":0.95,"coverage":1,"total_questions":60,"correct_questions":57,"valid_questions":60,"failed_questions":0,"provider_errors":0,"parse_errors":0,"incomplete_responses":0,"question_accuracy_ci":null}},{"topic":"Occupational safety and health","questions":20,"metrics":{"question_accuracy":0.8,"valid_response_accuracy":0.8,"coverage":1,"total_questions":60,"correct_questions":48,"valid_questions":60,"failed_questions":0,"provider_errors":0,"parse_errors":0,"incomplete_responses":0,"question_accuracy_ci":null}},{"topic":"Regulatory administration","questions":2,"metrics":{"question_accuracy":1,"valid_response_accuracy":1,"coverage":1,"total_questions":6,"correct_questions":6,"valid_questions":6,"failed_questions":0,"provider_errors":0,"parse_errors":0,"incomplete_responses":0,"question_accuracy_ci":null}}],"baselines":{"all_false_question_accuracy":0,"all_true_question_accuracy":0},"operational_metrics":{"reported_cost_usd":"0.0344093","responses_with_cost":300,"responses_without_cost":0,"prompt_tokens":50058,"completion_tokens":58807,"reasoning_tokens":53211,"latency_median_seconds":3.2153081670039683,"latency_p95_seconds":7.356098958000075,"providers":{"OpenAI":300},"format_compliant_responses":300,"format_checked_responses":300},"infrastructure_attempt_count":0},{"model":"openai/gpt-6.1-sol","provider":"openai-compatible","track":"answer-only","completed_at":"2026-10-01T18:36:23.320657Z","status":"success","protocol":{"track":"answer-only","name":"hse-multiselect-zero-shot-v10","selected_dataset_sha256":"f8d2c31d1095aed302e4a2c05ea795c9f897f061275265599fc1e5dba212c239","topics_sha256":"4897b9e828053e1c9ebde4729c8af75baffcd547d251e49a21006eab961d1fff","prompt_sha256":"d1ed6b553cb6997860d5eb7cb36e6399f091187c175b5872e568e40676ab5e68","implementation_sha256":"9b2c30085d7db94fdb18d9ebd839bf8340992e93211fad108162f4215644a69a","selected_questions":100,"repetitions":3,"temperature":null,"max_output_tokens":16384,"timeout_seconds":180,"sampling_method":"all","sampling_seed":42,"answer_order":"fixed","reasoning_settings":{"effort":"low"},"routing_settings":{"only":["openai"],"allow_fallbacks":false,"require_parameters":true},"question_concurrency":8,"request_concurrency":8,"batch_concurrency":1},"metrics":{"question_accuracy":0.7133333333333334,"valid_response_accuracy":0.7133333333333334,"coverage":1,"total_questions":300,"correct_questions":214,"valid_questions":300,"failed_questions":0,"provider_errors":0,"parse_errors":0,"incomplete_responses":0,"question_accuracy_ci":null},"repetition_metrics":[{"question_accuracy":0.71,"valid_response_accuracy":0.71,"coverage":1,"total_questions":100,"correct_questions":71,"valid_questions":100,"failed_questions":0,"provider_errors":0,"parse_errors":0,"incomplete_responses":0,"question_accuracy_ci":{"method":"wilson","confidence_level":0.95,"lower":0.6146110882442888,"upper":0.7898516425087772}},{"question_accuracy":0.71,"valid_response_accuracy":0.71,"coverage":1,"total_questions":100,"correct_questions":71,"valid_questions":100,"failed_questions":0,"provider_errors":0,"parse_errors":0,"incomplete_responses":0,"question_accuracy_ci":{"method":"wilson","confidence_level":0.95,"lower":0.6146110882442888,"upper":0.7898516425087772}},{"question_accuracy":0.72,"valid_response_accuracy":0.72,"coverage":1,"total_questions":100,"correct_questions":72,"valid_questions":100,"failed_questions":0,"provider_errors":0,"parse_errors":0,"incomplete_responses":0,"question_accuracy_ci":{"method":"wilson","confidence_level":0.95,"lower":0.6251197129007884,"upper":0.7986031478881379}}],"topic_results":[{"topic":"Environmental protection and waste","questions":20,"metrics":{"question_accuracy":0.6666666666666666,"valid_response_accuracy":0.6666666666666666,"coverage":1,"total_questions":60,"correct_questions":40,"valid_questions":60,"failed_questions":0,"provider_errors":0,"parse_errors":0,"incomplete_responses":0,"question_accuracy_ci":null}},{"topic":"Fire safety","questions":18,"metrics":{"question_accuracy":0.8703703703703703,"valid_response_accuracy":0.8703703703703703,"coverage":1,"total_questions":54,"correct_questions":47,"valid_questions":54,"failed_questions":0,"provider_errors":0,"parse_errors":0,"incomplete_responses":0,"question_accuracy_ci":null}},{"topic":"First aid","questions":20,"metrics":{"question_accuracy":0.4,"valid_response_accuracy":0.4,"coverage":1,"total_questions":60,"correct_questions":24,"valid_questions":60,"failed_questions":0,"provider_errors":0,"parse_errors":0,"incomplete_responses":0,"question_accuracy_ci":null}},{"topic":"Industrial and facility safety","questions":20,"metrics":{"question_accuracy":0.9,"valid_response_accuracy":0.9,"coverage":1,"total_questions":60,"correct_questions":54,"valid_questions":60,"failed_questions":0,"provider_errors":0,"parse_errors":0,"incomplete_responses":0,"question_accuracy_ci":null}},{"topic":"Occupational safety and health","questions":20,"metrics":{"question_accuracy":0.75,"valid_response_accuracy":0.75,"coverage":1,"total_questions":60,"correct_questions":45,"valid_questions":60,"failed_questions":0,"provider_errors":0,"parse_errors":0,"incomplete_responses":0,"question_accuracy_ci":null}},{"topic":"Regulatory administration","questions":2,"metrics":{"question_accuracy":0.6666666666666666,"valid_response_accuracy":0.6666666666666666,"coverage":1,"total_questions":6,"correct_questions":4,"valid_questions":6,"failed_questions":0,"provider_errors":0,"parse_errors":0,"incomplete_responses":0,"question_accuracy_ci":null}}],"baselines":{"all_false_question_accuracy":0,"all_true_question_accuracy":0},"operational_metrics":{"reported_cost_usd":"0.315046","responses_with_cost":300,"responses_without_cost":0,"prompt_tokens":55158,"completion_tokens":20473,"reasoning_tokens":16295,"latency_median_seconds":3.9302748540030734,"latency_p95_seconds":8.27550445900124,"providers":{"OpenAI":300},"format_compliant_responses":300,"format_checked_responses":300},"infrastructure_attempt_count":0},{"model":"openai/gpt-6.1-sol","provider":"openai-compatible","track":"reasoning-assisted","completed_at":"2026-10-01T18:40:28.778146Z","status":"success","protocol":{"track":"reasoning-assisted","name":"hse-multiselect-zero-shot-v10","selected_dataset_sha256":"f8d2c31d1095aed302e4a2c05ea795c9f897f061275265599fc1e5dba212c239","topics_sha256":"4897b9e828053e1c9ebde4729c8af75baffcd547d251e49a21006eab961d1fff","prompt_sha256":"1e82d8ac2c47f18b33cf9cda4901214138aba1621d45b4d34123be7253a8a2f6","implementation_sha256":"9b2c30085d7db94fdb18d9ebd839bf8340992e93211fad108162f4215644a69a","selected_questions":100,"repetitions":3,"temperature":null,"max_output_tokens":16384,"timeout_seconds":180,"sampling_method":"all","sampling_seed":42,"answer_order":"fixed","reasoning_settings":{"effort":"low"},"routing_settings":{"only":["openai"],"allow_fallbacks":false,"require_parameters":true},"question_concurrency":8,"request_concurrency":8,"batch_concurrency":1},"metrics":{"question_accuracy":0.77,"valid_response_accuracy":0.77,"coverage":1,"total_questions":300,"correct_questions":231,"valid_questions":300,"failed_questions":0,"provider_errors":0,"parse_errors":0,"incomplete_responses":0,"question_accuracy_ci":null},"repetition_metrics":[{"question_accuracy":0.78,"valid_response_accuracy":0.78,"coverage":1,"total_questions":100,"correct_questions":78,"valid_questions":100,"failed_questions":0,"provider_errors":0,"parse_errors":0,"incomplete_responses":0,"question_accuracy_ci":{"method":"wilson","confidence_level":0.95,"lower":0.6892964648501421,"upper":0.8499871761539459}},{"question_accuracy":0.77,"valid_response_accuracy":0.77,"coverage":1,"total_questions":100,"correct_questions":77,"valid_questions":100,"failed_questions":0,"provider_errors":0,"parse_errors":0,"incomplete_responses":0,"question_accuracy_ci":{"method":"wilson","confidence_level":0.95,"lower":0.6784561697712622,"upper":0.8415673411969654}},{"question_accuracy":0.76,"valid_response_accuracy":0.76,"coverage":1,"total_questions":100,"correct_questions":76,"valid_questions":100,"failed_questions":0,"provider_errors":0,"parse_errors":0,"incomplete_responses":0,"question_accuracy_ci":{"method":"wilson","confidence_level":0.95,"lower":0.6676766365018553,"upper":0.8330867444305122}}],"topic_results":[{"topic":"Environmental protection and waste","questions":20,"metrics":{"question_accuracy":0.7,"valid_response_accuracy":0.7,"coverage":1,"total_questions":60,"correct_questions":42,"valid_questions":60,"failed_questions":0,"provider_errors":0,"parse_errors":0,"incomplete_responses":0,"question_accuracy_ci":null}},{"topic":"Fire safety","questions":18,"metrics":{"question_accuracy":0.8703703703703703,"valid_response_accuracy":0.8703703703703703,"coverage":1,"total_questions":54,"correct_questions":47,"valid_questions":54,"failed_questions":0,"provider_errors":0,"parse_errors":0,"incomplete_responses":0,"question_accuracy_ci":null}},{"topic":"First aid","questions":20,"metrics":{"question_accuracy":0.5166666666666667,"valid_response_accuracy":0.5166666666666667,"coverage":1,"total_questions":60,"correct_questions":31,"valid_questions":60,"failed_questions":0,"provider_errors":0,"parse_errors":0,"incomplete_responses":0,"question_accuracy_ci":null}},{"topic":"Industrial and facility safety","questions":20,"metrics":{"question_accuracy":0.9166666666666666,"valid_response_accuracy":0.9166666666666666,"coverage":1,"total_questions":60,"correct_questions":55,"valid_questions":60,"failed_questions":0,"provider_errors":0,"parse_errors":0,"incomplete_responses":0,"question_accuracy_ci":null}},{"topic":"Occupational safety and health","questions":20,"metrics":{"question_accuracy":0.85,"valid_response_accuracy":0.85,"coverage":1,"total_questions":60,"correct_questions":51,"valid_questions":60,"failed_questions":0,"provider_errors":0,"parse_errors":0,"incomplete_responses":0,"question_accuracy_ci":null}},{"topic":"Regulatory administration","questions":2,"metrics":{"question_accuracy":0.8333333333333334,"valid_response_accuracy":0.8333333333333334,"coverage":1,"total_questions":6,"correct_questions":5,"valid_questions":6,"failed_questions":0,"provider_errors":0,"parse_errors":0,"incomplete_responses":0,"question_accuracy_ci":null}}],"baselines":{"all_false_question_accuracy":0,"all_true_question_accuracy":0},"operational_metrics":{"reported_cost_usd":"0.627246","responses_with_cost":300,"responses_without_cost":0,"prompt_tokens":46758,"completion_tokens":53373,"reasoning_tokens":16975,"latency_median_seconds":5.738022042005468,"latency_p95_seconds":11.147088833000453,"providers":{"OpenAI":300},"format_compliant_responses":300,"format_checked_responses":300},"infrastructure_attempt_count":0},{"model":"openai/gpt-6.1-sol","provider":"openai-compatible","track":"structured-output","completed_at":"2026-10-01T18:43:29.697492Z","status":"success","protocol":{"track":"structured-output","name":"hse-multiselect-zero-shot-v10","selected_dataset_sha256":"f8d2c31d1095aed302e4a2c05ea795c9f897f061275265599fc1e5dba212c239","topics_sha256":"4897b9e828053e1c9ebde4729c8af75baffcd547d251e49a21006eab961d1fff","prompt_sha256":"8cf0f6b182fe981a596b4d48bc132286a16ebf2f25ffbb15df4e27c5a12a07c1","implementation_sha256":"9b2c30085d7db94fdb18d9ebd839bf8340992e93211fad108162f4215644a69a","selected_questions":100,"repetitions":3,"temperature":null,"max_output_tokens":16384,"timeout_seconds":180,"sampling_method":"all","sampling_seed":42,"answer_order":"fixed","reasoning_settings":{"effort":"low"},"routing_settings":{"only":["openai"],"allow_fallbacks":false,"require_parameters":true},"question_concurrency":8,"request_concurrency":8,"batch_concurrency":1},"metrics":{"question_accuracy":0.7466666666666667,"valid_response_accuracy":0.7466666666666667,"coverage":1,"total_questions":300,"correct_questions":224,"valid_questions":300,"failed_questions":0,"provider_errors":0,"parse_errors":0,"incomplete_responses":0,"question_accuracy_ci":null},"repetition_metrics":[{"question_accuracy":0.76,"valid_response_accuracy":0.76,"coverage":1,"total_questions":100,"correct_questions":76,"valid_questions":100,"failed_questions":0,"provider_errors":0,"parse_errors":0,"incomplete_responses":0,"question_accuracy_ci":{"method":"wilson","confidence_level":0.95,"lower":0.6676766365018553,"upper":0.8330867444305122}},{"question_accuracy":0.74,"valid_response_accuracy":0.74,"coverage":1,"total_questions":100,"correct_questions":74,"valid_questions":100,"failed_questions":0,"provider_errors":0,"parse_errors":0,"incomplete_responses":0,"question_accuracy_ci":{"method":"wilson","confidence_level":0.95,"lower":0.6462901055081282,"upper":0.8159530153525186}},{"question_accuracy":0.74,"valid_response_accuracy":0.74,"coverage":1,"total_questions":100,"correct_questions":74,"valid_questions":100,"failed_questions":0,"provider_errors":0,"parse_errors":0,"incomplete_responses":0,"question_accuracy_ci":{"method":"wilson","confidence_level":0.95,"lower":0.6462901055081282,"upper":0.8159530153525186}}],"topic_results":[{"topic":"Environmental protection and waste","questions":20,"metrics":{"question_accuracy":0.7166666666666667,"valid_response_accuracy":0.7166666666666667,"coverage":1,"total_questions":60,"correct_questions":43,"valid_questions":60,"failed_questions":0,"provider_errors":0,"parse_errors":0,"incomplete_responses":0,"question_accuracy_ci":null}},{"topic":"Fire safety","questions":18,"metrics":{"question_accuracy":0.8888888888888888,"valid_response_accuracy":0.8888888888888888,"coverage":1,"total_questions":54,"correct_questions":48,"valid_questions":54,"failed_questions":0,"provider_errors":0,"parse_errors":0,"incomplete_responses":0,"question_accuracy_ci":null}},{"topic":"First aid","questions":20,"metrics":{"question_accuracy":0.43333333333333335,"valid_response_accuracy":0.43333333333333335,"coverage":1,"total_questions":60,"correct_questions":26,"valid_questions":60,"failed_questions":0,"provider_errors":0,"parse_errors":0,"incomplete_responses":0,"question_accuracy_ci":null}},{"topic":"Industrial and facility safety","questions":20,"metrics":{"question_accuracy":0.8833333333333333,"valid_response_accuracy":0.8833333333333333,"coverage":1,"total_questions":60,"correct_questions":53,"valid_questions":60,"failed_questions":0,"provider_errors":0,"parse_errors":0,"incomplete_responses":0,"question_accuracy_ci":null}},{"topic":"Occupational safety and health","questions":20,"metrics":{"question_accuracy":0.8166666666666667,"valid_response_accuracy":0.8166666666666667,"coverage":1,"total_questions":60,"correct_questions":49,"valid_questions":60,"failed_questions":0,"provider_errors":0,"parse_errors":0,"incomplete_responses":0,"question_accuracy_ci":null}},{"topic":"Regulatory administration","questions":2,"metrics":{"question_accuracy":0.8333333333333334,"valid_response_accuracy":0.8333333333333334,"coverage":1,"total_questions":6,"correct_questions":5,"valid_questions":6,"failed_questions":0,"provider_errors":0,"parse_errors":0,"incomplete_responses":0,"question_accuracy_ci":null}}],"baselines":{"all_false_question_accuracy":0,"all_true_question_accuracy":0},"operational_metrics":{"reported_cost_usd":"0.302386","responses_with_cost":300,"responses_without_cost":0,"prompt_tokens":50058,"completion_tokens":20227,"reasoning_tokens":14803,"latency_median_seconds":4.362988353994297,"latency_p95_seconds":9.303257208004652,"providers":{"OpenAI":300},"format_compliant_responses":300,"format_checked_responses":300},"infrastructure_attempt_count":0},{"model":"moonshotai/kimi-k3","provider":"openai-compatible","track":"answer-only","completed_at":"2026-10-01T19:55:22.591538Z","status":"success","protocol":{"track":"answer-only","name":"hse-multiselect-zero-shot-v10","selected_dataset_sha256":"f8d2c31d1095aed302e4a2c05ea795c9f897f061275265599fc1e5dba212c239","topics_sha256":"4897b9e828053e1c9ebde4729c8af75baffcd547d251e49a21006eab961d1fff","prompt_sha256":"d1ed6b553cb6997860d5eb7cb36e6399f091187c175b5872e568e40676ab5e68","implementation_sha256":"9b2c30085d7db94fdb18d9ebd839bf8340992e93211fad108162f4215644a69a","selected_questions":100,"repetitions":3,"temperature":null,"max_output_tokens":16384,"timeout_seconds":180,"sampling_method":"all","sampling_seed":42,"answer_order":"fixed","reasoning_settings":{"effort":"low"},"routing_settings":{"only":["moonshotai/mxfp4"],"allow_fallbacks":false,"require_parameters":true},"question_concurrency":8,"request_concurrency":8,"batch_concurrency":1},"metrics":{"question_accuracy":0.75,"valid_response_accuracy":0.75,"coverage":1,"total_questions":300,"correct_questions":225,"valid_questions":300,"failed_questions":0,"provider_errors":0,"parse_errors":0,"incomplete_responses":0,"question_accuracy_ci":null},"repetition_metrics":[{"question_accuracy":0.72,"valid_response_accuracy":0.72,"coverage":1,"total_questions":100,"correct_questions":72,"valid_questions":100,"failed_questions":0,"provider_errors":0,"parse_errors":0,"incomplete_responses":0,"question_accuracy_ci":{"method":"wilson","confidence_level":0.95,"lower":0.6251197129007884,"upper":0.7986031478881379}},{"question_accuracy":0.73,"valid_response_accuracy":0.73,"coverage":1,"total_questions":100,"correct_questions":73,"valid_questions":100,"failed_questions":0,"provider_errors":0,"parse_errors":0,"incomplete_responses":0,"question_accuracy_ci":{"method":"wilson","confidence_level":0.95,"lower":0.6356788323205499,"upper":0.8073041585042365}},{"question_accuracy":0.8,"valid_response_accuracy":0.8,"coverage":1,"total_questions":100,"correct_questions":80,"valid_questions":100,"failed_questions":0,"provider_errors":0,"parse_errors":0,"incomplete_responses":0,"question_accuracy_ci":{"method":"wilson","confidence_level":0.95,"lower":0.7111708344068411,"upper":0.8666330666689676}}],"topic_results":[{"topic":"Environmental protection and waste","questions":20,"metrics":{"question_accuracy":0.6166666666666667,"valid_response_accuracy":0.6166666666666667,"coverage":1,"total_questions":60,"correct_questions":37,"valid_questions":60,"failed_questions":0,"provider_errors":0,"parse_errors":0,"incomplete_responses":0,"question_accuracy_ci":null}},{"topic":"Fire safety","questions":18,"metrics":{"question_accuracy":0.7777777777777778,"valid_response_accuracy":0.7777777777777778,"coverage":1,"total_questions":54,"correct_questions":42,"valid_questions":54,"failed_questions":0,"provider_errors":0,"parse_errors":0,"incomplete_responses":0,"question_accuracy_ci":null}},{"topic":"First aid","questions":20,"metrics":{"question_accuracy":0.5666666666666667,"valid_response_accuracy":0.5666666666666667,"coverage":1,"total_questions":60,"correct_questions":34,"valid_questions":60,"failed_questions":0,"provider_errors":0,"parse_errors":0,"incomplete_responses":0,"question_accuracy_ci":null}},{"topic":"Industrial and facility safety","questions":20,"metrics":{"question_accuracy":0.9,"valid_response_accuracy":0.9,"coverage":1,"total_questions":60,"correct_questions":54,"valid_questions":60,"failed_questions":0,"provider_errors":0,"parse_errors":0,"incomplete_responses":0,"question_accuracy_ci":null}},{"topic":"Occupational safety and health","questions":20,"metrics":{"question_accuracy":0.9,"valid_response_accuracy":0.9,"coverage":1,"total_questions":60,"correct_questions":54,"valid_questions":60,"failed_questions":0,"provider_errors":0,"parse_errors":0,"incomplete_responses":0,"question_accuracy_ci":null}},{"topic":"Regulatory administration","questions":2,"metrics":{"question_accuracy":0.6666666666666666,"valid_response_accuracy":0.6666666666666666,"coverage":1,"total_questions":6,"correct_questions":4,"valid_questions":6,"failed_questions":0,"provider_errors":0,"parse_errors":0,"incomplete_responses":0,"question_accuracy_ci":null}}],"baselines":{"all_false_question_accuracy":0,"all_true_question_accuracy":0},"operational_metrics":{"reported_cost_usd":"1.2936741","responses_with_cost":300,"responses_without_cost":0,"prompt_tokens":81324,"completion_tokens":72973,"reasoning_tokens":66043,"latency_median_seconds":7.817861000003177,"latency_p95_seconds":24.046437124998192,"providers":{"Moonshot AI":300},"format_compliant_responses":300,"format_checked_responses":300},"infrastructure_attempt_count":4},{"model":"moonshotai/kimi-k3","provider":"openai-compatible","track":"reasoning-assisted","completed_at":"2026-10-01T20:04:35.526195Z","status":"success","protocol":{"track":"reasoning-assisted","name":"hse-multiselect-zero-shot-v10","selected_dataset_sha256":"f8d2c31d1095aed302e4a2c05ea795c9f897f061275265599fc1e5dba212c239","topics_sha256":"4897b9e828053e1c9ebde4729c8af75baffcd547d251e49a21006eab961d1fff","prompt_sha256":"1e82d8ac2c47f18b33cf9cda4901214138aba1621d45b4d34123be7253a8a2f6","implementation_sha256":"9b2c30085d7db94fdb18d9ebd839bf8340992e93211fad108162f4215644a69a","selected_questions":100,"repetitions":3,"temperature":null,"max_output_tokens":16384,"timeout_seconds":180,"sampling_method":"all","sampling_seed":42,"answer_order":"fixed","reasoning_settings":{"effort":"low"},"routing_settings":{"only":["moonshotai/mxfp4"],"allow_fallbacks":false,"require_parameters":true},"question_concurrency":8,"request_concurrency":8,"batch_concurrency":1},"metrics":{"question_accuracy":0.7433333333333333,"valid_response_accuracy":0.7433333333333333,"coverage":1,"total_questions":300,"correct_questions":223,"valid_questions":300,"failed_questions":0,"provider_errors":0,"parse_errors":0,"incomplete_responses":0,"question_accuracy_ci":null},"repetition_metrics":[{"question_accuracy":0.75,"valid_response_accuracy":0.75,"coverage":1,"total_questions":100,"correct_questions":75,"valid_questions":100,"failed_questions":0,"provider_errors":0,"parse_errors":0,"incomplete_responses":0,"question_accuracy_ci":{"method":"wilson","confidence_level":0.95,"lower":0.656955364519384,"upper":0.8245478863771232}},{"question_accuracy":0.72,"valid_response_accuracy":0.72,"coverage":1,"total_questions":100,"correct_questions":72,"valid_questions":100,"failed_questions":0,"provider_errors":0,"parse_errors":0,"incomplete_responses":0,"question_accuracy_ci":{"method":"wilson","confidence_level":0.95,"lower":0.6251197129007884,"upper":0.7986031478881379}},{"question_accuracy":0.76,"valid_response_accuracy":0.76,"coverage":1,"total_questions":100,"correct_questions":76,"valid_questions":100,"failed_questions":0,"provider_errors":0,"parse_errors":0,"incomplete_responses":0,"question_accuracy_ci":{"method":"wilson","confidence_level":0.95,"lower":0.6676766365018553,"upper":0.8330867444305122}}],"topic_results":[{"topic":"Environmental protection and waste","questions":20,"metrics":{"question_accuracy":0.65,"valid_response_accuracy":0.65,"coverage":1,"total_questions":60,"correct_questions":39,"valid_questions":60,"failed_questions":0,"provider_errors":0,"parse_errors":0,"incomplete_responses":0,"question_accuracy_ci":null}},{"topic":"Fire safety","questions":18,"metrics":{"question_accuracy":0.8148148148148148,"valid_response_accuracy":0.8148148148148148,"coverage":1,"total_questions":54,"correct_questions":44,"valid_questions":54,"failed_questions":0,"provider_errors":0,"parse_errors":0,"incomplete_responses":0,"question_accuracy_ci":null}},{"topic":"First aid","questions":20,"metrics":{"question_accuracy":0.5166666666666667,"valid_response_accuracy":0.5166666666666667,"coverage":1,"total_questions":60,"correct_questions":31,"valid_questions":60,"failed_questions":0,"provider_errors":0,"parse_errors":0,"incomplete_responses":0,"question_accuracy_ci":null}},{"topic":"Industrial and facility safety","questions":20,"metrics":{"question_accuracy":0.9,"valid_response_accuracy":0.9,"coverage":1,"total_questions":60,"correct_questions":54,"valid_questions":60,"failed_questions":0,"provider_errors":0,"parse_errors":0,"incomplete_responses":0,"question_accuracy_ci":null}},{"topic":"Occupational safety and health","questions":20,"metrics":{"question_accuracy":0.85,"valid_response_accuracy":0.85,"coverage":1,"total_questions":60,"correct_questions":51,"valid_questions":60,"failed_questions":0,"provider_errors":0,"parse_errors":0,"incomplete_responses":0,"question_accuracy_ci":null}},{"topic":"Regulatory administration","questions":2,"metrics":{"question_accuracy":0.6666666666666666,"valid_response_accuracy":0.6666666666666666,"coverage":1,"total_questions":6,"correct_questions":4,"valid_questions":6,"failed_questions":0,"provider_errors":0,"parse_errors":0,"incomplete_responses":0,"question_accuracy_ci":null}}],"baselines":{"all_false_question_accuracy":0,"all_true_question_accuracy":0},"operational_metrics":{"reported_cost_usd":"2.0228472","responses_with_cost":300,"responses_without_cost":0,"prompt_tokens":71430,"completion_tokens":122688,"reasoning_tokens":82884,"latency_median_seconds":13.028300437501457,"latency_p95_seconds":25.132296957999642,"providers":{"Moonshot AI":300},"format_compliant_responses":3,"format_checked_responses":300},"infrastructure_attempt_count":0},{"model":"moonshotai/kimi-k3","provider":"openai-compatible","track":"structured-output","completed_at":"2026-10-01T20:11:52.363826Z","status":"success","protocol":{"track":"structured-output","name":"hse-multiselect-zero-shot-v10","selected_dataset_sha256":"f8d2c31d1095aed302e4a2c05ea795c9f897f061275265599fc1e5dba212c239","topics_sha256":"4897b9e828053e1c9ebde4729c8af75baffcd547d251e49a21006eab961d1fff","prompt_sha256":"8cf0f6b182fe981a596b4d48bc132286a16ebf2f25ffbb15df4e27c5a12a07c1","implementation_sha256":"9b2c30085d7db94fdb18d9ebd839bf8340992e93211fad108162f4215644a69a","selected_questions":100,"repetitions":3,"temperature":null,"max_output_tokens":16384,"timeout_seconds":180,"sampling_method":"all","sampling_seed":42,"answer_order":"fixed","reasoning_settings":{"effort":"low"},"routing_settings":{"only":["moonshotai/mxfp4"],"allow_fallbacks":false,"require_parameters":true},"question_concurrency":8,"request_concurrency":8,"batch_concurrency":1},"metrics":{"question_accuracy":0.72,"valid_response_accuracy":0.72,"coverage":1,"total_questions":300,"correct_questions":216,"valid_questions":300,"failed_questions":0,"provider_errors":0,"parse_errors":0,"incomplete_responses":0,"question_accuracy_ci":null},"repetition_metrics":[{"question_accuracy":0.75,"valid_response_accuracy":0.75,"coverage":1,"total_questions":100,"correct_questions":75,"valid_questions":100,"failed_questions":0,"provider_errors":0,"parse_errors":0,"incomplete_responses":0,"question_accuracy_ci":{"method":"wilson","confidence_level":0.95,"lower":0.656955364519384,"upper":0.8245478863771232}},{"question_accuracy":0.69,"valid_response_accuracy":0.69,"coverage":1,"total_questions":100,"correct_questions":69,"valid_questions":100,"failed_questions":0,"provider_errors":0,"parse_errors":0,"incomplete_responses":0,"question_accuracy_ci":{"method":"wilson","confidence_level":0.95,"lower":0.5937394428051057,"upper":0.7722030278762395}},{"question_accuracy":0.72,"valid_response_accuracy":0.72,"coverage":1,"total_questions":100,"correct_questions":72,"valid_questions":100,"failed_questions":0,"provider_errors":0,"parse_errors":0,"incomplete_responses":0,"question_accuracy_ci":{"method":"wilson","confidence_level":0.95,"lower":0.6251197129007884,"upper":0.7986031478881379}}],"topic_results":[{"topic":"Environmental protection and waste","questions":20,"metrics":{"question_accuracy":0.6333333333333333,"valid_response_accuracy":0.6333333333333333,"coverage":1,"total_questions":60,"correct_questions":38,"valid_questions":60,"failed_questions":0,"provider_errors":0,"parse_errors":0,"incomplete_responses":0,"question_accuracy_ci":null}},{"topic":"Fire safety","questions":18,"metrics":{"question_accuracy":0.8333333333333334,"valid_response_accuracy":0.8333333333333334,"coverage":1,"total_questions":54,"correct_questions":45,"valid_questions":54,"failed_questions":0,"provider_errors":0,"parse_errors":0,"incomplete_responses":0,"question_accuracy_ci":null}},{"topic":"First aid","questions":20,"metrics":{"question_accuracy":0.5,"valid_response_accuracy":0.5,"coverage":1,"total_questions":60,"correct_questions":30,"valid_questions":60,"failed_questions":0,"provider_errors":0,"parse_errors":0,"incomplete_responses":0,"question_accuracy_ci":null}},{"topic":"Industrial and facility safety","questions":20,"metrics":{"question_accuracy":0.7833333333333333,"valid_response_accuracy":0.7833333333333333,"coverage":1,"total_questions":60,"correct_questions":47,"valid_questions":60,"failed_questions":0,"provider_errors":0,"parse_errors":0,"incomplete_responses":0,"question_accuracy_ci":null}},{"topic":"Occupational safety and health","questions":20,"metrics":{"question_accuracy":0.85,"valid_response_accuracy":0.85,"coverage":1,"total_questions":60,"correct_questions":51,"valid_questions":60,"failed_questions":0,"provider_errors":0,"parse_errors":0,"incomplete_responses":0,"question_accuracy_ci":null}},{"topic":"Regulatory administration","questions":2,"metrics":{"question_accuracy":0.8333333333333334,"valid_response_accuracy":0.8333333333333334,"coverage":1,"total_questions":6,"correct_questions":5,"valid_questions":6,"failed_questions":0,"provider_errors":0,"parse_errors":0,"incomplete_responses":0,"question_accuracy_ci":null}}],"baselines":{"all_false_question_accuracy":0,"all_true_question_accuracy":0},"operational_metrics":{"reported_cost_usd":"1.4149200","responses_with_cost":300,"responses_without_cost":0,"prompt_tokens":94830,"completion_tokens":84578,"reasoning_tokens":77780,"latency_median_seconds":8.780905749496014,"latency_p95_seconds":27.625357500000973,"providers":{"Moonshot AI":300},"format_compliant_responses":300,"format_checked_responses":300},"infrastructure_attempt_count":0},{"model":"mistralai/mistral-small-2603","provider":"openai-compatible","track":"answer-only","completed_at":"2026-10-01T19:17:58.142839Z","status":"success","protocol":{"track":"answer-only","name":"hse-multiselect-zero-shot-v10","selected_dataset_sha256":"f8d2c31d1095aed302e4a2c05ea795c9f897f061275265599fc1e5dba212c239","topics_sha256":"4897b9e828053e1c9ebde4729c8af75baffcd547d251e49a21006eab961d1fff","prompt_sha256":"d1ed6b553cb6997860d5eb7cb36e6399f091187c175b5872e568e40676ab5e68","implementation_sha256":"9b2c30085d7db94fdb18d9ebd839bf8340992e93211fad108162f4215644a69a","selected_questions":100,"repetitions":3,"temperature":null,"max_output_tokens":16384,"timeout_seconds":180,"sampling_method":"all","sampling_seed":42,"answer_order":"fixed","reasoning_settings":{"effort":"low"},"routing_settings":{"only":["mistral"],"allow_fallbacks":false,"require_parameters":true},"question_concurrency":8,"request_concurrency":8,"batch_concurrency":1},"metrics":{"question_accuracy":0.32666666666666666,"valid_response_accuracy":0.32666666666666666,"coverage":1,"total_questions":300,"correct_questions":98,"valid_questions":300,"failed_questions":0,"provider_errors":0,"parse_errors":0,"incomplete_responses":0,"question_accuracy_ci":null},"repetition_metrics":[{"question_accuracy":0.28,"valid_response_accuracy":0.28,"coverage":1,"total_questions":100,"correct_questions":28,"valid_questions":100,"failed_questions":0,"provider_errors":0,"parse_errors":0,"incomplete_responses":0,"question_accuracy_ci":{"method":"wilson","confidence_level":0.95,"lower":0.20139685211186212,"upper":0.3748802870992116}},{"question_accuracy":0.32,"valid_response_accuracy":0.32,"coverage":1,"total_questions":100,"correct_questions":32,"valid_questions":100,"failed_questions":0,"provider_errors":0,"parse_errors":0,"incomplete_responses":0,"question_accuracy_ci":{"method":"wilson","confidence_level":0.95,"lower":0.23669147324999093,"upper":0.41662618610452384}},{"question_accuracy":0.38,"valid_response_accuracy":0.38,"coverage":1,"total_questions":100,"correct_questions":38,"valid_questions":100,"failed_questions":0,"provider_errors":0,"parse_errors":0,"incomplete_responses":0,"question_accuracy_ci":{"method":"wilson","confidence_level":0.95,"lower":0.29097599252478734,"upper":0.47790244704488916}}],"topic_results":[{"topic":"Environmental protection and waste","questions":20,"metrics":{"question_accuracy":0.3,"valid_response_accuracy":0.3,"coverage":1,"total_questions":60,"correct_questions":18,"valid_questions":60,"failed_questions":0,"provider_errors":0,"parse_errors":0,"incomplete_responses":0,"question_accuracy_ci":null}},{"topic":"Fire safety","questions":18,"metrics":{"question_accuracy":0.3888888888888889,"valid_response_accuracy":0.3888888888888889,"coverage":1,"total_questions":54,"correct_questions":21,"valid_questions":54,"failed_questions":0,"provider_errors":0,"parse_errors":0,"incomplete_responses":0,"question_accuracy_ci":null}},{"topic":"First aid","questions":20,"metrics":{"question_accuracy":0.18333333333333332,"valid_response_accuracy":0.18333333333333332,"coverage":1,"total_questions":60,"correct_questions":11,"valid_questions":60,"failed_questions":0,"provider_errors":0,"parse_errors":0,"incomplete_responses":0,"question_accuracy_ci":null}},{"topic":"Industrial and facility safety","questions":20,"metrics":{"question_accuracy":0.45,"valid_response_accuracy":0.45,"coverage":1,"total_questions":60,"correct_questions":27,"valid_questions":60,"failed_questions":0,"provider_errors":0,"parse_errors":0,"incomplete_responses":0,"question_accuracy_ci":null}},{"topic":"Occupational safety and health","questions":20,"metrics":{"question_accuracy":0.3333333333333333,"valid_response_accuracy":0.3333333333333333,"coverage":1,"total_questions":60,"correct_questions":20,"valid_questions":60,"failed_questions":0,"provider_errors":0,"parse_errors":0,"incomplete_responses":0,"question_accuracy_ci":null}},{"topic":"Regulatory administration","questions":2,"metrics":{"question_accuracy":0.16666666666666666,"valid_response_accuracy":0.16666666666666666,"coverage":1,"total_questions":6,"correct_questions":1,"valid_questions":6,"failed_questions":0,"provider_errors":0,"parse_errors":0,"incomplete_responses":0,"question_accuracy_ci":null}}],"baselines":{"all_false_question_accuracy":0,"all_true_question_accuracy":0},"operational_metrics":{"reported_cost_usd":"0.09760314","responses_with_cost":300,"responses_without_cost":0,"prompt_tokens":58470,"completion_tokens":156050,"reasoning_tokens":190172,"latency_median_seconds":3.2165314165031305,"latency_p95_seconds":7.260401332998299,"providers":{"Mistral":300},"format_compliant_responses":300,"format_checked_responses":300},"infrastructure_attempt_count":0},{"model":"mistralai/mistral-small-2603","provider":"openai-compatible","track":"reasoning-assisted","completed_at":"2026-10-01T19:34:03.523791Z","status":"partial","protocol":{"track":"reasoning-assisted","name":"hse-multiselect-zero-shot-v10","selected_dataset_sha256":"f8d2c31d1095aed302e4a2c05ea795c9f897f061275265599fc1e5dba212c239","topics_sha256":"4897b9e828053e1c9ebde4729c8af75baffcd547d251e49a21006eab961d1fff","prompt_sha256":"1e82d8ac2c47f18b33cf9cda4901214138aba1621d45b4d34123be7253a8a2f6","implementation_sha256":"9b2c30085d7db94fdb18d9ebd839bf8340992e93211fad108162f4215644a69a","selected_questions":100,"repetitions":3,"temperature":null,"max_output_tokens":16384,"timeout_seconds":180,"sampling_method":"all","sampling_seed":42,"answer_order":"fixed","reasoning_settings":{"effort":"low"},"routing_settings":{"only":["mistral"],"allow_fallbacks":false,"require_parameters":true},"question_concurrency":8,"request_concurrency":8,"batch_concurrency":1},"metrics":{"question_accuracy":0.45,"valid_response_accuracy":0.4623287671232877,"coverage":0.9733333333333334,"total_questions":300,"correct_questions":135,"valid_questions":292,"failed_questions":8,"provider_errors":0,"parse_errors":8,"incomplete_responses":0,"question_accuracy_ci":null},"repetition_metrics":[{"question_accuracy":0.43,"valid_response_accuracy":0.45263157894736844,"coverage":0.95,"total_questions":100,"correct_questions":43,"valid_questions":95,"failed_questions":5,"provider_errors":0,"parse_errors":5,"incomplete_responses":0,"question_accuracy_ci":{"method":"wilson","confidence_level":0.95,"lower":0.33733298524110106,"upper":0.5278461045078768}},{"question_accuracy":0.46,"valid_response_accuracy":0.46938775510204084,"coverage":0.98,"total_questions":100,"correct_questions":46,"valid_questions":98,"failed_questions":2,"provider_errors":0,"parse_errors":2,"incomplete_responses":0,"question_accuracy_ci":{"method":"wilson","confidence_level":0.95,"lower":0.36560808309024106,"upper":0.5573513967663177}},{"question_accuracy":0.46,"valid_response_accuracy":0.46464646464646464,"coverage":0.99,"total_questions":100,"correct_questions":46,"valid_questions":99,"failed_questions":1,"provider_errors":0,"parse_errors":1,"incomplete_responses":0,"question_accuracy_ci":{"method":"wilson","confidence_level":0.95,"lower":0.36560808309024106,"upper":0.5573513967663177}}],"topic_results":[{"topic":"Environmental protection and waste","questions":20,"metrics":{"question_accuracy":0.36666666666666664,"valid_response_accuracy":0.3793103448275862,"coverage":0.9666666666666667,"total_questions":60,"correct_questions":22,"valid_questions":58,"failed_questions":2,"provider_errors":0,"parse_errors":2,"incomplete_responses":0,"question_accuracy_ci":null}},{"topic":"Fire safety","questions":18,"metrics":{"question_accuracy":0.5185185185185185,"valid_response_accuracy":0.5185185185185185,"coverage":1,"total_questions":54,"correct_questions":28,"valid_questions":54,"failed_questions":0,"provider_errors":0,"parse_errors":0,"incomplete_responses":0,"question_accuracy_ci":null}},{"topic":"First aid","questions":20,"metrics":{"question_accuracy":0.3,"valid_response_accuracy":0.3,"coverage":1,"total_questions":60,"correct_questions":18,"valid_questions":60,"failed_questions":0,"provider_errors":0,"parse_errors":0,"incomplete_responses":0,"question_accuracy_ci":null}},{"topic":"Industrial and facility safety","questions":20,"metrics":{"question_accuracy":0.6166666666666667,"valid_response_accuracy":0.6851851851851852,"coverage":0.9,"total_questions":60,"correct_questions":37,"valid_questions":54,"failed_questions":6,"provider_errors":0,"parse_errors":6,"incomplete_responses":0,"question_accuracy_ci":null}},{"topic":"Occupational safety and health","questions":20,"metrics":{"question_accuracy":0.5,"valid_response_accuracy":0.5,"coverage":1,"total_questions":60,"correct_questions":30,"valid_questions":60,"failed_questions":0,"provider_errors":0,"parse_errors":0,"incomplete_responses":0,"question_accuracy_ci":null}},{"topic":"Regulatory administration","questions":2,"metrics":{"question_accuracy":0,"valid_response_accuracy":0,"coverage":1,"total_questions":6,"correct_questions":0,"valid_questions":6,"failed_questions":0,"provider_errors":0,"parse_errors":0,"incomplete_responses":0,"question_accuracy_ci":null}}],"baselines":{"all_false_question_accuracy":0,"all_true_question_accuracy":0},"operational_metrics":{"reported_cost_usd":"0.34376814","responses_with_cost":300,"responses_without_cost":0,"prompt_tokens":51270,"completion_tokens":562509,"reasoning_tokens":623792,"latency_median_seconds":8.646382750001067,"latency_p95_seconds":32.5579712500039,"providers":{"Mistral":300},"format_compliant_responses":0,"format_checked_responses":300},"infrastructure_attempt_count":7},{"model":"mistralai/mistral-small-2603","provider":"openai-compatible","track":"structured-output","completed_at":"2026-10-01T19:39:02.402716Z","status":"success","protocol":{"track":"structured-output","name":"hse-multiselect-zero-shot-v10","selected_dataset_sha256":"f8d2c31d1095aed302e4a2c05ea795c9f897f061275265599fc1e5dba212c239","topics_sha256":"4897b9e828053e1c9ebde4729c8af75baffcd547d251e49a21006eab961d1fff","prompt_sha256":"8cf0f6b182fe981a596b4d48bc132286a16ebf2f25ffbb15df4e27c5a12a07c1","implementation_sha256":"9b2c30085d7db94fdb18d9ebd839bf8340992e93211fad108162f4215644a69a","selected_questions":100,"repetitions":3,"temperature":null,"max_output_tokens":16384,"timeout_seconds":180,"sampling_method":"all","sampling_seed":42,"answer_order":"fixed","reasoning_settings":{"effort":"low"},"routing_settings":{"only":["mistral"],"allow_fallbacks":false,"require_parameters":true},"question_concurrency":8,"request_concurrency":8,"batch_concurrency":1},"metrics":{"question_accuracy":0.43333333333333335,"valid_response_accuracy":0.43333333333333335,"coverage":1,"total_questions":300,"correct_questions":130,"valid_questions":300,"failed_questions":0,"provider_errors":0,"parse_errors":0,"incomplete_responses":0,"question_accuracy_ci":null},"repetition_metrics":[{"question_accuracy":0.47,"valid_response_accuracy":0.47,"coverage":1,"total_questions":100,"correct_questions":47,"valid_questions":100,"failed_questions":0,"provider_errors":0,"parse_errors":0,"incomplete_responses":0,"question_accuracy_ci":{"method":"wilson","confidence_level":0.95,"lower":0.3751081795934127,"upper":0.5671114302990063}},{"question_accuracy":0.42,"valid_response_accuracy":0.42,"coverage":1,"total_questions":100,"correct_questions":42,"valid_questions":100,"failed_questions":0,"provider_errors":0,"parse_errors":0,"incomplete_responses":0,"question_accuracy_ci":{"method":"wilson","confidence_level":0.95,"lower":0.3279838267435473,"upper":0.5179351329695703}},{"question_accuracy":0.41,"valid_response_accuracy":0.41,"coverage":1,"total_questions":100,"correct_questions":41,"valid_questions":100,"failed_questions":0,"provider_errors":0,"parse_errors":0,"incomplete_responses":0,"question_accuracy_ci":{"method":"wilson","confidence_level":0.95,"lower":0.3186731302113651,"upper":0.5079856994658921}}],"topic_results":[{"topic":"Environmental protection and waste","questions":20,"metrics":{"question_accuracy":0.3,"valid_response_accuracy":0.3,"coverage":1,"total_questions":60,"correct_questions":18,"valid_questions":60,"failed_questions":0,"provider_errors":0,"parse_errors":0,"incomplete_responses":0,"question_accuracy_ci":null}},{"topic":"Fire safety","questions":18,"metrics":{"question_accuracy":0.5,"valid_response_accuracy":0.5,"coverage":1,"total_questions":54,"correct_questions":27,"valid_questions":54,"failed_questions":0,"provider_errors":0,"parse_errors":0,"incomplete_responses":0,"question_accuracy_ci":null}},{"topic":"First aid","questions":20,"metrics":{"question_accuracy":0.35,"valid_response_accuracy":0.35,"coverage":1,"total_questions":60,"correct_questions":21,"valid_questions":60,"failed_questions":0,"provider_errors":0,"parse_errors":0,"incomplete_responses":0,"question_accuracy_ci":null}},{"topic":"Industrial and facility safety","questions":20,"metrics":{"question_accuracy":0.6,"valid_response_accuracy":0.6,"coverage":1,"total_questions":60,"correct_questions":36,"valid_questions":60,"failed_questions":0,"provider_errors":0,"parse_errors":0,"incomplete_responses":0,"question_accuracy_ci":null}},{"topic":"Occupational safety and health","questions":20,"metrics":{"question_accuracy":0.4666666666666667,"valid_response_accuracy":0.4666666666666667,"coverage":1,"total_questions":60,"correct_questions":28,"valid_questions":60,"failed_questions":0,"provider_errors":0,"parse_errors":0,"incomplete_responses":0,"question_accuracy_ci":null}},{"topic":"Regulatory administration","questions":2,"metrics":{"question_accuracy":0,"valid_response_accuracy":0,"coverage":1,"total_questions":6,"correct_questions":0,"valid_questions":6,"failed_questions":0,"provider_errors":0,"parse_errors":0,"incomplete_responses":0,"question_accuracy_ci":null}}],"baselines":{"all_false_question_accuracy":0,"all_true_question_accuracy":0},"operational_metrics":{"reported_cost_usd":"0.089824335","responses_with_cost":300,"responses_without_cost":0,"prompt_tokens":38070,"completion_tokens":140437,"reasoning_tokens":174532,"latency_median_seconds":2.7934143745005713,"latency_p95_seconds":7.407358750002459,"providers":{"Mistral":300},"format_compliant_responses":300,"format_checked_responses":300},"infrastructure_attempt_count":4},{"model":"qwen/qwen3.8-max-0902","provider":"openai-compatible","track":"answer-only","completed_at":"2026-10-01T18:17:43.645572Z","status":"success","protocol":{"track":"answer-only","name":"hse-multiselect-zero-shot-v10","selected_dataset_sha256":"f8d2c31d1095aed302e4a2c05ea795c9f897f061275265599fc1e5dba212c239","topics_sha256":"4897b9e828053e1c9ebde4729c8af75baffcd547d251e49a21006eab961d1fff","prompt_sha256":"d1ed6b553cb6997860d5eb7cb36e6399f091187c175b5872e568e40676ab5e68","implementation_sha256":"9b2c30085d7db94fdb18d9ebd839bf8340992e93211fad108162f4215644a69a","selected_questions":100,"repetitions":3,"temperature":null,"max_output_tokens":16384,"timeout_seconds":180,"sampling_method":"all","sampling_seed":42,"answer_order":"fixed","reasoning_settings":{"effort":"low"},"routing_settings":{"only":["alibaba"],"allow_fallbacks":false,"require_parameters":true},"question_concurrency":8,"request_concurrency":8,"batch_concurrency":1},"metrics":{"question_accuracy":0.7033333333333334,"valid_response_accuracy":0.7033333333333334,"coverage":1,"total_questions":300,"correct_questions":211,"valid_questions":300,"failed_questions":0,"provider_errors":0,"parse_errors":0,"incomplete_responses":0,"question_accuracy_ci":null},"repetition_metrics":[{"question_accuracy":0.69,"valid_response_accuracy":0.69,"coverage":1,"total_questions":100,"correct_questions":69,"valid_questions":100,"failed_questions":0,"provider_errors":0,"parse_errors":0,"incomplete_responses":0,"question_accuracy_ci":{"method":"wilson","confidence_level":0.95,"lower":0.5937394428051057,"upper":0.7722030278762395}},{"question_accuracy":0.7,"valid_response_accuracy":0.7,"coverage":1,"total_questions":100,"correct_questions":70,"valid_questions":100,"failed_questions":0,"provider_errors":0,"parse_errors":0,"incomplete_responses":0,"question_accuracy_ci":{"method":"wilson","confidence_level":0.95,"lower":0.6041514536665332,"upper":0.7810511470506724}},{"question_accuracy":0.72,"valid_response_accuracy":0.72,"coverage":1,"total_questions":100,"correct_questions":72,"valid_questions":100,"failed_questions":0,"provider_errors":0,"parse_errors":0,"incomplete_responses":0,"question_accuracy_ci":{"method":"wilson","confidence_level":0.95,"lower":0.6251197129007884,"upper":0.7986031478881379}}],"topic_results":[{"topic":"Environmental protection and waste","questions":20,"metrics":{"question_accuracy":0.6166666666666667,"valid_response_accuracy":0.6166666666666667,"coverage":1,"total_questions":60,"correct_questions":37,"valid_questions":60,"failed_questions":0,"provider_errors":0,"parse_errors":0,"incomplete_responses":0,"question_accuracy_ci":null}},{"topic":"Fire safety","questions":18,"metrics":{"question_accuracy":0.7592592592592593,"valid_response_accuracy":0.7592592592592593,"coverage":1,"total_questions":54,"correct_questions":41,"valid_questions":54,"failed_questions":0,"provider_errors":0,"parse_errors":0,"incomplete_responses":0,"question_accuracy_ci":null}},{"topic":"First aid","questions":20,"metrics":{"question_accuracy":0.48333333333333334,"valid_response_accuracy":0.48333333333333334,"coverage":1,"total_questions":60,"correct_questions":29,"valid_questions":60,"failed_questions":0,"provider_errors":0,"parse_errors":0,"incomplete_responses":0,"question_accuracy_ci":null}},{"topic":"Industrial and facility safety","questions":20,"metrics":{"question_accuracy":0.8833333333333333,"valid_response_accuracy":0.8833333333333333,"coverage":1,"total_questions":60,"correct_questions":53,"valid_questions":60,"failed_questions":0,"provider_errors":0,"parse_errors":0,"incomplete_responses":0,"question_accuracy_ci":null}},{"topic":"Occupational safety and health","questions":20,"metrics":{"question_accuracy":0.7833333333333333,"valid_response_accuracy":0.7833333333333333,"coverage":1,"total_questions":60,"correct_questions":47,"valid_questions":60,"failed_questions":0,"provider_errors":0,"parse_errors":0,"incomplete_responses":0,"question_accuracy_ci":null}},{"topic":"Regulatory administration","questions":2,"metrics":{"question_accuracy":0.6666666666666666,"valid_response_accuracy":0.6666666666666666,"coverage":1,"total_questions":6,"correct_questions":4,"valid_questions":6,"failed_questions":0,"provider_errors":0,"parse_errors":0,"incomplete_responses":0,"question_accuracy_ci":null}}],"baselines":{"all_false_question_accuracy":0,"all_true_question_accuracy":0},"operational_metrics":{"reported_cost_usd":"0.714186","responses_with_cost":300,"responses_without_cost":0,"prompt_tokens":67827,"completion_tokens":96422,"reasoning_tokens":93026,"latency_median_seconds":6.840645541000413,"latency_p95_seconds":15.085513250000076,"providers":{"Alibaba":300},"format_compliant_responses":300,"format_checked_responses":300},"infrastructure_attempt_count":0},{"model":"qwen/qwen3.8-max-0902","provider":"openai-compatible","track":"reasoning-assisted","completed_at":"2026-10-01T18:28:21.515689Z","status":"success","protocol":{"track":"reasoning-assisted","name":"hse-multiselect-zero-shot-v10","selected_dataset_sha256":"f8d2c31d1095aed302e4a2c05ea795c9f897f061275265599fc1e5dba212c239","topics_sha256":"4897b9e828053e1c9ebde4729c8af75baffcd547d251e49a21006eab961d1fff","prompt_sha256":"1e82d8ac2c47f18b33cf9cda4901214138aba1621d45b4d34123be7253a8a2f6","implementation_sha256":"9b2c30085d7db94fdb18d9ebd839bf8340992e93211fad108162f4215644a69a","selected_questions":100,"repetitions":3,"temperature":null,"max_output_tokens":16384,"timeout_seconds":180,"sampling_method":"all","sampling_seed":42,"answer_order":"fixed","reasoning_settings":{"effort":"low"},"routing_settings":{"only":["alibaba"],"allow_fallbacks":false,"require_parameters":true},"question_concurrency":8,"request_concurrency":8,"batch_concurrency":1},"metrics":{"question_accuracy":0.7466666666666667,"valid_response_accuracy":0.7466666666666667,"coverage":1,"total_questions":300,"correct_questions":224,"valid_questions":300,"failed_questions":0,"provider_errors":0,"parse_errors":0,"incomplete_responses":0,"question_accuracy_ci":null},"repetition_metrics":[{"question_accuracy":0.75,"valid_response_accuracy":0.75,"coverage":1,"total_questions":100,"correct_questions":75,"valid_questions":100,"failed_questions":0,"provider_errors":0,"parse_errors":0,"incomplete_responses":0,"question_accuracy_ci":{"method":"wilson","confidence_level":0.95,"lower":0.656955364519384,"upper":0.8245478863771232}},{"question_accuracy":0.76,"valid_response_accuracy":0.76,"coverage":1,"total_questions":100,"correct_questions":76,"valid_questions":100,"failed_questions":0,"provider_errors":0,"parse_errors":0,"incomplete_responses":0,"question_accuracy_ci":{"method":"wilson","confidence_level":0.95,"lower":0.6676766365018553,"upper":0.8330867444305122}},{"question_accuracy":0.73,"valid_response_accuracy":0.73,"coverage":1,"total_questions":100,"correct_questions":73,"valid_questions":100,"failed_questions":0,"provider_errors":0,"parse_errors":0,"incomplete_responses":0,"question_accuracy_ci":{"method":"wilson","confidence_level":0.95,"lower":0.6356788323205499,"upper":0.8073041585042365}}],"topic_results":[{"topic":"Environmental protection and waste","questions":20,"metrics":{"question_accuracy":0.65,"valid_response_accuracy":0.65,"coverage":1,"total_questions":60,"correct_questions":39,"valid_questions":60,"failed_questions":0,"provider_errors":0,"parse_errors":0,"incomplete_responses":0,"question_accuracy_ci":null}},{"topic":"Fire safety","questions":18,"metrics":{"question_accuracy":0.8148148148148148,"valid_response_accuracy":0.8148148148148148,"coverage":1,"total_questions":54,"correct_questions":44,"valid_questions":54,"failed_questions":0,"provider_errors":0,"parse_errors":0,"incomplete_responses":0,"question_accuracy_ci":null}},{"topic":"First aid","questions":20,"metrics":{"question_accuracy":0.5833333333333334,"valid_response_accuracy":0.5833333333333334,"coverage":1,"total_questions":60,"correct_questions":35,"valid_questions":60,"failed_questions":0,"provider_errors":0,"parse_errors":0,"incomplete_responses":0,"question_accuracy_ci":null}},{"topic":"Industrial and facility safety","questions":20,"metrics":{"question_accuracy":0.85,"valid_response_accuracy":0.85,"coverage":1,"total_questions":60,"correct_questions":51,"valid_questions":60,"failed_questions":0,"provider_errors":0,"parse_errors":0,"incomplete_responses":0,"question_accuracy_ci":null}},{"topic":"Occupational safety and health","questions":20,"metrics":{"question_accuracy":0.8166666666666667,"valid_response_accuracy":0.8166666666666667,"coverage":1,"total_questions":60,"correct_questions":49,"valid_questions":60,"failed_questions":0,"provider_errors":0,"parse_errors":0,"incomplete_responses":0,"question_accuracy_ci":null}},{"topic":"Regulatory administration","questions":2,"metrics":{"question_accuracy":1,"valid_response_accuracy":1,"coverage":1,"total_questions":6,"correct_questions":6,"valid_questions":6,"failed_questions":0,"provider_errors":0,"parse_errors":0,"incomplete_responses":0,"question_accuracy_ci":null}}],"baselines":{"all_false_question_accuracy":0,"all_true_question_accuracy":0},"operational_metrics":{"reported_cost_usd":"1.427436","responses_with_cost":300,"responses_without_cost":0,"prompt_tokens":62727,"completion_tokens":216997,"reasoning_tokens":137099,"latency_median_seconds":15.48931716649895,"latency_p95_seconds":27.19829862500046,"providers":{"Alibaba":300},"format_compliant_responses":0,"format_checked_responses":300},"infrastructure_attempt_count":0},{"model":"qwen/qwen3.8-max-0902","provider":"openai-compatible","track":"structured-output","completed_at":"2026-10-01T18:34:13.507047Z","status":"success","protocol":{"track":"structured-output","name":"hse-multiselect-zero-shot-v10","selected_dataset_sha256":"f8d2c31d1095aed302e4a2c05ea795c9f897f061275265599fc1e5dba212c239","topics_sha256":"4897b9e828053e1c9ebde4729c8af75baffcd547d251e49a21006eab961d1fff","prompt_sha256":"8cf0f6b182fe981a596b4d48bc132286a16ebf2f25ffbb15df4e27c5a12a07c1","implementation_sha256":"9b2c30085d7db94fdb18d9ebd839bf8340992e93211fad108162f4215644a69a","selected_questions":100,"repetitions":3,"temperature":null,"max_output_tokens":16384,"timeout_seconds":180,"sampling_method":"all","sampling_seed":42,"answer_order":"fixed","reasoning_settings":{"effort":"low"},"routing_settings":{"only":["alibaba"],"allow_fallbacks":false,"require_parameters":true},"question_concurrency":8,"request_concurrency":8,"batch_concurrency":1},"metrics":{"question_accuracy":0.62,"valid_response_accuracy":0.62,"coverage":1,"total_questions":300,"correct_questions":186,"valid_questions":300,"failed_questions":0,"provider_errors":0,"parse_errors":0,"incomplete_responses":0,"question_accuracy_ci":null},"repetition_metrics":[{"question_accuracy":0.65,"valid_response_accuracy":0.65,"coverage":1,"total_questions":100,"correct_questions":65,"valid_questions":100,"failed_questions":0,"provider_errors":0,"parse_errors":0,"incomplete_responses":0,"question_accuracy_ci":{"method":"wilson","confidence_level":0.95,"lower":0.5525444329688796,"upper":0.7363575175690247}},{"question_accuracy":0.61,"valid_response_accuracy":0.61,"coverage":1,"total_questions":100,"correct_questions":61,"valid_questions":100,"failed_questions":0,"provider_errors":0,"parse_errors":0,"incomplete_responses":0,"question_accuracy_ci":{"method":"wilson","confidence_level":0.95,"lower":0.5120301704584073,"upper":0.6998312599360558}},{"question_accuracy":0.6,"valid_response_accuracy":0.6,"coverage":1,"total_questions":100,"correct_questions":60,"valid_questions":100,"failed_questions":0,"provider_errors":0,"parse_errors":0,"incomplete_responses":0,"question_accuracy_ci":{"method":"wilson","confidence_level":0.95,"lower":0.5020025867910618,"upper":0.6905987135675411}}],"topic_results":[{"topic":"Environmental protection and waste","questions":20,"metrics":{"question_accuracy":0.55,"valid_response_accuracy":0.55,"coverage":1,"total_questions":60,"correct_questions":33,"valid_questions":60,"failed_questions":0,"provider_errors":0,"parse_errors":0,"incomplete_responses":0,"question_accuracy_ci":null}},{"topic":"Fire safety","questions":18,"metrics":{"question_accuracy":0.6296296296296297,"valid_response_accuracy":0.6296296296296297,"coverage":1,"total_questions":54,"correct_questions":34,"valid_questions":54,"failed_questions":0,"provider_errors":0,"parse_errors":0,"incomplete_responses":0,"question_accuracy_ci":null}},{"topic":"First aid","questions":20,"metrics":{"question_accuracy":0.55,"valid_response_accuracy":0.55,"coverage":1,"total_questions":60,"correct_questions":33,"valid_questions":60,"failed_questions":0,"provider_errors":0,"parse_errors":0,"incomplete_responses":0,"question_accuracy_ci":null}},{"topic":"Industrial and facility safety","questions":20,"metrics":{"question_accuracy":0.7333333333333333,"valid_response_accuracy":0.7333333333333333,"coverage":1,"total_questions":60,"correct_questions":44,"valid_questions":60,"failed_questions":0,"provider_errors":0,"parse_errors":0,"incomplete_responses":0,"question_accuracy_ci":null}},{"topic":"Occupational safety and health","questions":20,"metrics":{"question_accuracy":0.6666666666666666,"valid_response_accuracy":0.6666666666666666,"coverage":1,"total_questions":60,"correct_questions":40,"valid_questions":60,"failed_questions":0,"provider_errors":0,"parse_errors":0,"incomplete_responses":0,"question_accuracy_ci":null}},{"topic":"Regulatory administration","questions":2,"metrics":{"question_accuracy":0.3333333333333333,"valid_response_accuracy":0.3333333333333333,"coverage":1,"total_questions":6,"correct_questions":2,"valid_questions":6,"failed_questions":0,"provider_errors":0,"parse_errors":0,"incomplete_responses":0,"question_accuracy_ci":null}}],"baselines":{"all_false_question_accuracy":0,"all_true_question_accuracy":0},"operational_metrics":{"reported_cost_usd":"0.754566","responses_with_cost":300,"responses_without_cost":0,"prompt_tokens":54927,"completion_tokens":107452,"reasoning_tokens":103636,"latency_median_seconds":8.02132672949665,"latency_p95_seconds":17.60823508300382,"providers":{"Alibaba":300},"format_compliant_responses":300,"format_checked_responses":300},"infrastructure_attempt_count":0}]}