{"benchmark":{"excluded_question_count":9,"exclusion_categories":{"malformed":1,"multiple_defensible":7,"wrong_key":1},"name":"GPQA-Diamond-Clean","original_question_count":198,"retained_question_count":189,"version":"1.2.2"},"canonical_evidence":{"contract_version":"gpqa-results-identity-v1","one_row_per_configuration":true,"score_blind":true,"selection_timestamp_policy":"All hash-pinned accepted inputs are assigned the frozen release timestamp only for deterministic canonical selection. This administrative timestamp is not evidence creation, run, first-seen, or real admission time."},"current_release_facts":{"all_accepted_change_summary":{"configuration_count":268,"decrease_count":8,"definition":"Every canonical accepted direct, compatibility, derived-uniform, targeted-recovery, and independent-native configuration in the combined ranking.","first_quartile_delta":{"denominator":88704,"numerator":1087,"percentage_points":"+1.23 pp"},"increase_count":260,"label":"All accepted configurations","median_delta":{"denominator":8316,"numerator":211,"percentage_points":"+2.54 pp"},"third_quartile_delta":{"denominator":66528,"numerator":2039,"percentage_points":"+3.06 pp"},"thresholds":[{"clean_count":68,"crossed_down_count":0,"crossed_up_count":26,"paired_original_count":42,"threshold_percent":90},{"clean_count":29,"crossed_down_count":0,"crossed_up_count":26,"paired_original_count":3,"threshold_percent":95}],"unchanged_count":0},"direct_complete_epoch_change_summary":{"configuration_count":223,"decrease_count":8,"definition":"Only the 223 strict or compatibility-complete direct Epoch public artifacts; excludes derived, targeted-recovery, and independent-native configurations.","first_quartile_delta":{"denominator":12096,"numerator":107,"percentage_points":"+0.88 pp"},"increase_count":215,"label":"Direct complete Epoch artifacts","median_delta":{"denominator":693,"numerator":16,"percentage_points":"+2.31 pp"},"third_quartile_delta":{"denominator":66528,"numerator":1945,"percentage_points":"+2.92 pp"},"thresholds":[{"clean_count":31,"crossed_down_count":0,"crossed_up_count":18,"paired_original_count":13,"threshold_percent":90},{"clean_count":6,"crossed_down_count":0,"crossed_up_count":6,"paired_original_count":0,"threshold_percent":95}],"unchanged_count":0},"ranked_configuration_count":268,"top_50_coverage":{"covered_count":38,"gap_count":12,"source_top_n":50}},"disclosure":{"minimum_configurations":5,"minimum_questions":5,"prohibited":"Question text, options, keys, completions, reasoning, raw evaluation artifacts, private evidence locations, and per-configuration-by-question matrices.","question_output":"aggregate_only"},"eligibility":{"material_source_conflicts_ranked_as_pure_source_results":false,"provider_transport_failures":"Evidence states unless a predeclared terminal-no-answer policy explicitly scores the outcome incorrect.","required_clean_question_coverage":189,"required_original_question_coverage":198,"required_uniform_attempts":true},"evidence_profile_counts":{"derived_uniform":6,"direct_compatibility":116,"direct_strict":107,"project_native":35,"targeted_recovery":4},"independence":"Independent research project; inclusion is not a provider submission, certification, or endorsement.","paired_score_contract":{"clean":"Arithmetic mean accuracy over all 189 retained questions.","delta":"Clean minus paired Original; a published source aggregate is never mixed into this subtraction.","original":"Original 198 is reconstructed from the same accepted attempt universe as Clean 189."},"ranking":{"aggregation":"combined_mean_at_n","compute_normalized":false,"metric":"Clean accuracy over 189 retained questions","ordering":"descending exact rational Clean score","protocol_interpretation":"mean@N is the arithmetic mean across N attempts; it is not best-of-N, pass@N, or majority voting.","tie_policy":"competition exact-rational ties"},"release_id":"gpqa-clean-results-v0.1.14","schema_version":"gpqa-clean-results-methodology-v2","title":"GPQA Diamond Clean results methodology","uncertainty":{"confidence_level":"95%","interpretation":"Observed rank is descriptive; overlapping rank intervals show frontier order fragility.","method":"Synchronized Record-ID cluster bootstrap shared across configurations.","replicates":20000,"seed":20260817}}