{"schema_version":1,"report_date":"2026-10-02","status":"verified_saved_local_and_hosted_results","actions":["keep","drop","mask","pseudonymize","generalize","review"],"local_source_sha256":"df3e1c9c75d104f0ea22ba9efa05482f57e934761cedccd05893c52992a26b1c","counts":{"local_configurations":41,"local_completed_runs_including_repeat":42,"local_selected_predictions":8118,"local_panel_rows":164,"hosted_configurations":6,"hosted_selected_predictions":1188,"evaluated_configurations":47,"selected_predictions":9306,"selected_panel_rows":188,"completed_runs_including_local_repeat":48},"panels":[{"id":"diagnostic43-no_samples","n":43},{"id":"diagnostic43-compact_no_samples","n":43},{"id":"challenge56-no_samples","n":56},{"id":"challenge56-compact_no_samples","n":56}],"records":[{"id":"Mapika--decider-0.8b","platform":"Local","model_id":"Mapika/decider-0.8b","revision":"a0a01d6f8135298f400a8c856b355793012ae971","loading":"float16 / none","scoring_kind":"Native decision readout","settings":{"dtype":"float16","quantization":null,"max_tokens":4096,"effective_max_tokens":4096,"native_temperature":1.03,"trust_remote_code":false,"local_files_only":true,"use_safetensors":true,"guard_threshold":0.45,"seed":0,"tf32":false},"scoring_metadata":{"backend":"decider","protocol":"hf-local-sweep-backends-v1","probability_source":"native_choice_probability_unrounded","latency_scope":"synchronous_forward_scoring_including_transfer_excluding_tokenization","precision":"float16","equal_mass_convention":"rank bins with floor boundaries at floor(i*n/k), k=min(10,n//2); minimum two rows"},"load_ms":8283.399200066924,"peak_allocated_bytes":1620644352,"peak_reserved_bytes":1690304512,"panels":{"diagnostic43-no_samples":{"n":43,"raw":{"n":43,"acceptable":22,"unsafe":0,"review":0,"automated_count":43,"automated_acceptable":22,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":0.5116279069767442,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.5116279069767442,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.3675230846944931,"upper":0.6538255245328142,"confidence_level":0.95}},"effective":{"n":43,"acceptable":22,"unsafe":0,"review":5,"automated_count":38,"automated_acceptable":21,"automated_unsafe":0,"unnecessary_reviews":4,"accuracy":0.5116279069767442,"unsafe_rate":0.0,"review_rate":0.11627906976744186,"automated_coverage":0.8837209302325582,"conditional_correctness":0.5526315789473685,"conditional_unsafe":0.0,"unnecessary_review_rate":0.09302325581395349,"accuracy_wilson_95":{"lower":0.3675230846944931,"upper":0.6538255245328142,"confidence_level":0.95}},"confusion":{"raw":{"keep":{"keep":0,"drop":7,"mask":4,"pseudonymize":1,"generalize":0,"review":0},"drop":{"keep":0,"drop":13,"mask":3,"pseudonymize":0,"generalize":0,"review":0},"mask":{"keep":0,"drop":0,"mask":2,"pseudonymize":0,"generalize":0,"review":0},"pseudonymize":{"keep":0,"drop":0,"mask":0,"pseudonymize":2,"generalize":0,"review":0},"generalize":{"keep":0,"drop":5,"mask":0,"pseudonymize":1,"generalize":0,"review":0},"review":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0}},"effective":{"keep":{"keep":0,"drop":6,"mask":4,"pseudonymize":0,"generalize":0,"review":2},"drop":{"keep":0,"drop":13,"mask":3,"pseudonymize":0,"generalize":0,"review":0},"mask":{"keep":0,"drop":0,"mask":2,"pseudonymize":0,"generalize":0,"review":0},"pseudonymize":{"keep":0,"drop":0,"mask":0,"pseudonymize":2,"generalize":0,"review":0},"generalize":{"keep":0,"drop":4,"mask":0,"pseudonymize":0,"generalize":0,"review":2},"review":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0}}},"probability_metrics":{"singleton_n":38,"multiacceptable_n":5,"multiacceptable_coverage":0.11627906976744186,"nll":1.6385723627428084,"nll_zero_mass_n":0,"brier":0.8201204453069588,"acceptable_set_n":43,"acceptable_set_nll":1.4647212707348432,"acceptable_set_zero_mass_n":0},"calibration_all_labels":{"equal_width":{"n":43,"bin_count":10,"ece":0.2643659087114556,"signed_mean_gap":0.23060883339061294,"absolute_mean_gap":0.23060883339061294,"bins":[{"index":0,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.0,"upper":0.1,"upper_inclusive":false},{"index":1,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.1,"upper":0.2,"upper_inclusive":false},{"index":2,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.2,"upper":0.3,"upper_inclusive":false},{"index":3,"count":2,"confidence":0.3309427350759506,"empirical_correctness":0.5,"signed_gap":-0.16905726492404938,"absolute_gap":0.16905726492404938,"lower":0.3,"upper":0.4,"upper_inclusive":false},{"index":4,"count":5,"confidence":0.4719554722309113,"empirical_correctness":0.4,"signed_gap":0.07195547223091125,"absolute_gap":0.07195547223091125,"lower":0.4,"upper":0.5,"upper_inclusive":false},{"index":5,"count":3,"confidence":0.5374458034833273,"empirical_correctness":0.6666666666666666,"signed_gap":-0.12922086318333936,"absolute_gap":0.12922086318333936,"lower":0.5,"upper":0.6,"upper_inclusive":false},{"index":6,"count":6,"confidence":0.6474883258342743,"empirical_correctness":0.3333333333333333,"signed_gap":0.314154992500941,"absolute_gap":0.314154992500941,"lower":0.6,"upper":0.7,"upper_inclusive":false},{"index":7,"count":6,"confidence":0.744072417418162,"empirical_correctness":0.3333333333333333,"signed_gap":0.4107390840848287,"absolute_gap":0.4107390840848287,"lower":0.7,"upper":0.8,"upper_inclusive":false},{"index":8,"count":10,"confidence":0.8442199885845184,"empirical_correctness":0.6,"signed_gap":0.24421998858451843,"absolute_gap":0.24421998858451843,"lower":0.8,"upper":0.9,"upper_inclusive":false},{"index":9,"count":11,"confidence":0.9536922953345559,"empirical_correctness":0.6363636363636364,"signed_gap":0.31732865897091955,"absolute_gap":0.31732865897091955,"lower":0.9,"upper":1.0,"upper_inclusive":true}]},"equal_mass":{"n":43,"bin_count":10,"ece":0.25279789885809256,"signed_mean_gap":0.23060883339061294,"absolute_mean_gap":0.23060883339061294,"bins":[{"index":0,"count":4,"confidence":0.39076776802539825,"empirical_correctness":0.5,"signed_gap":-0.10923223197460175,"absolute_gap":0.10923223197460175,"min_confidence":0.32724788784980774,"max_confidence":0.4655061960220337},{"index":1,"count":4,"confidence":0.4899660050868988,"empirical_correctness":0.5,"signed_gap":-0.010033994913101196,"absolute_gap":0.010033994913101196,"min_confidence":0.4726715087890625,"max_confidence":0.5012722611427307},{"index":2,"count":4,"confidence":0.5892544388771057,"empirical_correctness":0.5,"signed_gap":0.08925443887710571,"absolute_gap":0.08925443887710571,"min_confidence":0.548438549041748,"max_confidence":0.6234889030456543},{"index":3,"count":5,"confidence":0.6690680623054505,"empirical_correctness":0.2,"signed_gap":0.46906806230545045,"absolute_gap":0.46906806230545045,"min_confidence":0.6394628286361694,"max_confidence":0.7063629627227783},{"index":4,"count":4,"confidence":0.7452692687511444,"empirical_correctness":0.5,"signed_gap":0.2452692687511444,"absolute_gap":0.2452692687511444,"min_confidence":0.7286015152931213,"max_confidence":0.7659128308296204},{"index":5,"count":4,"confidence":0.8151156306266785,"empirical_correctness":0.25,"signed_gap":0.5651156306266785,"absolute_gap":0.5651156306266785,"min_confidence":0.7769944667816162,"max_confidence":0.8335738778114319},{"index":6,"count":5,"confidence":0.8457273960113525,"empirical_correctness":0.6,"signed_gap":0.24572739601135252,"absolute_gap":0.24572739601135252,"min_confidence":0.8353443145751953,"max_confidence":0.8601949214935303},{"index":7,"count":4,"confidence":0.8896246254444122,"empirical_correctness":0.75,"signed_gap":0.13962462544441223,"absolute_gap":0.13962462544441223,"min_confidence":0.862488329410553,"max_confidence":0.9233272075653076},{"index":8,"count":4,"confidence":0.9474639445543289,"empirical_correctness":0.5,"signed_gap":0.4474639445543289,"absolute_gap":0.4474639445543289,"min_confidence":0.9399647116661072,"max_confidence":0.960590124130249},{"index":9,"count":5,"confidence":0.9744711637496948,"empirical_correctness":0.8,"signed_gap":0.17447116374969474,"absolute_gap":0.17447116374969474,"min_confidence":0.9674275517463684,"max_confidence":0.979998767375946}]}},"calibration_singleton_labels":{"equal_width":{"n":38,"bin_count":10,"ece":0.3038340767747478,"signed_mean_gap":0.3038340767747478,"absolute_mean_gap":0.3038340767747478,"bins":[{"index":0,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.0,"upper":0.1,"upper_inclusive":false},{"index":1,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.1,"upper":0.2,"upper_inclusive":false},{"index":2,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.2,"upper":0.3,"upper_inclusive":false},{"index":3,"count":1,"confidence":0.3346375823020935,"empirical_correctness":0.0,"signed_gap":0.3346375823020935,"absolute_gap":0.3346375823020935,"lower":0.3,"upper":0.4,"upper_inclusive":false},{"index":4,"count":5,"confidence":0.4719554722309113,"empirical_correctness":0.4,"signed_gap":0.07195547223091125,"absolute_gap":0.07195547223091125,"lower":0.4,"upper":0.5,"upper_inclusive":false},{"index":5,"count":2,"confidence":0.5319494307041168,"empirical_correctness":0.5,"signed_gap":0.03194943070411682,"absolute_gap":0.03194943070411682,"lower":0.5,"upper":0.6,"upper_inclusive":false},{"index":6,"count":5,"confidence":0.6524932503700256,"empirical_correctness":0.2,"signed_gap":0.4524932503700256,"absolute_gap":0.4524932503700256,"lower":0.6,"upper":0.7,"upper_inclusive":false},{"index":7,"count":6,"confidence":0.744072417418162,"empirical_correctness":0.3333333333333333,"signed_gap":0.4107390840848287,"absolute_gap":0.4107390840848287,"lower":0.7,"upper":0.8,"upper_inclusive":false},{"index":8,"count":10,"confidence":0.8442199885845184,"empirical_correctness":0.6,"signed_gap":0.24421998858451843,"absolute_gap":0.24421998858451843,"lower":0.8,"upper":0.9,"upper_inclusive":false},{"index":9,"count":9,"confidence":0.9575867189301385,"empirical_correctness":0.5555555555555556,"signed_gap":0.4020311633745829,"absolute_gap":0.4020311633745829,"lower":0.9,"upper":1.0,"upper_inclusive":true}]},"equal_mass":{"n":38,"bin_count":10,"ece":0.31095027217739507,"signed_mean_gap":0.3038340767747478,"absolute_mean_gap":0.3038340767747478,"bins":[{"index":0,"count":3,"confidence":0.4119410614172618,"empirical_correctness":0.3333333333333333,"signed_gap":0.07860772808392846,"absolute_gap":0.07860772808392846,"min_confidence":0.3346375823020935,"max_confidence":0.4655061960220337},{"index":1,"count":4,"confidence":0.4899660050868988,"empirical_correctness":0.5,"signed_gap":-0.010033994913101196,"absolute_gap":0.010033994913101196,"min_confidence":0.4726715087890625,"max_confidence":0.5012722611427307},{"index":2,"count":4,"confidence":0.6185442805290222,"empirical_correctness":0.0,"signed_gap":0.6185442805290222,"absolute_gap":0.6185442805290222,"min_confidence":0.5626266002655029,"max_confidence":0.6485987901687622},{"index":3,"count":4,"confidence":0.6964700520038605,"empirical_correctness":0.5,"signed_gap":0.19647005200386047,"absolute_gap":0.19647005200386047,"min_confidence":0.673489511013031,"max_confidence":0.7286015152931213},{"index":4,"count":4,"confidence":0.7573675066232681,"empirical_correctness":0.25,"signed_gap":0.5073675066232681,"absolute_gap":0.5073675066232681,"min_confidence":0.7400485277175903,"max_confidence":0.7769944667816162},{"index":5,"count":3,"confidence":0.8278226852416992,"empirical_correctness":0.3333333333333333,"signed_gap":0.4944893519083659,"absolute_gap":0.4944893519083659,"min_confidence":0.818774938583374,"max_confidence":0.8335738778114319},{"index":6,"count":4,"confidence":0.8421105146408081,"empirical_correctness":0.5,"signed_gap":0.3421105146408081,"absolute_gap":0.3421105146408081,"min_confidence":0.8353443145751953,"max_confidence":0.8540207743644714},{"index":7,"count":4,"confidence":0.8738415539264679,"empirical_correctness":0.75,"signed_gap":0.1238415539264679,"absolute_gap":0.1238415539264679,"min_confidence":0.8601949214935303,"max_confidence":0.9050764441490173},{"index":8,"count":4,"confidence":0.9520689398050308,"empirical_correctness":0.25,"signed_gap":0.7020689398050308,"absolute_gap":0.7020689398050308,"min_confidence":0.9399647116661072,"max_confidence":0.9674275517463684},{"index":9,"count":4,"confidence":0.9762320667505264,"empirical_correctness":1.0,"signed_gap":-0.023767933249473572,"absolute_gap":0.023767933249473572,"min_confidence":0.9717748165130615,"max_confidence":0.979998767375946}]}},"latency_ms":{"n":43,"median":69.52110002748668,"p95":74.67240004334599},"per_action":{"keep":{"n":12,"raw":{"n":12,"acceptable":0,"unsafe":0,"review":0,"automated_count":12,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":2.7755575615628914e-17,"upper":0.24249400665524085,"confidence_level":0.95}},"effective":{"n":12,"acceptable":0,"unsafe":0,"review":2,"automated_count":10,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":2,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":0.16666666666666666,"automated_coverage":0.8333333333333334,"conditional_correctness":0.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.16666666666666666,"accuracy_wilson_95":{"lower":2.7755575615628914e-17,"upper":0.24249400665524085,"confidence_level":0.95}}},"drop":{"n":16,"raw":{"n":16,"acceptable":13,"unsafe":0,"review":0,"automated_count":16,"automated_acceptable":13,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":0.8125,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.8125,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.5699111903802586,"upper":0.9340840092857187,"confidence_level":0.95}},"effective":{"n":16,"acceptable":13,"unsafe":0,"review":0,"automated_count":16,"automated_acceptable":13,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":0.8125,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.8125,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.5699111903802586,"upper":0.9340840092857187,"confidence_level":0.95}}},"mask":{"n":2,"raw":{"n":2,"acceptable":2,"unsafe":0,"review":0,"automated_count":2,"automated_acceptable":2,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":1.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.34238022750665303,"upper":1.0,"confidence_level":0.95}},"effective":{"n":2,"acceptable":2,"unsafe":0,"review":0,"automated_count":2,"automated_acceptable":2,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":1.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.34238022750665303,"upper":1.0,"confidence_level":0.95}}},"pseudonymize":{"n":2,"raw":{"n":2,"acceptable":2,"unsafe":0,"review":0,"automated_count":2,"automated_acceptable":2,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":1.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.34238022750665303,"upper":1.0,"confidence_level":0.95}},"effective":{"n":2,"acceptable":2,"unsafe":0,"review":0,"automated_count":2,"automated_acceptable":2,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":1.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.34238022750665303,"upper":1.0,"confidence_level":0.95}}},"generalize":{"n":6,"raw":{"n":6,"acceptable":0,"unsafe":0,"review":0,"automated_count":6,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":2.7755575615628914e-17,"upper":0.3903342879021653,"confidence_level":0.95}},"effective":{"n":6,"acceptable":0,"unsafe":0,"review":2,"automated_count":4,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":2,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":0.3333333333333333,"automated_coverage":0.6666666666666666,"conditional_correctness":0.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.3333333333333333,"accuracy_wilson_95":{"lower":2.7755575615628914e-17,"upper":0.3903342879021653,"confidence_level":0.95}}},"review":{"n":0,"raw":{"n":0,"acceptable":0,"unsafe":0,"review":0,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":null,"unsafe_rate":null,"review_rate":null,"automated_coverage":null,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":null},"effective":{"n":0,"acceptable":0,"unsafe":0,"review":0,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":null,"unsafe_rate":null,"review_rate":null,"automated_coverage":null,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":null}}}},"diagnostic43-compact_no_samples":{"n":43,"raw":{"n":43,"acceptable":25,"unsafe":0,"review":0,"automated_count":43,"automated_acceptable":25,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":0.5813953488372093,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.5813953488372093,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.4332857738696422,"upper":0.7161544907215082,"confidence_level":0.95}},"effective":{"n":43,"acceptable":25,"unsafe":0,"review":4,"automated_count":39,"automated_acceptable":25,"automated_unsafe":0,"unnecessary_reviews":4,"accuracy":0.5813953488372093,"unsafe_rate":0.0,"review_rate":0.09302325581395349,"automated_coverage":0.9069767441860465,"conditional_correctness":0.6410256410256411,"conditional_unsafe":0.0,"unnecessary_review_rate":0.09302325581395349,"accuracy_wilson_95":{"lower":0.4332857738696422,"upper":0.7161544907215082,"confidence_level":0.95}},"confusion":{"raw":{"keep":{"keep":2,"drop":9,"mask":1,"pseudonymize":0,"generalize":0,"review":0},"drop":{"keep":0,"drop":15,"mask":1,"pseudonymize":0,"generalize":0,"review":0},"mask":{"keep":0,"drop":0,"mask":2,"pseudonymize":0,"generalize":0,"review":0},"pseudonymize":{"keep":0,"drop":1,"mask":0,"pseudonymize":1,"generalize":0,"review":0},"generalize":{"keep":0,"drop":4,"mask":0,"pseudonymize":2,"generalize":0,"review":0},"review":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0}},"effective":{"keep":{"keep":2,"drop":8,"mask":1,"pseudonymize":0,"generalize":0,"review":1},"drop":{"keep":0,"drop":15,"mask":0,"pseudonymize":0,"generalize":0,"review":1},"mask":{"keep":0,"drop":0,"mask":2,"pseudonymize":0,"generalize":0,"review":0},"pseudonymize":{"keep":0,"drop":1,"mask":0,"pseudonymize":1,"generalize":0,"review":0},"generalize":{"keep":0,"drop":4,"mask":0,"pseudonymize":0,"generalize":0,"review":2},"review":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0}}},"probability_metrics":{"singleton_n":38,"multiacceptable_n":5,"multiacceptable_coverage":0.11627906976744186,"nll":1.4051646528740804,"nll_zero_mass_n":0,"brier":0.6827980289837686,"acceptable_set_n":43,"acceptable_set_nll":1.2504626438382018,"acceptable_set_zero_mass_n":0},"calibration_all_labels":{"equal_width":{"n":43,"bin_count":10,"ece":0.26446252090986383,"signed_mean_gap":0.26446252090986383,"absolute_mean_gap":0.26446252090986383,"bins":[{"index":0,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.0,"upper":0.1,"upper_inclusive":false},{"index":1,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.1,"upper":0.2,"upper_inclusive":false},{"index":2,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.2,"upper":0.3,"upper_inclusive":false},{"index":3,"count":1,"confidence":0.39648815989494324,"empirical_correctness":0.0,"signed_gap":0.39648815989494324,"absolute_gap":0.39648815989494324,"lower":0.3,"upper":0.4,"upper_inclusive":false},{"index":4,"count":4,"confidence":0.46459510177373886,"empirical_correctness":0.25,"signed_gap":0.21459510177373886,"absolute_gap":0.21459510177373886,"lower":0.4,"upper":0.5,"upper_inclusive":false},{"index":5,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.5,"upper":0.6,"upper_inclusive":false},{"index":6,"count":2,"confidence":0.6484495997428894,"empirical_correctness":0.5,"signed_gap":0.1484495997428894,"absolute_gap":0.1484495997428894,"lower":0.6,"upper":0.7,"upper_inclusive":false},{"index":7,"count":5,"confidence":0.7383742809295655,"empirical_correctness":0.4,"signed_gap":0.33837428092956545,"absolute_gap":0.33837428092956545,"lower":0.7,"upper":0.8,"upper_inclusive":false},{"index":8,"count":6,"confidence":0.8474394579728445,"empirical_correctness":0.16666666666666666,"signed_gap":0.6807727913061778,"absolute_gap":0.6807727913061778,"lower":0.8,"upper":0.9,"upper_inclusive":false},{"index":9,"count":25,"confidence":0.9617444992065429,"empirical_correctness":0.8,"signed_gap":0.16174449920654288,"absolute_gap":0.16174449920654288,"lower":0.9,"upper":1.0,"upper_inclusive":true}]},"equal_mass":{"n":43,"bin_count":10,"ece":0.2694015114806419,"signed_mean_gap":0.26446252090986383,"absolute_mean_gap":0.26446252090986383,"bins":[{"index":0,"count":4,"confidence":0.44328897446393967,"empirical_correctness":0.25,"signed_gap":0.19328897446393967,"absolute_gap":0.19328897446393967,"min_confidence":0.39648815989494324,"max_confidence":0.47320398688316345},{"index":1,"count":4,"confidence":0.6228713169693947,"empirical_correctness":0.25,"signed_gap":0.3728713169693947,"absolute_gap":0.3728713169693947,"min_confidence":0.48171266913414,"max_confidence":0.7128733992576599},{"index":2,"count":4,"confidence":0.7447495013475418,"empirical_correctness":0.5,"signed_gap":0.2447495013475418,"absolute_gap":0.2447495013475418,"min_confidence":0.7243574261665344,"max_confidence":0.7572187781333923},{"index":3,"count":5,"confidence":0.8381195306777954,"empirical_correctness":0.2,"signed_gap":0.6381195306777954,"absolute_gap":0.6381195306777954,"min_confidence":0.8047348260879517,"max_confidence":0.8792406916618347},{"index":4,"count":4,"confidence":0.9057529866695404,"empirical_correctness":0.5,"signed_gap":0.4057529866695404,"absolute_gap":0.4057529866695404,"min_confidence":0.8940390944480896,"max_confidence":0.9195314645767212},{"index":5,"count":4,"confidence":0.9395738840103149,"empirical_correctness":0.75,"signed_gap":0.18957388401031494,"absolute_gap":0.18957388401031494,"min_confidence":0.9335989952087402,"max_confidence":0.9448042511940002},{"index":6,"count":5,"confidence":0.9541914582252502,"empirical_correctness":0.6,"signed_gap":0.3541914582252502,"absolute_gap":0.3541914582252502,"min_confidence":0.946972668170929,"max_confidence":0.9614132046699524},{"index":7,"count":4,"confidence":0.9728937745094299,"empirical_correctness":0.75,"signed_gap":0.22289377450942993,"absolute_gap":0.22289377450942993,"min_confidence":0.9667752981185913,"max_confidence":0.9768229722976685},{"index":8,"count":4,"confidence":0.9828091412782669,"empirical_correctness":1.0,"signed_gap":-0.017190858721733093,"absolute_gap":0.017190858721733093,"min_confidence":0.9774104952812195,"max_confidence":0.9862850904464722},{"index":9,"count":5,"confidence":0.9925150275230408,"empirical_correctness":1.0,"signed_gap":-0.0074849724769592285,"absolute_gap":0.0074849724769592285,"min_confidence":0.988344669342041,"max_confidence":0.9960477948188782}]}},"calibration_singleton_labels":{"equal_width":{"n":38,"bin_count":10,"ece":0.32235834629912125,"signed_mean_gap":0.32235834629912125,"absolute_mean_gap":0.32235834629912125,"bins":[{"index":0,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.0,"upper":0.1,"upper_inclusive":false},{"index":1,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.1,"upper":0.2,"upper_inclusive":false},{"index":2,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.2,"upper":0.3,"upper_inclusive":false},{"index":3,"count":1,"confidence":0.39648815989494324,"empirical_correctness":0.0,"signed_gap":0.39648815989494324,"absolute_gap":0.39648815989494324,"lower":0.3,"upper":0.4,"upper_inclusive":false},{"index":4,"count":4,"confidence":0.46459510177373886,"empirical_correctness":0.25,"signed_gap":0.21459510177373886,"absolute_gap":0.21459510177373886,"lower":0.4,"upper":0.5,"upper_inclusive":false},{"index":5,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.5,"upper":0.6,"upper_inclusive":false},{"index":6,"count":1,"confidence":0.6258283853530884,"empirical_correctness":0.0,"signed_gap":0.6258283853530884,"absolute_gap":0.6258283853530884,"lower":0.6,"upper":0.7,"upper_inclusive":false},{"index":7,"count":4,"confidence":0.7418784946203232,"empirical_correctness":0.25,"signed_gap":0.4918784946203232,"absolute_gap":0.4918784946203232,"lower":0.7,"upper":0.8,"upper_inclusive":false},{"index":8,"count":5,"confidence":0.8509471416473389,"empirical_correctness":0.0,"signed_gap":0.8509471416473389,"absolute_gap":0.8509471416473389,"lower":0.8,"upper":0.9,"upper_inclusive":false},{"index":9,"count":23,"confidence":0.962898718274158,"empirical_correctness":0.782608695652174,"signed_gap":0.18029002262198401,"absolute_gap":0.18029002262198401,"lower":0.9,"upper":1.0,"upper_inclusive":true}]},"equal_mass":{"n":38,"bin_count":10,"ece":0.3355097786376351,"signed_mean_gap":0.32235834629912125,"absolute_mean_gap":0.32235834629912125,"bins":[{"index":0,"count":3,"confidence":0.43331730365753174,"empirical_correctness":0.3333333333333333,"signed_gap":0.09998397032419842,"absolute_gap":0.09998397032419842,"min_confidence":0.39648815989494324,"max_confidence":0.4668441116809845},{"index":1,"count":4,"confidence":0.5734046101570129,"empirical_correctness":0.0,"signed_gap":0.5734046101570129,"absolute_gap":0.5734046101570129,"min_confidence":0.47320398688316345,"max_confidence":0.7128733992576599},{"index":2,"count":4,"confidence":0.7648438513278961,"empirical_correctness":0.25,"signed_gap":0.5148438513278961,"absolute_gap":0.5148438513278961,"min_confidence":0.7458792924880981,"max_confidence":0.8047348260879517},{"index":3,"count":4,"confidence":0.8625002205371857,"empirical_correctness":0.0,"signed_gap":0.8625002205371857,"absolute_gap":0.8625002205371857,"min_confidence":0.8121876120567322,"max_confidence":0.8940390944480896},{"index":4,"count":4,"confidence":0.9202470034360886,"empirical_correctness":0.5,"signed_gap":0.42024700343608856,"absolute_gap":0.42024700343608856,"min_confidence":0.9033174514770508,"max_confidence":0.9379476308822632},{"index":5,"count":3,"confidence":0.9445738593737284,"empirical_correctness":1.0,"signed_gap":-0.0554261406262716,"absolute_gap":0.0554261406262716,"min_confidence":0.9419446587562561,"max_confidence":0.946972668170929},{"index":6,"count":4,"confidence":0.9559961557388306,"empirical_correctness":0.5,"signed_gap":0.45599615573883057,"absolute_gap":0.45599615573883057,"min_confidence":0.950930118560791,"max_confidence":0.9614132046699524},{"index":7,"count":4,"confidence":0.9728937745094299,"empirical_correctness":0.75,"signed_gap":0.22289377450942993,"absolute_gap":0.22289377450942993,"min_confidence":0.9667752981185913,"max_confidence":0.9768229722976685},{"index":8,"count":4,"confidence":0.9855426847934723,"empirical_correctness":1.0,"signed_gap":-0.01445731520652771,"absolute_gap":0.01445731520652771,"min_confidence":0.983698844909668,"max_confidence":0.988344669342041},{"index":9,"count":4,"confidence":0.9935576170682907,"empirical_correctness":1.0,"signed_gap":-0.0064423829317092896,"absolute_gap":0.0064423829317092896,"min_confidence":0.9911326766014099,"max_confidence":0.9960477948188782}]}},"latency_ms":{"n":43,"median":69.67699993401766,"p95":79.18910996522754},"per_action":{"keep":{"n":12,"raw":{"n":12,"acceptable":2,"unsafe":0,"review":0,"automated_count":12,"automated_acceptable":2,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":0.16666666666666666,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.16666666666666666,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.04696514218385378,"upper":0.4480308622529734,"confidence_level":0.95}},"effective":{"n":12,"acceptable":2,"unsafe":0,"review":1,"automated_count":11,"automated_acceptable":2,"automated_unsafe":0,"unnecessary_reviews":1,"accuracy":0.16666666666666666,"unsafe_rate":0.0,"review_rate":0.08333333333333333,"automated_coverage":0.9166666666666666,"conditional_correctness":0.18181818181818182,"conditional_unsafe":0.0,"unnecessary_review_rate":0.08333333333333333,"accuracy_wilson_95":{"lower":0.04696514218385378,"upper":0.4480308622529734,"confidence_level":0.95}}},"drop":{"n":16,"raw":{"n":16,"acceptable":15,"unsafe":0,"review":0,"automated_count":16,"automated_acceptable":15,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":0.9375,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.9375,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.7167126242970107,"upper":0.9888806552353576,"confidence_level":0.95}},"effective":{"n":16,"acceptable":15,"unsafe":0,"review":1,"automated_count":15,"automated_acceptable":15,"automated_unsafe":0,"unnecessary_reviews":1,"accuracy":0.9375,"unsafe_rate":0.0,"review_rate":0.0625,"automated_coverage":0.9375,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0625,"accuracy_wilson_95":{"lower":0.7167126242970107,"upper":0.9888806552353576,"confidence_level":0.95}}},"mask":{"n":2,"raw":{"n":2,"acceptable":2,"unsafe":0,"review":0,"automated_count":2,"automated_acceptable":2,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":1.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.34238022750665303,"upper":1.0,"confidence_level":0.95}},"effective":{"n":2,"acceptable":2,"unsafe":0,"review":0,"automated_count":2,"automated_acceptable":2,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":1.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.34238022750665303,"upper":1.0,"confidence_level":0.95}}},"pseudonymize":{"n":2,"raw":{"n":2,"acceptable":1,"unsafe":0,"review":0,"automated_count":2,"automated_acceptable":1,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":0.5,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.5,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.09453120573423074,"upper":0.9054687942657693,"confidence_level":0.95}},"effective":{"n":2,"acceptable":1,"unsafe":0,"review":0,"automated_count":2,"automated_acceptable":1,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":0.5,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.5,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.09453120573423074,"upper":0.9054687942657693,"confidence_level":0.95}}},"generalize":{"n":6,"raw":{"n":6,"acceptable":0,"unsafe":0,"review":0,"automated_count":6,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":2.7755575615628914e-17,"upper":0.3903342879021653,"confidence_level":0.95}},"effective":{"n":6,"acceptable":0,"unsafe":0,"review":2,"automated_count":4,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":2,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":0.3333333333333333,"automated_coverage":0.6666666666666666,"conditional_correctness":0.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.3333333333333333,"accuracy_wilson_95":{"lower":2.7755575615628914e-17,"upper":0.3903342879021653,"confidence_level":0.95}}},"review":{"n":0,"raw":{"n":0,"acceptable":0,"unsafe":0,"review":0,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":null,"unsafe_rate":null,"review_rate":null,"automated_coverage":null,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":null},"effective":{"n":0,"acceptable":0,"unsafe":0,"review":0,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":null,"unsafe_rate":null,"review_rate":null,"automated_coverage":null,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":null}}}},"challenge56-no_samples":{"n":56,"raw":{"n":56,"acceptable":44,"unsafe":12,"review":0,"automated_count":56,"automated_acceptable":44,"automated_unsafe":12,"unnecessary_reviews":0,"accuracy":0.7857142857142857,"unsafe_rate":0.21428571428571427,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.7857142857142857,"conditional_unsafe":0.21428571428571427,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.6618053643659488,"upper":0.8729409575090435,"confidence_level":0.95}},"effective":{"n":56,"acceptable":37,"unsafe":0,"review":21,"automated_count":35,"automated_acceptable":35,"automated_unsafe":0,"unnecessary_reviews":19,"accuracy":0.6607142857142857,"unsafe_rate":0.0,"review_rate":0.375,"automated_coverage":0.625,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.3392857142857143,"accuracy_wilson_95":{"lower":0.5299945300706023,"upper":0.770800275984081,"confidence_level":0.95}},"confusion":{"raw":{"keep":{"keep":8,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0},"drop":{"keep":8,"drop":14,"mask":0,"pseudonymize":0,"generalize":0,"review":0},"mask":{"keep":0,"drop":0,"mask":8,"pseudonymize":0,"generalize":0,"review":0},"pseudonymize":{"keep":2,"drop":0,"mask":0,"pseudonymize":6,"generalize":0,"review":0},"generalize":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":8,"review":0},"review":{"keep":2,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0}},"effective":{"keep":{"keep":5,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":3},"drop":{"keep":0,"drop":14,"mask":0,"pseudonymize":0,"generalize":0,"review":8},"mask":{"keep":0,"drop":0,"mask":8,"pseudonymize":0,"generalize":0,"review":0},"pseudonymize":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":8},"generalize":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":8,"review":0},"review":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":2}}},"probability_metrics":{"singleton_n":56,"multiacceptable_n":0,"multiacceptable_coverage":0.0,"nll":0.49801964970724144,"nll_zero_mass_n":0,"brier":0.24713159265375806,"acceptable_set_n":56,"acceptable_set_nll":0.49801964970724144,"acceptable_set_zero_mass_n":0},"calibration_all_labels":{"equal_width":{"n":56,"bin_count":10,"ece":0.10454284452966281,"signed_mean_gap":-0.014086342815841955,"absolute_mean_gap":0.014086342815841955,"bins":[{"index":0,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.0,"upper":0.1,"upper_inclusive":false},{"index":1,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.1,"upper":0.2,"upper_inclusive":false},{"index":2,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.2,"upper":0.3,"upper_inclusive":false},{"index":3,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.3,"upper":0.4,"upper_inclusive":false},{"index":4,"count":4,"confidence":0.46418293565511703,"empirical_correctness":0.0,"signed_gap":0.46418293565511703,"absolute_gap":0.46418293565511703,"lower":0.4,"upper":0.5,"upper_inclusive":false},{"index":5,"count":10,"confidence":0.5676050305366516,"empirical_correctness":0.5,"signed_gap":0.06760503053665157,"absolute_gap":0.06760503053665157,"lower":0.5,"upper":0.6,"upper_inclusive":false},{"index":6,"count":13,"confidence":0.6481926807990441,"empirical_correctness":0.7692307692307693,"signed_gap":-0.12103808843172514,"absolute_gap":0.12103808843172514,"lower":0.6,"upper":0.7,"upper_inclusive":false},{"index":7,"count":2,"confidence":0.7565469741821289,"empirical_correctness":1.0,"signed_gap":-0.2434530258178711,"absolute_gap":0.2434530258178711,"lower":0.7,"upper":0.8,"upper_inclusive":false},{"index":8,"count":4,"confidence":0.864308089017868,"empirical_correctness":1.0,"signed_gap":-0.13569191098213196,"absolute_gap":0.13569191098213196,"lower":0.8,"upper":0.9,"upper_inclusive":false},{"index":9,"count":23,"confidence":0.9687631130218506,"empirical_correctness":1.0,"signed_gap":-0.031236886978149414,"absolute_gap":0.031236886978149414,"lower":0.9,"upper":1.0,"upper_inclusive":true}]},"equal_mass":{"n":56,"bin_count":10,"ece":0.11229331738182477,"signed_mean_gap":-0.01408634281584194,"absolute_mean_gap":0.01408634281584194,"bins":[{"index":0,"count":5,"confidence":0.47270994782447817,"empirical_correctness":0.0,"signed_gap":0.47270994782447817,"absolute_gap":0.47270994782447817,"min_confidence":0.4464477598667145,"max_confidence":0.5068179965019226},{"index":1,"count":6,"confidence":0.5623320440451304,"empirical_correctness":0.5,"signed_gap":0.062332044045130375,"absolute_gap":0.062332044045130375,"min_confidence":0.5329667329788208,"max_confidence":0.5802369713783264},{"index":2,"count":5,"confidence":0.6024506568908692,"empirical_correctness":0.6,"signed_gap":0.0024506568908692072,"absolute_gap":0.0024506568908692072,"min_confidence":0.5965055227279663,"max_confidence":0.6164640784263611},{"index":3,"count":6,"confidence":0.6402617990970612,"empirical_correctness":0.6666666666666666,"signed_gap":-0.026404867569605472,"absolute_gap":0.026404867569605472,"min_confidence":0.6326086521148682,"max_confidence":0.647704541683197},{"index":4,"count":6,"confidence":0.6809718211491903,"empirical_correctness":1.0,"signed_gap":-0.3190281788508097,"absolute_gap":0.3190281788508097,"min_confidence":0.6499901413917542,"max_confidence":0.71791011095047},{"index":5,"count":5,"confidence":0.850483238697052,"empirical_correctness":1.0,"signed_gap":-0.149516761302948,"absolute_gap":0.149516761302948,"min_confidence":0.7951838374137878,"max_confidence":0.8998989462852478},{"index":6,"count":6,"confidence":0.9214481810728709,"empirical_correctness":1.0,"signed_gap":-0.07855181892712915,"absolute_gap":0.07855181892712915,"min_confidence":0.9049102663993835,"max_confidence":0.9353373050689697},{"index":7,"count":5,"confidence":0.9649941444396972,"empirical_correctness":1.0,"signed_gap":-0.03500585556030278,"absolute_gap":0.03500585556030278,"min_confidence":0.9476906657218933,"max_confidence":0.9823557734489441},{"index":8,"count":6,"confidence":0.9907487233479818,"empirical_correctness":1.0,"signed_gap":-0.009251276652018192,"absolute_gap":0.009251276652018192,"min_confidence":0.9836227297782898,"max_confidence":0.9956503510475159},{"index":9,"count":6,"confidence":0.9972332417964935,"empirical_correctness":1.0,"signed_gap":-0.0027667582035064697,"absolute_gap":0.0027667582035064697,"min_confidence":0.9967424273490906,"max_confidence":0.9976711869239807}]}},"calibration_singleton_labels":{"equal_width":{"n":56,"bin_count":10,"ece":0.10454284452966281,"signed_mean_gap":-0.014086342815841955,"absolute_mean_gap":0.014086342815841955,"bins":[{"index":0,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.0,"upper":0.1,"upper_inclusive":false},{"index":1,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.1,"upper":0.2,"upper_inclusive":false},{"index":2,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.2,"upper":0.3,"upper_inclusive":false},{"index":3,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.3,"upper":0.4,"upper_inclusive":false},{"index":4,"count":4,"confidence":0.46418293565511703,"empirical_correctness":0.0,"signed_gap":0.46418293565511703,"absolute_gap":0.46418293565511703,"lower":0.4,"upper":0.5,"upper_inclusive":false},{"index":5,"count":10,"confidence":0.5676050305366516,"empirical_correctness":0.5,"signed_gap":0.06760503053665157,"absolute_gap":0.06760503053665157,"lower":0.5,"upper":0.6,"upper_inclusive":false},{"index":6,"count":13,"confidence":0.6481926807990441,"empirical_correctness":0.7692307692307693,"signed_gap":-0.12103808843172514,"absolute_gap":0.12103808843172514,"lower":0.6,"upper":0.7,"upper_inclusive":false},{"index":7,"count":2,"confidence":0.7565469741821289,"empirical_correctness":1.0,"signed_gap":-0.2434530258178711,"absolute_gap":0.2434530258178711,"lower":0.7,"upper":0.8,"upper_inclusive":false},{"index":8,"count":4,"confidence":0.864308089017868,"empirical_correctness":1.0,"signed_gap":-0.13569191098213196,"absolute_gap":0.13569191098213196,"lower":0.8,"upper":0.9,"upper_inclusive":false},{"index":9,"count":23,"confidence":0.9687631130218506,"empirical_correctness":1.0,"signed_gap":-0.031236886978149414,"absolute_gap":0.031236886978149414,"lower":0.9,"upper":1.0,"upper_inclusive":true}]},"equal_mass":{"n":56,"bin_count":10,"ece":0.11229331738182477,"signed_mean_gap":-0.01408634281584194,"absolute_mean_gap":0.01408634281584194,"bins":[{"index":0,"count":5,"confidence":0.47270994782447817,"empirical_correctness":0.0,"signed_gap":0.47270994782447817,"absolute_gap":0.47270994782447817,"min_confidence":0.4464477598667145,"max_confidence":0.5068179965019226},{"index":1,"count":6,"confidence":0.5623320440451304,"empirical_correctness":0.5,"signed_gap":0.062332044045130375,"absolute_gap":0.062332044045130375,"min_confidence":0.5329667329788208,"max_confidence":0.5802369713783264},{"index":2,"count":5,"confidence":0.6024506568908692,"empirical_correctness":0.6,"signed_gap":0.0024506568908692072,"absolute_gap":0.0024506568908692072,"min_confidence":0.5965055227279663,"max_confidence":0.6164640784263611},{"index":3,"count":6,"confidence":0.6402617990970612,"empirical_correctness":0.6666666666666666,"signed_gap":-0.026404867569605472,"absolute_gap":0.026404867569605472,"min_confidence":0.6326086521148682,"max_confidence":0.647704541683197},{"index":4,"count":6,"confidence":0.6809718211491903,"empirical_correctness":1.0,"signed_gap":-0.3190281788508097,"absolute_gap":0.3190281788508097,"min_confidence":0.6499901413917542,"max_confidence":0.71791011095047},{"index":5,"count":5,"confidence":0.850483238697052,"empirical_correctness":1.0,"signed_gap":-0.149516761302948,"absolute_gap":0.149516761302948,"min_confidence":0.7951838374137878,"max_confidence":0.8998989462852478},{"index":6,"count":6,"confidence":0.9214481810728709,"empirical_correctness":1.0,"signed_gap":-0.07855181892712915,"absolute_gap":0.07855181892712915,"min_confidence":0.9049102663993835,"max_confidence":0.9353373050689697},{"index":7,"count":5,"confidence":0.9649941444396972,"empirical_correctness":1.0,"signed_gap":-0.03500585556030278,"absolute_gap":0.03500585556030278,"min_confidence":0.9476906657218933,"max_confidence":0.9823557734489441},{"index":8,"count":6,"confidence":0.9907487233479818,"empirical_correctness":1.0,"signed_gap":-0.009251276652018192,"absolute_gap":0.009251276652018192,"min_confidence":0.9836227297782898,"max_confidence":0.9956503510475159},{"index":9,"count":6,"confidence":0.9972332417964935,"empirical_correctness":1.0,"signed_gap":-0.0027667582035064697,"absolute_gap":0.0027667582035064697,"min_confidence":0.9967424273490906,"max_confidence":0.9976711869239807}]}},"latency_ms":{"n":56,"median":61.66365009266883,"p95":65.90482487808913},"per_action":{"keep":{"n":8,"raw":{"n":8,"acceptable":8,"unsafe":0,"review":0,"automated_count":8,"automated_acceptable":8,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":1.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.6755924351161198,"upper":1.0,"confidence_level":0.95}},"effective":{"n":8,"acceptable":5,"unsafe":0,"review":3,"automated_count":5,"automated_acceptable":5,"automated_unsafe":0,"unnecessary_reviews":3,"accuracy":0.625,"unsafe_rate":0.0,"review_rate":0.375,"automated_coverage":0.625,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.375,"accuracy_wilson_95":{"lower":0.30574239460262737,"upper":0.8631557141764026,"confidence_level":0.95}}},"drop":{"n":22,"raw":{"n":22,"acceptable":14,"unsafe":8,"review":0,"automated_count":22,"automated_acceptable":14,"automated_unsafe":8,"unnecessary_reviews":0,"accuracy":0.6363636363636364,"unsafe_rate":0.36363636363636365,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.6363636363636364,"conditional_unsafe":0.36363636363636365,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.42951708090131013,"upper":0.8026679532963387,"confidence_level":0.95}},"effective":{"n":22,"acceptable":14,"unsafe":0,"review":8,"automated_count":14,"automated_acceptable":14,"automated_unsafe":0,"unnecessary_reviews":8,"accuracy":0.6363636363636364,"unsafe_rate":0.0,"review_rate":0.36363636363636365,"automated_coverage":0.6363636363636364,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.36363636363636365,"accuracy_wilson_95":{"lower":0.42951708090131013,"upper":0.8026679532963387,"confidence_level":0.95}}},"mask":{"n":8,"raw":{"n":8,"acceptable":8,"unsafe":0,"review":0,"automated_count":8,"automated_acceptable":8,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":1.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.6755924351161198,"upper":1.0,"confidence_level":0.95}},"effective":{"n":8,"acceptable":8,"unsafe":0,"review":0,"automated_count":8,"automated_acceptable":8,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":1.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.6755924351161198,"upper":1.0,"confidence_level":0.95}}},"pseudonymize":{"n":8,"raw":{"n":8,"acceptable":6,"unsafe":2,"review":0,"automated_count":8,"automated_acceptable":6,"automated_unsafe":2,"unnecessary_reviews":0,"accuracy":0.75,"unsafe_rate":0.25,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.75,"conditional_unsafe":0.25,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.40927543031016883,"upper":0.9285207872478909,"confidence_level":0.95}},"effective":{"n":8,"acceptable":0,"unsafe":0,"review":8,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":8,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.32440756488388023,"confidence_level":0.95}}},"generalize":{"n":8,"raw":{"n":8,"acceptable":8,"unsafe":0,"review":0,"automated_count":8,"automated_acceptable":8,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":1.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.6755924351161198,"upper":1.0,"confidence_level":0.95}},"effective":{"n":8,"acceptable":8,"unsafe":0,"review":0,"automated_count":8,"automated_acceptable":8,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":1.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.6755924351161198,"upper":1.0,"confidence_level":0.95}}},"review":{"n":2,"raw":{"n":2,"acceptable":0,"unsafe":2,"review":0,"automated_count":2,"automated_acceptable":0,"automated_unsafe":2,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":1.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":1.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.6576197724933469,"confidence_level":0.95}},"effective":{"n":2,"acceptable":2,"unsafe":0,"review":2,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":1.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.34238022750665303,"upper":1.0,"confidence_level":0.95}}}}},"challenge56-compact_no_samples":{"n":56,"raw":{"n":56,"acceptable":32,"unsafe":22,"review":0,"automated_count":56,"automated_acceptable":32,"automated_unsafe":22,"unnecessary_reviews":0,"accuracy":0.5714285714285714,"unsafe_rate":0.39285714285714285,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.5714285714285714,"conditional_unsafe":0.39285714285714285,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.4413762073081303,"upper":0.6923103731606176,"confidence_level":0.95}},"effective":{"n":56,"acceptable":31,"unsafe":0,"review":26,"automated_count":30,"automated_acceptable":29,"automated_unsafe":0,"unnecessary_reviews":24,"accuracy":0.5535714285714286,"unsafe_rate":0.0,"review_rate":0.4642857142857143,"automated_coverage":0.5357142857142857,"conditional_correctness":0.9666666666666667,"conditional_unsafe":0.0,"unnecessary_review_rate":0.42857142857142855,"accuracy_wilson_95":{"lower":0.4241321478993947,"upper":0.6761327874521665,"confidence_level":0.95}},"confusion":{"raw":{"keep":{"keep":8,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0},"drop":{"keep":7,"drop":15,"mask":0,"pseudonymize":0,"generalize":0,"review":0},"mask":{"keep":0,"drop":0,"mask":8,"pseudonymize":0,"generalize":0,"review":0},"pseudonymize":{"keep":8,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0},"generalize":{"keep":5,"drop":2,"mask":0,"pseudonymize":0,"generalize":1,"review":0},"review":{"keep":2,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0}},"effective":{"keep":{"keep":5,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":3},"drop":{"keep":0,"drop":15,"mask":0,"pseudonymize":0,"generalize":0,"review":7},"mask":{"keep":0,"drop":0,"mask":8,"pseudonymize":0,"generalize":0,"review":0},"pseudonymize":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":8},"generalize":{"keep":0,"drop":1,"mask":0,"pseudonymize":0,"generalize":1,"review":6},"review":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":2}}},"probability_metrics":{"singleton_n":56,"multiacceptable_n":0,"multiacceptable_coverage":0.0,"nll":1.084847027809396,"nll_zero_mass_n":0,"brier":0.573589576243228,"acceptable_set_n":56,"acceptable_set_nll":1.084847027809396,"acceptable_set_zero_mass_n":0},"calibration_all_labels":{"equal_width":{"n":56,"bin_count":10,"ece":0.22480608842202596,"signed_mean_gap":0.20658752002886363,"absolute_mean_gap":0.20658752002886363,"bins":[{"index":0,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.0,"upper":0.1,"upper_inclusive":false},{"index":1,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.1,"upper":0.2,"upper_inclusive":false},{"index":2,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.2,"upper":0.3,"upper_inclusive":false},{"index":3,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.3,"upper":0.4,"upper_inclusive":false},{"index":4,"count":2,"confidence":0.461515873670578,"empirical_correctness":0.5,"signed_gap":-0.038484126329422,"absolute_gap":0.038484126329422,"lower":0.4,"upper":0.5,"upper_inclusive":false},{"index":5,"count":5,"confidence":0.5451355934143066,"empirical_correctness":0.4,"signed_gap":0.1451355934143066,"absolute_gap":0.1451355934143066,"lower":0.5,"upper":0.6,"upper_inclusive":false},{"index":6,"count":11,"confidence":0.6422357667576183,"empirical_correctness":0.36363636363636365,"signed_gap":0.27859940312125464,"absolute_gap":0.27859940312125464,"lower":0.6,"upper":0.7,"upper_inclusive":false},{"index":7,"count":11,"confidence":0.7450778646902605,"empirical_correctness":0.18181818181818182,"signed_gap":0.5632596828720786,"absolute_gap":0.5632596828720786,"lower":0.7,"upper":0.8,"upper_inclusive":false},{"index":8,"count":12,"confidence":0.8410744269688925,"empirical_correctness":0.6666666666666666,"signed_gap":0.17440776030222582,"absolute_gap":0.17440776030222582,"lower":0.8,"upper":0.9,"upper_inclusive":false},{"index":9,"count":15,"confidence":0.9711232225100199,"empirical_correctness":1.0,"signed_gap":-0.028876777489980077,"absolute_gap":0.028876777489980077,"lower":0.9,"upper":1.0,"upper_inclusive":true}]},"equal_mass":{"n":56,"bin_count":10,"ece":0.21395421986069,"signed_mean_gap":0.20658752002886363,"absolute_mean_gap":0.20658752002886363,"bins":[{"index":0,"count":5,"confidence":0.4942778468132019,"empirical_correctness":0.4,"signed_gap":0.0942778468132019,"absolute_gap":0.0942778468132019,"min_confidence":0.427301824092865,"max_confidence":0.5296404957771301},{"index":1,"count":6,"confidence":0.6073012053966522,"empirical_correctness":0.3333333333333333,"signed_gap":0.2739678720633189,"absolute_gap":0.2739678720633189,"min_confidence":0.5858256816864014,"max_confidence":0.6230586171150208},{"index":2,"count":5,"confidence":0.6464778900146484,"empirical_correctness":0.2,"signed_gap":0.4464778900146484,"absolute_gap":0.4464778900146484,"min_confidence":0.6250602006912231,"max_confidence":0.6694173216819763},{"index":3,"count":6,"confidence":0.6997710466384888,"empirical_correctness":0.3333333333333333,"signed_gap":0.36643771330515545,"absolute_gap":0.36643771330515545,"min_confidence":0.6774469614028931,"max_confidence":0.7210413813591003},{"index":4,"count":6,"confidence":0.7617478668689728,"empirical_correctness":0.3333333333333333,"signed_gap":0.42841453353563946,"absolute_gap":0.42841453353563946,"min_confidence":0.7303625345230103,"max_confidence":0.7871072292327881},{"index":5,"count":5,"confidence":0.8157665014266968,"empirical_correctness":0.6,"signed_gap":0.21576650142669684,"absolute_gap":0.21576650142669684,"min_confidence":0.7924602627754211,"max_confidence":0.8283162713050842},{"index":6,"count":6,"confidence":0.8445344666639963,"empirical_correctness":0.6666666666666666,"signed_gap":0.1778677999973297,"absolute_gap":0.1778677999973297,"min_confidence":0.8337398171424866,"max_confidence":0.8535558581352234},{"index":7,"count":5,"confidence":0.9024860024452209,"empirical_correctness":0.8,"signed_gap":0.10248600244522088,"absolute_gap":0.10248600244522088,"min_confidence":0.869296133518219,"max_confidence":0.9316741824150085},{"index":8,"count":6,"confidence":0.9718227783838908,"empirical_correctness":1.0,"signed_gap":-0.02817722161610925,"absolute_gap":0.02817722161610925,"min_confidence":0.933527410030365,"max_confidence":0.9922462105751038},{"index":9,"count":6,"confidence":0.9937992890675863,"empirical_correctness":1.0,"signed_gap":-0.0062007109324137,"absolute_gap":0.0062007109324137,"min_confidence":0.9928542971611023,"max_confidence":0.995458722114563}]}},"calibration_singleton_labels":{"equal_width":{"n":56,"bin_count":10,"ece":0.22480608842202596,"signed_mean_gap":0.20658752002886363,"absolute_mean_gap":0.20658752002886363,"bins":[{"index":0,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.0,"upper":0.1,"upper_inclusive":false},{"index":1,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.1,"upper":0.2,"upper_inclusive":false},{"index":2,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.2,"upper":0.3,"upper_inclusive":false},{"index":3,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.3,"upper":0.4,"upper_inclusive":false},{"index":4,"count":2,"confidence":0.461515873670578,"empirical_correctness":0.5,"signed_gap":-0.038484126329422,"absolute_gap":0.038484126329422,"lower":0.4,"upper":0.5,"upper_inclusive":false},{"index":5,"count":5,"confidence":0.5451355934143066,"empirical_correctness":0.4,"signed_gap":0.1451355934143066,"absolute_gap":0.1451355934143066,"lower":0.5,"upper":0.6,"upper_inclusive":false},{"index":6,"count":11,"confidence":0.6422357667576183,"empirical_correctness":0.36363636363636365,"signed_gap":0.27859940312125464,"absolute_gap":0.27859940312125464,"lower":0.6,"upper":0.7,"upper_inclusive":false},{"index":7,"count":11,"confidence":0.7450778646902605,"empirical_correctness":0.18181818181818182,"signed_gap":0.5632596828720786,"absolute_gap":0.5632596828720786,"lower":0.7,"upper":0.8,"upper_inclusive":false},{"index":8,"count":12,"confidence":0.8410744269688925,"empirical_correctness":0.6666666666666666,"signed_gap":0.17440776030222582,"absolute_gap":0.17440776030222582,"lower":0.8,"upper":0.9,"upper_inclusive":false},{"index":9,"count":15,"confidence":0.9711232225100199,"empirical_correctness":1.0,"signed_gap":-0.028876777489980077,"absolute_gap":0.028876777489980077,"lower":0.9,"upper":1.0,"upper_inclusive":true}]},"equal_mass":{"n":56,"bin_count":10,"ece":0.21395421986069,"signed_mean_gap":0.20658752002886363,"absolute_mean_gap":0.20658752002886363,"bins":[{"index":0,"count":5,"confidence":0.4942778468132019,"empirical_correctness":0.4,"signed_gap":0.0942778468132019,"absolute_gap":0.0942778468132019,"min_confidence":0.427301824092865,"max_confidence":0.5296404957771301},{"index":1,"count":6,"confidence":0.6073012053966522,"empirical_correctness":0.3333333333333333,"signed_gap":0.2739678720633189,"absolute_gap":0.2739678720633189,"min_confidence":0.5858256816864014,"max_confidence":0.6230586171150208},{"index":2,"count":5,"confidence":0.6464778900146484,"empirical_correctness":0.2,"signed_gap":0.4464778900146484,"absolute_gap":0.4464778900146484,"min_confidence":0.6250602006912231,"max_confidence":0.6694173216819763},{"index":3,"count":6,"confidence":0.6997710466384888,"empirical_correctness":0.3333333333333333,"signed_gap":0.36643771330515545,"absolute_gap":0.36643771330515545,"min_confidence":0.6774469614028931,"max_confidence":0.7210413813591003},{"index":4,"count":6,"confidence":0.7617478668689728,"empirical_correctness":0.3333333333333333,"signed_gap":0.42841453353563946,"absolute_gap":0.42841453353563946,"min_confidence":0.7303625345230103,"max_confidence":0.7871072292327881},{"index":5,"count":5,"confidence":0.8157665014266968,"empirical_correctness":0.6,"signed_gap":0.21576650142669684,"absolute_gap":0.21576650142669684,"min_confidence":0.7924602627754211,"max_confidence":0.8283162713050842},{"index":6,"count":6,"confidence":0.8445344666639963,"empirical_correctness":0.6666666666666666,"signed_gap":0.1778677999973297,"absolute_gap":0.1778677999973297,"min_confidence":0.8337398171424866,"max_confidence":0.8535558581352234},{"index":7,"count":5,"confidence":0.9024860024452209,"empirical_correctness":0.8,"signed_gap":0.10248600244522088,"absolute_gap":0.10248600244522088,"min_confidence":0.869296133518219,"max_confidence":0.9316741824150085},{"index":8,"count":6,"confidence":0.9718227783838908,"empirical_correctness":1.0,"signed_gap":-0.02817722161610925,"absolute_gap":0.02817722161610925,"min_confidence":0.933527410030365,"max_confidence":0.9922462105751038},{"index":9,"count":6,"confidence":0.9937992890675863,"empirical_correctness":1.0,"signed_gap":-0.0062007109324137,"absolute_gap":0.0062007109324137,"min_confidence":0.9928542971611023,"max_confidence":0.995458722114563}]}},"latency_ms":{"n":56,"median":69.51519986614585,"p95":73.29559983918443},"per_action":{"keep":{"n":8,"raw":{"n":8,"acceptable":8,"unsafe":0,"review":0,"automated_count":8,"automated_acceptable":8,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":1.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.6755924351161198,"upper":1.0,"confidence_level":0.95}},"effective":{"n":8,"acceptable":5,"unsafe":0,"review":3,"automated_count":5,"automated_acceptable":5,"automated_unsafe":0,"unnecessary_reviews":3,"accuracy":0.625,"unsafe_rate":0.0,"review_rate":0.375,"automated_coverage":0.625,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.375,"accuracy_wilson_95":{"lower":0.30574239460262737,"upper":0.8631557141764026,"confidence_level":0.95}}},"drop":{"n":22,"raw":{"n":22,"acceptable":15,"unsafe":7,"review":0,"automated_count":22,"automated_acceptable":15,"automated_unsafe":7,"unnecessary_reviews":0,"accuracy":0.6818181818181818,"unsafe_rate":0.3181818181818182,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.6818181818181818,"conditional_unsafe":0.3181818181818182,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.47318597741042745,"upper":0.8363940681864375,"confidence_level":0.95}},"effective":{"n":22,"acceptable":15,"unsafe":0,"review":7,"automated_count":15,"automated_acceptable":15,"automated_unsafe":0,"unnecessary_reviews":7,"accuracy":0.6818181818181818,"unsafe_rate":0.0,"review_rate":0.3181818181818182,"automated_coverage":0.6818181818181818,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.3181818181818182,"accuracy_wilson_95":{"lower":0.47318597741042745,"upper":0.8363940681864375,"confidence_level":0.95}}},"mask":{"n":8,"raw":{"n":8,"acceptable":8,"unsafe":0,"review":0,"automated_count":8,"automated_acceptable":8,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":1.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.6755924351161198,"upper":1.0,"confidence_level":0.95}},"effective":{"n":8,"acceptable":8,"unsafe":0,"review":0,"automated_count":8,"automated_acceptable":8,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":1.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.6755924351161198,"upper":1.0,"confidence_level":0.95}}},"pseudonymize":{"n":8,"raw":{"n":8,"acceptable":0,"unsafe":8,"review":0,"automated_count":8,"automated_acceptable":0,"automated_unsafe":8,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":1.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":1.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.32440756488388023,"confidence_level":0.95}},"effective":{"n":8,"acceptable":0,"unsafe":0,"review":8,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":8,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.32440756488388023,"confidence_level":0.95}}},"generalize":{"n":8,"raw":{"n":8,"acceptable":1,"unsafe":5,"review":0,"automated_count":8,"automated_acceptable":1,"automated_unsafe":5,"unnecessary_reviews":0,"accuracy":0.125,"unsafe_rate":0.625,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.125,"conditional_unsafe":0.625,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.02241749145005667,"upper":0.4708881822128535,"confidence_level":0.95}},"effective":{"n":8,"acceptable":1,"unsafe":0,"review":6,"automated_count":2,"automated_acceptable":1,"automated_unsafe":0,"unnecessary_reviews":6,"accuracy":0.125,"unsafe_rate":0.0,"review_rate":0.75,"automated_coverage":0.25,"conditional_correctness":0.5,"conditional_unsafe":0.0,"unnecessary_review_rate":0.75,"accuracy_wilson_95":{"lower":0.02241749145005667,"upper":0.4708881822128535,"confidence_level":0.95}}},"review":{"n":2,"raw":{"n":2,"acceptable":0,"unsafe":2,"review":0,"automated_count":2,"automated_acceptable":0,"automated_unsafe":2,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":1.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":1.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.6576197724933469,"confidence_level":0.95}},"effective":{"n":2,"acceptable":2,"unsafe":0,"review":2,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":1.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.34238022750665303,"upper":1.0,"confidence_level":0.95}}}}}},"composite":{"n":198,"scope":"descriptive sum across four correlated panels","raw":{"n":198,"acceptable":123,"unsafe":34,"review":0,"automated_count":198,"automated_acceptable":123,"automated_unsafe":34,"unnecessary_reviews":0,"accuracy":0.6212121212121212,"unsafe_rate":0.1717171717171717,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.6212121212121212,"conditional_unsafe":0.1717171717171717,"unnecessary_review_rate":0.0},"effective":{"n":198,"acceptable":115,"unsafe":0,"review":56,"automated_count":142,"automated_acceptable":110,"automated_unsafe":0,"unnecessary_reviews":51,"accuracy":0.5808080808080808,"unsafe_rate":0.0,"review_rate":0.2828282828282828,"automated_coverage":0.7171717171717171,"conditional_correctness":0.7746478873239436,"conditional_unsafe":0.0,"unnecessary_review_rate":0.25757575757575757}}},{"id":"Mapika--decider-2b","platform":"Local","model_id":"Mapika/decider-2b","revision":"533964dae8be954c5b5e19fa4948e48408094c1e","loading":"bfloat16 / none","scoring_kind":"Native decision readout","settings":{"dtype":"bfloat16","quantization":null,"max_tokens":4096,"effective_max_tokens":4096,"native_temperature":1.145,"trust_remote_code":false,"local_files_only":true,"use_safetensors":true,"guard_threshold":0.45,"seed":0,"tf32":false},"scoring_metadata":{"backend":"decider","protocol":"hf-local-sweep-backends-v1","probability_source":"native_choice_probability_unrounded","latency_scope":"synchronous_forward_scoring_including_transfer_excluding_tokenization","precision":"bfloat16","equal_mass_convention":"rank bins with floor boundaries at floor(i*n/k), k=min(10,n//2); minimum two rows"},"load_ms":3297.888393048197,"peak_allocated_bytes":3863021568,"peak_reserved_bytes":3888119808,"panels":{"diagnostic43-no_samples":{"n":43,"raw":{"n":43,"acceptable":22,"unsafe":3,"review":0,"automated_count":43,"automated_acceptable":22,"automated_unsafe":3,"unnecessary_reviews":0,"accuracy":0.5116279069767442,"unsafe_rate":0.06976744186046512,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.5116279069767442,"conditional_unsafe":0.06976744186046512,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.3675230846944931,"upper":0.6538255245328142,"confidence_level":0.95}},"effective":{"n":43,"acceptable":22,"unsafe":1,"review":5,"automated_count":38,"automated_acceptable":21,"automated_unsafe":1,"unnecessary_reviews":4,"accuracy":0.5116279069767442,"unsafe_rate":0.023255813953488372,"review_rate":0.11627906976744186,"automated_coverage":0.8837209302325582,"conditional_correctness":0.5526315789473685,"conditional_unsafe":0.02631578947368421,"unnecessary_review_rate":0.09302325581395349,"accuracy_wilson_95":{"lower":0.3675230846944931,"upper":0.6538255245328142,"confidence_level":0.95}},"confusion":{"raw":{"keep":{"keep":0,"drop":5,"mask":6,"pseudonymize":1,"generalize":0,"review":0},"drop":{"keep":0,"drop":9,"mask":4,"pseudonymize":3,"generalize":0,"review":0},"mask":{"keep":0,"drop":0,"mask":2,"pseudonymize":0,"generalize":0,"review":0},"pseudonymize":{"keep":0,"drop":0,"mask":0,"pseudonymize":2,"generalize":0,"review":0},"generalize":{"keep":0,"drop":1,"mask":0,"pseudonymize":1,"generalize":4,"review":0},"review":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0}},"effective":{"keep":{"keep":0,"drop":5,"mask":6,"pseudonymize":0,"generalize":0,"review":1},"drop":{"keep":0,"drop":9,"mask":4,"pseudonymize":1,"generalize":0,"review":2},"mask":{"keep":0,"drop":0,"mask":2,"pseudonymize":0,"generalize":0,"review":0},"pseudonymize":{"keep":0,"drop":0,"mask":0,"pseudonymize":2,"generalize":0,"review":0},"generalize":{"keep":0,"drop":1,"mask":0,"pseudonymize":0,"generalize":4,"review":1},"review":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0}}},"probability_metrics":{"singleton_n":38,"multiacceptable_n":5,"multiacceptable_coverage":0.11627906976744186,"nll":1.5830192215070567,"nll_zero_mass_n":0,"brier":0.7713202317203682,"acceptable_set_n":43,"acceptable_set_nll":1.4092594819364583,"acceptable_set_zero_mass_n":0},"calibration_all_labels":{"equal_width":{"n":43,"bin_count":10,"ece":0.24531643612440243,"signed_mean_gap":0.1781628367512725,"absolute_mean_gap":0.1781628367512725,"bins":[{"index":0,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.0,"upper":0.1,"upper_inclusive":false},{"index":1,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.1,"upper":0.2,"upper_inclusive":false},{"index":2,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.2,"upper":0.3,"upper_inclusive":false},{"index":3,"count":2,"confidence":0.3679283410310745,"empirical_correctness":0.5,"signed_gap":-0.13207165896892548,"absolute_gap":0.13207165896892548,"lower":0.3,"upper":0.4,"upper_inclusive":false},{"index":4,"count":6,"confidence":0.47355252007643384,"empirical_correctness":0.3333333333333333,"signed_gap":0.14021918674310052,"absolute_gap":0.14021918674310052,"lower":0.4,"upper":0.5,"upper_inclusive":false},{"index":5,"count":6,"confidence":0.5444158613681793,"empirical_correctness":0.16666666666666666,"signed_gap":0.3777491947015127,"absolute_gap":0.3777491947015127,"lower":0.5,"upper":0.6,"upper_inclusive":false},{"index":6,"count":6,"confidence":0.6665917138258616,"empirical_correctness":0.3333333333333333,"signed_gap":0.33325838049252826,"absolute_gap":0.33325838049252826,"lower":0.6,"upper":0.7,"upper_inclusive":false},{"index":7,"count":13,"confidence":0.7554108408781198,"empirical_correctness":0.8461538461538461,"signed_gap":-0.09074300527572632,"absolute_gap":0.09074300527572632,"lower":0.7,"upper":0.8,"upper_inclusive":false},{"index":8,"count":5,"confidence":0.8653390049934387,"empirical_correctness":0.6,"signed_gap":0.26533900499343877,"absolute_gap":0.26533900499343877,"lower":0.8,"upper":0.9,"upper_inclusive":false},{"index":9,"count":5,"confidence":0.9341497540473938,"empirical_correctness":0.4,"signed_gap":0.5341497540473937,"absolute_gap":0.5341497540473937,"lower":0.9,"upper":1.0,"upper_inclusive":true}]},"equal_mass":{"n":43,"bin_count":10,"ece":0.2851108908653259,"signed_mean_gap":0.1781628367512725,"absolute_mean_gap":0.1781628367512725,"bins":[{"index":0,"count":4,"confidence":0.40532709658145905,"empirical_correctness":0.25,"signed_gap":0.15532709658145905,"absolute_gap":0.15532709658145905,"min_confidence":0.36376887559890747,"max_confidence":0.4438907206058502},{"index":1,"count":4,"confidence":0.48896585404872894,"empirical_correctness":0.5,"signed_gap":-0.011034145951271057,"absolute_gap":0.011034145951271057,"min_confidence":0.48362767696380615,"max_confidence":0.4936012327671051},{"index":2,"count":4,"confidence":0.5307264477014542,"empirical_correctness":0.25,"signed_gap":0.28072644770145416,"absolute_gap":0.28072644770145416,"min_confidence":0.5148321390151978,"max_confidence":0.5419347286224365},{"index":3,"count":5,"confidence":0.617360782623291,"empirical_correctness":0.0,"signed_gap":0.617360782623291,"absolute_gap":0.617360782623291,"min_confidence":0.5630437135696411,"max_confidence":0.6602174639701843},{"index":4,"count":4,"confidence":0.693947970867157,"empirical_correctness":0.5,"signed_gap":0.19394797086715698,"absolute_gap":0.19394797086715698,"min_confidence":0.6671083569526672,"max_confidence":0.719456136226654},{"index":5,"count":4,"confidence":0.7326613515615463,"empirical_correctness":1.0,"signed_gap":-0.2673386484384537,"absolute_gap":0.2673386484384537,"min_confidence":0.7231569290161133,"max_confidence":0.7431967854499817},{"index":6,"count":5,"confidence":0.7628216028213501,"empirical_correctness":1.0,"signed_gap":-0.23717839717864986,"absolute_gap":0.23717839717864986,"min_confidence":0.7584567070007324,"max_confidence":0.7684677243232727},{"index":7,"count":4,"confidence":0.7958982884883881,"empirical_correctness":0.75,"signed_gap":0.04589828848838806,"absolute_gap":0.04589828848838806,"min_confidence":0.772037923336029,"max_confidence":0.8274617791175842},{"index":8,"count":4,"confidence":0.8748083114624023,"empirical_correctness":0.5,"signed_gap":0.37480831146240234,"absolute_gap":0.37480831146240234,"min_confidence":0.8320838212966919,"max_confidence":0.8947604894638062},{"index":9,"count":5,"confidence":0.9341497540473938,"empirical_correctness":0.4,"signed_gap":0.5341497540473937,"absolute_gap":0.5341497540473937,"min_confidence":0.907824695110321,"max_confidence":0.9599548578262329}]}},"calibration_singleton_labels":{"equal_width":{"n":38,"bin_count":10,"ece":0.2780674374417255,"signed_mean_gap":0.2406083870875208,"absolute_mean_gap":0.2406083870875208,"bins":[{"index":0,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.0,"upper":0.1,"upper_inclusive":false},{"index":1,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.1,"upper":0.2,"upper_inclusive":false},{"index":2,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.2,"upper":0.3,"upper_inclusive":false},{"index":3,"count":1,"confidence":0.36376887559890747,"empirical_correctness":0.0,"signed_gap":0.36376887559890747,"absolute_gap":0.36376887559890747,"lower":0.3,"upper":0.4,"upper_inclusive":false},{"index":4,"count":6,"confidence":0.47355252007643384,"empirical_correctness":0.3333333333333333,"signed_gap":0.14021918674310052,"absolute_gap":0.14021918674310052,"lower":0.4,"upper":0.5,"upper_inclusive":false},{"index":5,"count":6,"confidence":0.5444158613681793,"empirical_correctness":0.16666666666666666,"signed_gap":0.3777491947015127,"absolute_gap":0.3777491947015127,"lower":0.5,"upper":0.6,"upper_inclusive":false},{"index":6,"count":5,"confidence":0.6664883852005005,"empirical_correctness":0.2,"signed_gap":0.46648838520050045,"absolute_gap":0.46648838520050045,"lower":0.6,"upper":0.7,"upper_inclusive":false},{"index":7,"count":11,"confidence":0.7534798221154646,"empirical_correctness":0.8181818181818182,"signed_gap":-0.06470199606635363,"absolute_gap":0.06470199606635363,"lower":0.7,"upper":0.8,"upper_inclusive":false},{"index":8,"count":5,"confidence":0.8653390049934387,"empirical_correctness":0.6,"signed_gap":0.26533900499343877,"absolute_gap":0.26533900499343877,"lower":0.8,"upper":0.9,"upper_inclusive":false},{"index":9,"count":4,"confidence":0.9310311377048492,"empirical_correctness":0.25,"signed_gap":0.6810311377048492,"absolute_gap":0.6810311377048492,"lower":0.9,"upper":1.0,"upper_inclusive":true}]},"equal_mass":{"n":38,"bin_count":10,"ece":0.3370002845400258,"signed_mean_gap":0.2406083870875208,"absolute_mean_gap":0.2406083870875208,"bins":[{"index":0,"count":3,"confidence":0.4164068599541982,"empirical_correctness":0.0,"signed_gap":0.4164068599541982,"absolute_gap":0.4164068599541982,"min_confidence":0.36376887559890747,"max_confidence":0.4438907206058502},{"index":1,"count":4,"confidence":0.48896585404872894,"empirical_correctness":0.5,"signed_gap":-0.011034145951271057,"absolute_gap":0.011034145951271057,"min_confidence":0.48362767696380615,"max_confidence":0.4936012327671051},{"index":2,"count":4,"confidence":0.5307264477014542,"empirical_correctness":0.25,"signed_gap":0.28072644770145416,"absolute_gap":0.28072644770145416,"min_confidence":0.5148321390151978,"max_confidence":0.5419347286224365},{"index":3,"count":4,"confidence":0.6066466122865677,"empirical_correctness":0.0,"signed_gap":0.6066466122865677,"absolute_gap":0.6066466122865677,"min_confidence":0.5630437135696411,"max_confidence":0.6436024308204651},{"index":4,"count":4,"confidence":0.6922252476215363,"empirical_correctness":0.25,"signed_gap":0.44222524762153625,"absolute_gap":0.44222524762153625,"min_confidence":0.6602174639701843,"max_confidence":0.719456136226654},{"index":5,"count":3,"confidence":0.7291495402654012,"empirical_correctness":1.0,"signed_gap":-0.27085045973459876,"absolute_gap":0.27085045973459876,"min_confidence":0.7231569290161133,"max_confidence":0.7405343651771545},{"index":6,"count":4,"confidence":0.7563104778528214,"empirical_correctness":1.0,"signed_gap":-0.24368952214717865,"absolute_gap":0.24368952214717865,"min_confidence":0.7431967854499817,"max_confidence":0.7629786729812622},{"index":7,"count":4,"confidence":0.7958982884883881,"empirical_correctness":0.75,"signed_gap":0.04589828848838806,"absolute_gap":0.04589828848838806,"min_confidence":0.772037923336029,"max_confidence":0.8274617791175842},{"index":8,"count":4,"confidence":0.8748083114624023,"empirical_correctness":0.5,"signed_gap":0.37480831146240234,"absolute_gap":0.37480831146240234,"min_confidence":0.8320838212966919,"max_confidence":0.8947604894638062},{"index":9,"count":4,"confidence":0.9310311377048492,"empirical_correctness":0.25,"signed_gap":0.6810311377048492,"absolute_gap":0.6810311377048492,"min_confidence":0.907824695110321,"max_confidence":0.9599548578262329}]}},"latency_ms":{"n":43,"median":35.834944050293416,"p95":37.21979308174923},"per_action":{"keep":{"n":12,"raw":{"n":12,"acceptable":0,"unsafe":0,"review":0,"automated_count":12,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":2.7755575615628914e-17,"upper":0.24249400665524085,"confidence_level":0.95}},"effective":{"n":12,"acceptable":0,"unsafe":0,"review":1,"automated_count":11,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":1,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":0.08333333333333333,"automated_coverage":0.9166666666666666,"conditional_correctness":0.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.08333333333333333,"accuracy_wilson_95":{"lower":2.7755575615628914e-17,"upper":0.24249400665524085,"confidence_level":0.95}}},"drop":{"n":16,"raw":{"n":16,"acceptable":9,"unsafe":3,"review":0,"automated_count":16,"automated_acceptable":9,"automated_unsafe":3,"unnecessary_reviews":0,"accuracy":0.5625,"unsafe_rate":0.1875,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.5625,"conditional_unsafe":0.1875,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.331785563988119,"upper":0.7690134759450765,"confidence_level":0.95}},"effective":{"n":16,"acceptable":9,"unsafe":1,"review":2,"automated_count":14,"automated_acceptable":9,"automated_unsafe":1,"unnecessary_reviews":2,"accuracy":0.5625,"unsafe_rate":0.0625,"review_rate":0.125,"automated_coverage":0.875,"conditional_correctness":0.6428571428571429,"conditional_unsafe":0.07142857142857142,"unnecessary_review_rate":0.125,"accuracy_wilson_95":{"lower":0.331785563988119,"upper":0.7690134759450765,"confidence_level":0.95}}},"mask":{"n":2,"raw":{"n":2,"acceptable":2,"unsafe":0,"review":0,"automated_count":2,"automated_acceptable":2,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":1.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.34238022750665303,"upper":1.0,"confidence_level":0.95}},"effective":{"n":2,"acceptable":2,"unsafe":0,"review":0,"automated_count":2,"automated_acceptable":2,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":1.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.34238022750665303,"upper":1.0,"confidence_level":0.95}}},"pseudonymize":{"n":2,"raw":{"n":2,"acceptable":2,"unsafe":0,"review":0,"automated_count":2,"automated_acceptable":2,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":1.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.34238022750665303,"upper":1.0,"confidence_level":0.95}},"effective":{"n":2,"acceptable":2,"unsafe":0,"review":0,"automated_count":2,"automated_acceptable":2,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":1.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.34238022750665303,"upper":1.0,"confidence_level":0.95}}},"generalize":{"n":6,"raw":{"n":6,"acceptable":4,"unsafe":0,"review":0,"automated_count":6,"automated_acceptable":4,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":0.6666666666666666,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.6666666666666666,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.299993315138392,"upper":0.9032285888942195,"confidence_level":0.95}},"effective":{"n":6,"acceptable":4,"unsafe":0,"review":1,"automated_count":5,"automated_acceptable":4,"automated_unsafe":0,"unnecessary_reviews":1,"accuracy":0.6666666666666666,"unsafe_rate":0.0,"review_rate":0.16666666666666666,"automated_coverage":0.8333333333333334,"conditional_correctness":0.8,"conditional_unsafe":0.0,"unnecessary_review_rate":0.16666666666666666,"accuracy_wilson_95":{"lower":0.299993315138392,"upper":0.9032285888942195,"confidence_level":0.95}}},"review":{"n":0,"raw":{"n":0,"acceptable":0,"unsafe":0,"review":0,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":null,"unsafe_rate":null,"review_rate":null,"automated_coverage":null,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":null},"effective":{"n":0,"acceptable":0,"unsafe":0,"review":0,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":null,"unsafe_rate":null,"review_rate":null,"automated_coverage":null,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":null}}}},"diagnostic43-compact_no_samples":{"n":43,"raw":{"n":43,"acceptable":32,"unsafe":0,"review":0,"automated_count":43,"automated_acceptable":32,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":0.7441860465116279,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.7441860465116279,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.5976159650115332,"upper":0.8507048287619176,"confidence_level":0.95}},"effective":{"n":43,"acceptable":30,"unsafe":0,"review":2,"automated_count":41,"automated_acceptable":30,"automated_unsafe":0,"unnecessary_reviews":2,"accuracy":0.6976744186046512,"unsafe_rate":0.0,"review_rate":0.046511627906976744,"automated_coverage":0.9534883720930233,"conditional_correctness":0.7317073170731707,"conditional_unsafe":0.0,"unnecessary_review_rate":0.046511627906976744,"accuracy_wilson_95":{"lower":0.5489463582227039,"upper":0.8139799986415182,"confidence_level":0.95}},"confusion":{"raw":{"keep":{"keep":4,"drop":3,"mask":5,"pseudonymize":0,"generalize":0,"review":0},"drop":{"keep":0,"drop":14,"mask":2,"pseudonymize":0,"generalize":0,"review":0},"mask":{"keep":0,"drop":0,"mask":2,"pseudonymize":0,"generalize":0,"review":0},"pseudonymize":{"keep":0,"drop":0,"mask":0,"pseudonymize":2,"generalize":0,"review":0},"generalize":{"keep":0,"drop":1,"mask":0,"pseudonymize":0,"generalize":5,"review":0},"review":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0}},"effective":{"keep":{"keep":2,"drop":3,"mask":5,"pseudonymize":0,"generalize":0,"review":2},"drop":{"keep":0,"drop":14,"mask":2,"pseudonymize":0,"generalize":0,"review":0},"mask":{"keep":0,"drop":0,"mask":2,"pseudonymize":0,"generalize":0,"review":0},"pseudonymize":{"keep":0,"drop":0,"mask":0,"pseudonymize":2,"generalize":0,"review":0},"generalize":{"keep":0,"drop":1,"mask":0,"pseudonymize":0,"generalize":5,"review":0},"review":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0}}},"probability_metrics":{"singleton_n":38,"multiacceptable_n":5,"multiacceptable_coverage":0.11627906976744186,"nll":0.7240165525643177,"nll_zero_mass_n":0,"brier":0.4042608278826911,"acceptable_set_n":43,"acceptable_set_nll":0.6445348305875499,"acceptable_set_zero_mass_n":0},"calibration_all_labels":{"equal_width":{"n":43,"bin_count":10,"ece":0.07925676606422248,"signed_mean_gap":-0.008712237657502626,"absolute_mean_gap":0.008712237657502626,"bins":[{"index":0,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.0,"upper":0.1,"upper_inclusive":false},{"index":1,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.1,"upper":0.2,"upper_inclusive":false},{"index":2,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.2,"upper":0.3,"upper_inclusive":false},{"index":3,"count":1,"confidence":0.32476288080215454,"empirical_correctness":1.0,"signed_gap":-0.6752371191978455,"absolute_gap":0.6752371191978455,"lower":0.3,"upper":0.4,"upper_inclusive":false},{"index":4,"count":5,"confidence":0.4583678960800171,"empirical_correctness":0.4,"signed_gap":0.05836789608001708,"absolute_gap":0.05836789608001708,"lower":0.4,"upper":0.5,"upper_inclusive":false},{"index":5,"count":5,"confidence":0.5372118949890137,"empirical_correctness":0.6,"signed_gap":-0.0627881050109863,"absolute_gap":0.0627881050109863,"lower":0.5,"upper":0.6,"upper_inclusive":false},{"index":6,"count":6,"confidence":0.6669165392716726,"empirical_correctness":0.6666666666666666,"signed_gap":0.0002498726050059741,"absolute_gap":0.0002498726050059741,"lower":0.6,"upper":0.7,"upper_inclusive":false},{"index":7,"count":8,"confidence":0.7622305080294609,"empirical_correctness":0.875,"signed_gap":-0.11276949197053909,"absolute_gap":0.11276949197053909,"lower":0.7,"upper":0.8,"upper_inclusive":false},{"index":8,"count":7,"confidence":0.8516535673822675,"empirical_correctness":0.7142857142857143,"signed_gap":0.1373678530965532,"absolute_gap":0.1373678530965532,"lower":0.8,"upper":0.9,"upper_inclusive":false},{"index":9,"count":11,"confidence":0.9328903339125894,"empirical_correctness":0.9090909090909091,"signed_gap":0.023799424821680293,"absolute_gap":0.023799424821680293,"lower":0.9,"upper":1.0,"upper_inclusive":true}]},"equal_mass":{"n":43,"bin_count":10,"ece":0.12925432033317036,"signed_mean_gap":-0.008712237657502637,"absolute_mean_gap":0.008712237657502637,"bins":[{"index":0,"count":4,"confidence":0.4149260073900223,"empirical_correctness":0.5,"signed_gap":-0.08507399260997772,"absolute_gap":0.08507399260997772,"min_confidence":0.32476288080215454,"max_confidence":0.4597942531108856},{"index":1,"count":4,"confidence":0.4970611482858658,"empirical_correctness":0.75,"signed_gap":-0.2529388517141342,"absolute_gap":0.2529388517141342,"min_confidence":0.4649674594402313,"max_confidence":0.5246106386184692},{"index":2,"count":4,"confidence":0.5753173530101776,"empirical_correctness":0.5,"signed_gap":0.07531735301017761,"absolute_gap":0.07531735301017761,"min_confidence":0.5250059366226196,"max_confidence":0.6465561985969543},{"index":3,"count":5,"confidence":0.6709886074066163,"empirical_correctness":0.6,"signed_gap":0.07098860740661628,"absolute_gap":0.07098860740661628,"min_confidence":0.6586300134658813,"max_confidence":0.682458758354187},{"index":4,"count":4,"confidence":0.7326588034629822,"empirical_correctness":1.0,"signed_gap":-0.2673411965370178,"absolute_gap":0.2673411965370178,"min_confidence":0.7113560438156128,"max_confidence":0.7541726231575012},{"index":5,"count":4,"confidence":0.7918022125959396,"empirical_correctness":0.75,"signed_gap":0.041802212595939636,"absolute_gap":0.041802212595939636,"min_confidence":0.7876652479171753,"max_confidence":0.7961496710777283},{"index":6,"count":5,"confidence":0.8380577564239502,"empirical_correctness":0.6,"signed_gap":0.23805775642395022,"absolute_gap":0.23805775642395022,"min_confidence":0.8165917992591858,"max_confidence":0.8690735101699829},{"index":7,"count":4,"confidence":0.8944861739873886,"empirical_correctness":0.75,"signed_gap":0.1444861739873886,"absolute_gap":0.1444861739873886,"min_confidence":0.880038857460022,"max_confidence":0.9045536518096924},{"index":8,"count":4,"confidence":0.925694078207016,"empirical_correctness":1.0,"signed_gap":-0.07430592179298401,"absolute_gap":0.07430592179298401,"min_confidence":0.9239993095397949,"max_confidence":0.9295586943626404},{"index":9,"count":5,"confidence":0.9504717707633972,"empirical_correctness":1.0,"signed_gap":-0.04952822923660283,"absolute_gap":0.04952822923660283,"min_confidence":0.9330214262008667,"max_confidence":0.9743514657020569}]}},"calibration_singleton_labels":{"equal_width":{"n":38,"bin_count":10,"ece":0.10306799647055176,"signed_mean_gap":0.02773137390613557,"absolute_mean_gap":0.02773137390613557,"bins":[{"index":0,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.0,"upper":0.1,"upper_inclusive":false},{"index":1,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.1,"upper":0.2,"upper_inclusive":false},{"index":2,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.2,"upper":0.3,"upper_inclusive":false},{"index":3,"count":1,"confidence":0.32476288080215454,"empirical_correctness":1.0,"signed_gap":-0.6752371191978455,"absolute_gap":0.6752371191978455,"lower":0.3,"upper":0.4,"upper_inclusive":false},{"index":4,"count":4,"confidence":0.45671800523996353,"empirical_correctness":0.25,"signed_gap":0.20671800523996353,"absolute_gap":0.20671800523996353,"lower":0.4,"upper":0.5,"upper_inclusive":false},{"index":5,"count":5,"confidence":0.5372118949890137,"empirical_correctness":0.6,"signed_gap":-0.0627881050109863,"absolute_gap":0.0627881050109863,"lower":0.5,"upper":0.6,"upper_inclusive":false},{"index":6,"count":5,"confidence":0.6652389407157898,"empirical_correctness":0.6,"signed_gap":0.06523894071578984,"absolute_gap":0.06523894071578984,"lower":0.6,"upper":0.7,"upper_inclusive":false},{"index":7,"count":6,"confidence":0.7596303025881449,"empirical_correctness":0.8333333333333334,"signed_gap":-0.07370303074518847,"absolute_gap":0.07370303074518847,"lower":0.7,"upper":0.8,"upper_inclusive":false},{"index":8,"count":6,"confidence":0.8450546065966288,"empirical_correctness":0.6666666666666666,"signed_gap":0.17838793992996216,"absolute_gap":0.17838793992996216,"lower":0.8,"upper":0.9,"upper_inclusive":false},{"index":9,"count":11,"confidence":0.9328903339125894,"empirical_correctness":0.9090909090909091,"signed_gap":0.023799424821680293,"absolute_gap":0.023799424821680293,"lower":0.9,"upper":1.0,"upper_inclusive":true}]},"equal_mass":{"n":38,"bin_count":10,"ece":0.09718860842679676,"signed_mean_gap":0.02773137390613556,"absolute_mean_gap":0.02773137390613556,"bins":[{"index":0,"count":3,"confidence":0.3999699254830678,"empirical_correctness":0.6666666666666666,"signed_gap":-0.2666967411835988,"absolute_gap":0.2666967411835988,"min_confidence":0.32476288080215454,"max_confidence":0.45245784521102905},{"index":1,"count":4,"confidence":0.49576784670352936,"empirical_correctness":0.5,"signed_gap":-0.004232153296470642,"absolute_gap":0.004232153296470642,"min_confidence":0.4597942531108856,"max_confidence":0.5246106386184692},{"index":2,"count":4,"confidence":0.5753173530101776,"empirical_correctness":0.5,"signed_gap":0.07531735301017761,"absolute_gap":0.07531735301017761,"min_confidence":0.5250059366226196,"max_confidence":0.6465561985969543},{"index":3,"count":4,"confidence":0.6699096262454987,"empirical_correctness":0.5,"signed_gap":0.16990962624549866,"absolute_gap":0.16990962624549866,"min_confidence":0.6586300134658813,"max_confidence":0.682458758354187},{"index":4,"count":4,"confidence":0.7415488809347153,"empirical_correctness":0.75,"signed_gap":-0.008451119065284729,"absolute_gap":0.008451119065284729,"min_confidence":0.7113560438156128,"max_confidence":0.7879573106765747},{"index":5,"count":3,"confidence":0.8027260303497314,"empirical_correctness":0.6666666666666666,"signed_gap":0.13605936368306482,"absolute_gap":0.13605936368306482,"min_confidence":0.7954366207122803,"max_confidence":0.8165917992591858},{"index":6,"count":4,"confidence":0.8434242457151413,"empirical_correctness":0.75,"signed_gap":0.0934242457151413,"absolute_gap":0.0934242457151413,"min_confidence":0.8206393122673035,"max_confidence":0.8690735101699829},{"index":7,"count":4,"confidence":0.9026741683483124,"empirical_correctness":0.75,"signed_gap":0.15267416834831238,"absolute_gap":0.15267416834831238,"min_confidence":0.880038857460022,"max_confidence":0.9239993095397949},{"index":8,"count":4,"confidence":0.9279496073722839,"empirical_correctness":1.0,"signed_gap":-0.07205039262771606,"absolute_gap":0.07205039262771606,"min_confidence":0.9241674542427063,"max_confidence":0.9330214262008667},{"index":9,"count":4,"confidence":0.9548343569040298,"empirical_correctness":1.0,"signed_gap":-0.045165643095970154,"absolute_gap":0.045165643095970154,"min_confidence":0.9365823864936829,"max_confidence":0.9743514657020569}]}},"latency_ms":{"n":43,"median":36.06692497851327,"p95":39.29853899171576},"per_action":{"keep":{"n":12,"raw":{"n":12,"acceptable":4,"unsafe":0,"review":0,"automated_count":12,"automated_acceptable":4,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":0.3333333333333333,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.3333333333333333,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.13812009109121315,"upper":0.6093779111272004,"confidence_level":0.95}},"effective":{"n":12,"acceptable":2,"unsafe":0,"review":2,"automated_count":10,"automated_acceptable":2,"automated_unsafe":0,"unnecessary_reviews":2,"accuracy":0.16666666666666666,"unsafe_rate":0.0,"review_rate":0.16666666666666666,"automated_coverage":0.8333333333333334,"conditional_correctness":0.2,"conditional_unsafe":0.0,"unnecessary_review_rate":0.16666666666666666,"accuracy_wilson_95":{"lower":0.04696514218385378,"upper":0.4480308622529734,"confidence_level":0.95}}},"drop":{"n":16,"raw":{"n":16,"acceptable":14,"unsafe":0,"review":0,"automated_count":16,"automated_acceptable":14,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":0.875,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.875,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.639771727342413,"upper":0.9650225122567595,"confidence_level":0.95}},"effective":{"n":16,"acceptable":14,"unsafe":0,"review":0,"automated_count":16,"automated_acceptable":14,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":0.875,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.875,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.639771727342413,"upper":0.9650225122567595,"confidence_level":0.95}}},"mask":{"n":2,"raw":{"n":2,"acceptable":2,"unsafe":0,"review":0,"automated_count":2,"automated_acceptable":2,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":1.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.34238022750665303,"upper":1.0,"confidence_level":0.95}},"effective":{"n":2,"acceptable":2,"unsafe":0,"review":0,"automated_count":2,"automated_acceptable":2,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":1.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.34238022750665303,"upper":1.0,"confidence_level":0.95}}},"pseudonymize":{"n":2,"raw":{"n":2,"acceptable":2,"unsafe":0,"review":0,"automated_count":2,"automated_acceptable":2,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":1.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.34238022750665303,"upper":1.0,"confidence_level":0.95}},"effective":{"n":2,"acceptable":2,"unsafe":0,"review":0,"automated_count":2,"automated_acceptable":2,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":1.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.34238022750665303,"upper":1.0,"confidence_level":0.95}}},"generalize":{"n":6,"raw":{"n":6,"acceptable":5,"unsafe":0,"review":0,"automated_count":6,"automated_acceptable":5,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":0.8333333333333334,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.8333333333333334,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.43649717781352987,"upper":0.9699466302516935,"confidence_level":0.95}},"effective":{"n":6,"acceptable":5,"unsafe":0,"review":0,"automated_count":6,"automated_acceptable":5,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":0.8333333333333334,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.8333333333333334,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.43649717781352987,"upper":0.9699466302516935,"confidence_level":0.95}}},"review":{"n":0,"raw":{"n":0,"acceptable":0,"unsafe":0,"review":0,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":null,"unsafe_rate":null,"review_rate":null,"automated_coverage":null,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":null},"effective":{"n":0,"acceptable":0,"unsafe":0,"review":0,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":null,"unsafe_rate":null,"review_rate":null,"automated_coverage":null,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":null}}}},"challenge56-no_samples":{"n":56,"raw":{"n":56,"acceptable":51,"unsafe":5,"review":0,"automated_count":56,"automated_acceptable":51,"automated_unsafe":5,"unnecessary_reviews":0,"accuracy":0.9107142857142857,"unsafe_rate":0.08928571428571429,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.9107142857142857,"conditional_unsafe":0.08928571428571429,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.8074399861448884,"upper":0.9612578515504131,"confidence_level":0.95}},"effective":{"n":56,"acceptable":42,"unsafe":0,"review":16,"automated_count":40,"automated_acceptable":40,"automated_unsafe":0,"unnecessary_reviews":14,"accuracy":0.75,"unsafe_rate":0.0,"review_rate":0.2857142857142857,"automated_coverage":0.7142857142857143,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.25,"accuracy_wilson_95":{"lower":0.6230735785233251,"upper":0.8448294531172932,"confidence_level":0.95}},"confusion":{"raw":{"keep":{"keep":8,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0},"drop":{"keep":0,"drop":22,"mask":0,"pseudonymize":0,"generalize":0,"review":0},"mask":{"keep":0,"drop":0,"mask":8,"pseudonymize":0,"generalize":0,"review":0},"pseudonymize":{"keep":0,"drop":0,"mask":0,"pseudonymize":8,"generalize":0,"review":0},"generalize":{"keep":3,"drop":0,"mask":0,"pseudonymize":0,"generalize":5,"review":0},"review":{"keep":2,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0}},"effective":{"keep":{"keep":5,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":3},"drop":{"keep":0,"drop":21,"mask":0,"pseudonymize":0,"generalize":0,"review":1},"mask":{"keep":0,"drop":0,"mask":8,"pseudonymize":0,"generalize":0,"review":0},"pseudonymize":{"keep":0,"drop":0,"mask":0,"pseudonymize":1,"generalize":0,"review":7},"generalize":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":5,"review":3},"review":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":2}}},"probability_metrics":{"singleton_n":56,"multiacceptable_n":0,"multiacceptable_coverage":0.0,"nll":0.3599132370156335,"nll_zero_mass_n":0,"brier":0.15463739054451892,"acceptable_set_n":56,"acceptable_set_nll":0.3599132370156335,"acceptable_set_zero_mass_n":0},"calibration_all_labels":{"equal_width":{"n":56,"bin_count":10,"ece":0.1849204706294196,"signed_mean_gap":-0.13755653479269572,"absolute_mean_gap":0.13755653479269572,"bins":[{"index":0,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.0,"upper":0.1,"upper_inclusive":false},{"index":1,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.1,"upper":0.2,"upper_inclusive":false},{"index":2,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.2,"upper":0.3,"upper_inclusive":false},{"index":3,"count":1,"confidence":0.38676586747169495,"empirical_correctness":1.0,"signed_gap":-0.613234132528305,"absolute_gap":0.613234132528305,"lower":0.3,"upper":0.4,"upper_inclusive":false},{"index":4,"count":3,"confidence":0.44206340114275616,"empirical_correctness":0.0,"signed_gap":0.44206340114275616,"absolute_gap":0.44206340114275616,"lower":0.4,"upper":0.5,"upper_inclusive":false},{"index":5,"count":8,"confidence":0.5561853647232056,"empirical_correctness":0.75,"signed_gap":-0.19381463527679443,"absolute_gap":0.19381463527679443,"lower":0.5,"upper":0.6,"upper_inclusive":false},{"index":6,"count":7,"confidence":0.6648866363934108,"empirical_correctness":1.0,"signed_gap":-0.33511336360658917,"absolute_gap":0.33511336360658917,"lower":0.6,"upper":0.7,"upper_inclusive":false},{"index":7,"count":9,"confidence":0.7618615958425734,"empirical_correctness":1.0,"signed_gap":-0.2381384041574266,"absolute_gap":0.2381384041574266,"lower":0.7,"upper":0.8,"upper_inclusive":false},{"index":8,"count":9,"confidence":0.8415497475200229,"empirical_correctness":1.0,"signed_gap":-0.15845025247997713,"absolute_gap":0.15845025247997713,"lower":0.8,"upper":0.9,"upper_inclusive":false},{"index":9,"count":19,"confidence":0.9499729746266415,"empirical_correctness":1.0,"signed_gap":-0.05002702537335846,"absolute_gap":0.05002702537335846,"lower":0.9,"upper":1.0,"upper_inclusive":true}]},"equal_mass":{"n":56,"bin_count":10,"ece":0.182008829499994,"signed_mean_gap":-0.13755653479269572,"absolute_mean_gap":0.13755653479269572,"bins":[{"index":0,"count":5,"confidence":0.4489328503608704,"empirical_correctness":0.2,"signed_gap":0.24893285036087037,"absolute_gap":0.24893285036087037,"min_confidence":0.38676586747169495,"max_confidence":0.5317081809043884},{"index":1,"count":6,"confidence":0.5539733072121938,"empirical_correctness":0.8333333333333334,"signed_gap":-0.2793600261211395,"absolute_gap":0.2793600261211395,"min_confidence":0.5335661768913269,"max_confidence":0.5791438221931458},{"index":2,"count":5,"confidence":0.6340040326118469,"empirical_correctness":1.0,"signed_gap":-0.3659959673881531,"absolute_gap":0.3659959673881531,"min_confidence":0.5939348936080933,"max_confidence":0.6820259094238281},{"index":3,"count":6,"confidence":0.7156859040260315,"empirical_correctness":1.0,"signed_gap":-0.2843140959739685,"absolute_gap":0.2843140959739685,"min_confidence":0.6824465394020081,"max_confidence":0.7423333525657654},{"index":4,"count":6,"confidence":0.7734600206216177,"empirical_correctness":1.0,"signed_gap":-0.22653997937838233,"absolute_gap":0.22653997937838233,"min_confidence":0.753269612789154,"max_confidence":0.796433687210083},{"index":5,"count":5,"confidence":0.8149956941604615,"empirical_correctness":1.0,"signed_gap":-0.18500430583953853,"absolute_gap":0.18500430583953853,"min_confidence":0.8015180826187134,"max_confidence":0.8310604691505432},{"index":6,"count":6,"confidence":0.8868390023708344,"empirical_correctness":1.0,"signed_gap":-0.11316099762916565,"absolute_gap":0.11316099762916565,"min_confidence":0.8406479954719543,"max_confidence":0.9133273959159851},{"index":7,"count":5,"confidence":0.9255618929862977,"empirical_correctness":1.0,"signed_gap":-0.07443810701370235,"absolute_gap":0.07443810701370235,"min_confidence":0.9140027761459351,"max_confidence":0.9325695037841797},{"index":8,"count":6,"confidence":0.9521501163641611,"empirical_correctness":1.0,"signed_gap":-0.047849883635838864,"absolute_gap":0.047849883635838864,"min_confidence":0.9391838312149048,"max_confidence":0.9733389616012573},{"index":9,"count":6,"confidence":0.9811185995737711,"empirical_correctness":1.0,"signed_gap":-0.018881400426228878,"absolute_gap":0.018881400426228878,"min_confidence":0.9765990376472473,"max_confidence":0.9847915172576904}]}},"calibration_singleton_labels":{"equal_width":{"n":56,"bin_count":10,"ece":0.1849204706294196,"signed_mean_gap":-0.13755653479269572,"absolute_mean_gap":0.13755653479269572,"bins":[{"index":0,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.0,"upper":0.1,"upper_inclusive":false},{"index":1,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.1,"upper":0.2,"upper_inclusive":false},{"index":2,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.2,"upper":0.3,"upper_inclusive":false},{"index":3,"count":1,"confidence":0.38676586747169495,"empirical_correctness":1.0,"signed_gap":-0.613234132528305,"absolute_gap":0.613234132528305,"lower":0.3,"upper":0.4,"upper_inclusive":false},{"index":4,"count":3,"confidence":0.44206340114275616,"empirical_correctness":0.0,"signed_gap":0.44206340114275616,"absolute_gap":0.44206340114275616,"lower":0.4,"upper":0.5,"upper_inclusive":false},{"index":5,"count":8,"confidence":0.5561853647232056,"empirical_correctness":0.75,"signed_gap":-0.19381463527679443,"absolute_gap":0.19381463527679443,"lower":0.5,"upper":0.6,"upper_inclusive":false},{"index":6,"count":7,"confidence":0.6648866363934108,"empirical_correctness":1.0,"signed_gap":-0.33511336360658917,"absolute_gap":0.33511336360658917,"lower":0.6,"upper":0.7,"upper_inclusive":false},{"index":7,"count":9,"confidence":0.7618615958425734,"empirical_correctness":1.0,"signed_gap":-0.2381384041574266,"absolute_gap":0.2381384041574266,"lower":0.7,"upper":0.8,"upper_inclusive":false},{"index":8,"count":9,"confidence":0.8415497475200229,"empirical_correctness":1.0,"signed_gap":-0.15845025247997713,"absolute_gap":0.15845025247997713,"lower":0.8,"upper":0.9,"upper_inclusive":false},{"index":9,"count":19,"confidence":0.9499729746266415,"empirical_correctness":1.0,"signed_gap":-0.05002702537335846,"absolute_gap":0.05002702537335846,"lower":0.9,"upper":1.0,"upper_inclusive":true}]},"equal_mass":{"n":56,"bin_count":10,"ece":0.182008829499994,"signed_mean_gap":-0.13755653479269572,"absolute_mean_gap":0.13755653479269572,"bins":[{"index":0,"count":5,"confidence":0.4489328503608704,"empirical_correctness":0.2,"signed_gap":0.24893285036087037,"absolute_gap":0.24893285036087037,"min_confidence":0.38676586747169495,"max_confidence":0.5317081809043884},{"index":1,"count":6,"confidence":0.5539733072121938,"empirical_correctness":0.8333333333333334,"signed_gap":-0.2793600261211395,"absolute_gap":0.2793600261211395,"min_confidence":0.5335661768913269,"max_confidence":0.5791438221931458},{"index":2,"count":5,"confidence":0.6340040326118469,"empirical_correctness":1.0,"signed_gap":-0.3659959673881531,"absolute_gap":0.3659959673881531,"min_confidence":0.5939348936080933,"max_confidence":0.6820259094238281},{"index":3,"count":6,"confidence":0.7156859040260315,"empirical_correctness":1.0,"signed_gap":-0.2843140959739685,"absolute_gap":0.2843140959739685,"min_confidence":0.6824465394020081,"max_confidence":0.7423333525657654},{"index":4,"count":6,"confidence":0.7734600206216177,"empirical_correctness":1.0,"signed_gap":-0.22653997937838233,"absolute_gap":0.22653997937838233,"min_confidence":0.753269612789154,"max_confidence":0.796433687210083},{"index":5,"count":5,"confidence":0.8149956941604615,"empirical_correctness":1.0,"signed_gap":-0.18500430583953853,"absolute_gap":0.18500430583953853,"min_confidence":0.8015180826187134,"max_confidence":0.8310604691505432},{"index":6,"count":6,"confidence":0.8868390023708344,"empirical_correctness":1.0,"signed_gap":-0.11316099762916565,"absolute_gap":0.11316099762916565,"min_confidence":0.8406479954719543,"max_confidence":0.9133273959159851},{"index":7,"count":5,"confidence":0.9255618929862977,"empirical_correctness":1.0,"signed_gap":-0.07443810701370235,"absolute_gap":0.07443810701370235,"min_confidence":0.9140027761459351,"max_confidence":0.9325695037841797},{"index":8,"count":6,"confidence":0.9521501163641611,"empirical_correctness":1.0,"signed_gap":-0.047849883635838864,"absolute_gap":0.047849883635838864,"min_confidence":0.9391838312149048,"max_confidence":0.9733389616012573},{"index":9,"count":6,"confidence":0.9811185995737711,"empirical_correctness":1.0,"signed_gap":-0.018881400426228878,"absolute_gap":0.018881400426228878,"min_confidence":0.9765990376472473,"max_confidence":0.9847915172576904}]}},"latency_ms":{"n":56,"median":31.500688462983817,"p95":33.67723274277523},"per_action":{"keep":{"n":8,"raw":{"n":8,"acceptable":8,"unsafe":0,"review":0,"automated_count":8,"automated_acceptable":8,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":1.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.6755924351161198,"upper":1.0,"confidence_level":0.95}},"effective":{"n":8,"acceptable":5,"unsafe":0,"review":3,"automated_count":5,"automated_acceptable":5,"automated_unsafe":0,"unnecessary_reviews":3,"accuracy":0.625,"unsafe_rate":0.0,"review_rate":0.375,"automated_coverage":0.625,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.375,"accuracy_wilson_95":{"lower":0.30574239460262737,"upper":0.8631557141764026,"confidence_level":0.95}}},"drop":{"n":22,"raw":{"n":22,"acceptable":22,"unsafe":0,"review":0,"automated_count":22,"automated_acceptable":22,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":1.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.8513451253913791,"upper":1.0,"confidence_level":0.95}},"effective":{"n":22,"acceptable":21,"unsafe":0,"review":1,"automated_count":21,"automated_acceptable":21,"automated_unsafe":0,"unnecessary_reviews":1,"accuracy":0.9545454545454546,"unsafe_rate":0.0,"review_rate":0.045454545454545456,"automated_coverage":0.9545454545454546,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.045454545454545456,"accuracy_wilson_95":{"lower":0.7820195416134267,"upper":0.991930572378736,"confidence_level":0.95}}},"mask":{"n":8,"raw":{"n":8,"acceptable":8,"unsafe":0,"review":0,"automated_count":8,"automated_acceptable":8,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":1.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.6755924351161198,"upper":1.0,"confidence_level":0.95}},"effective":{"n":8,"acceptable":8,"unsafe":0,"review":0,"automated_count":8,"automated_acceptable":8,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":1.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.6755924351161198,"upper":1.0,"confidence_level":0.95}}},"pseudonymize":{"n":8,"raw":{"n":8,"acceptable":8,"unsafe":0,"review":0,"automated_count":8,"automated_acceptable":8,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":1.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.6755924351161198,"upper":1.0,"confidence_level":0.95}},"effective":{"n":8,"acceptable":1,"unsafe":0,"review":7,"automated_count":1,"automated_acceptable":1,"automated_unsafe":0,"unnecessary_reviews":7,"accuracy":0.125,"unsafe_rate":0.0,"review_rate":0.875,"automated_coverage":0.125,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.875,"accuracy_wilson_95":{"lower":0.02241749145005667,"upper":0.4708881822128535,"confidence_level":0.95}}},"generalize":{"n":8,"raw":{"n":8,"acceptable":5,"unsafe":3,"review":0,"automated_count":8,"automated_acceptable":5,"automated_unsafe":3,"unnecessary_reviews":0,"accuracy":0.625,"unsafe_rate":0.375,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.625,"conditional_unsafe":0.375,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.30574239460262737,"upper":0.8631557141764026,"confidence_level":0.95}},"effective":{"n":8,"acceptable":5,"unsafe":0,"review":3,"automated_count":5,"automated_acceptable":5,"automated_unsafe":0,"unnecessary_reviews":3,"accuracy":0.625,"unsafe_rate":0.0,"review_rate":0.375,"automated_coverage":0.625,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.375,"accuracy_wilson_95":{"lower":0.30574239460262737,"upper":0.8631557141764026,"confidence_level":0.95}}},"review":{"n":2,"raw":{"n":2,"acceptable":0,"unsafe":2,"review":0,"automated_count":2,"automated_acceptable":0,"automated_unsafe":2,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":1.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":1.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.6576197724933469,"confidence_level":0.95}},"effective":{"n":2,"acceptable":2,"unsafe":0,"review":2,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":1.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.34238022750665303,"upper":1.0,"confidence_level":0.95}}}}},"challenge56-compact_no_samples":{"n":56,"raw":{"n":56,"acceptable":41,"unsafe":15,"review":0,"automated_count":56,"automated_acceptable":41,"automated_unsafe":15,"unnecessary_reviews":0,"accuracy":0.7321428571428571,"unsafe_rate":0.26785714285714285,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.7321428571428571,"conditional_unsafe":0.26785714285714285,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.6040544407084484,"upper":0.8304269458149828,"confidence_level":0.95}},"effective":{"n":56,"acceptable":34,"unsafe":0,"review":24,"automated_count":32,"automated_acceptable":32,"automated_unsafe":0,"unnecessary_reviews":22,"accuracy":0.6071428571428571,"unsafe_rate":0.0,"review_rate":0.42857142857142855,"automated_coverage":0.5714285714285714,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.39285714285714285,"accuracy_wilson_95":{"lower":0.4763340280042271,"upper":0.7241958426988949,"confidence_level":0.95}},"confusion":{"raw":{"keep":{"keep":8,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0},"drop":{"keep":8,"drop":14,"mask":0,"pseudonymize":0,"generalize":0,"review":0},"mask":{"keep":0,"drop":0,"mask":8,"pseudonymize":0,"generalize":0,"review":0},"pseudonymize":{"keep":2,"drop":0,"mask":0,"pseudonymize":6,"generalize":0,"review":0},"generalize":{"keep":3,"drop":0,"mask":0,"pseudonymize":0,"generalize":5,"review":0},"review":{"keep":2,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0}},"effective":{"keep":{"keep":5,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":3},"drop":{"keep":0,"drop":14,"mask":0,"pseudonymize":0,"generalize":0,"review":8},"mask":{"keep":0,"drop":0,"mask":8,"pseudonymize":0,"generalize":0,"review":0},"pseudonymize":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":8},"generalize":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":5,"review":3},"review":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":2}}},"probability_metrics":{"singleton_n":56,"multiacceptable_n":0,"multiacceptable_coverage":0.0,"nll":0.5574704243760253,"nll_zero_mass_n":0,"brier":0.3151526598725942,"acceptable_set_n":56,"acceptable_set_nll":0.5574704243760253,"acceptable_set_zero_mass_n":0},"calibration_all_labels":{"equal_width":{"n":56,"bin_count":10,"ece":0.20512112921902112,"signed_mean_gap":0.05501571457300867,"absolute_mean_gap":0.05501571457300867,"bins":[{"index":0,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.0,"upper":0.1,"upper_inclusive":false},{"index":1,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.1,"upper":0.2,"upper_inclusive":false},{"index":2,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.2,"upper":0.3,"upper_inclusive":false},{"index":3,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.3,"upper":0.4,"upper_inclusive":false},{"index":4,"count":4,"confidence":0.4500926658511162,"empirical_correctness":0.25,"signed_gap":0.20009266585111618,"absolute_gap":0.20009266585111618,"lower":0.4,"upper":0.5,"upper_inclusive":false},{"index":5,"count":5,"confidence":0.5372989892959594,"empirical_correctness":0.8,"signed_gap":-0.2627010107040406,"absolute_gap":0.2627010107040406,"lower":0.5,"upper":0.6,"upper_inclusive":false},{"index":6,"count":8,"confidence":0.6422559842467308,"empirical_correctness":0.125,"signed_gap":0.5172559842467308,"absolute_gap":0.5172559842467308,"lower":0.6,"upper":0.7,"upper_inclusive":false},{"index":7,"count":7,"confidence":0.7636304412569318,"empirical_correctness":0.42857142857142855,"signed_gap":0.33505901268550325,"absolute_gap":0.33505901268550325,"lower":0.7,"upper":0.8,"upper_inclusive":false},{"index":8,"count":12,"confidence":0.8474684655666351,"empirical_correctness":1.0,"signed_gap":-0.15253153443336487,"absolute_gap":0.15253153443336487,"lower":0.8,"upper":0.9,"upper_inclusive":false},{"index":9,"count":20,"confidence":0.9470465928316116,"empirical_correctness":1.0,"signed_gap":-0.05295340716838837,"absolute_gap":0.05295340716838837,"lower":0.9,"upper":1.0,"upper_inclusive":true}]},"equal_mass":{"n":56,"bin_count":10,"ece":0.19392737693020276,"signed_mean_gap":0.055015714573008655,"absolute_mean_gap":0.055015714573008655,"bins":[{"index":0,"count":5,"confidence":0.4607465207576752,"empirical_correctness":0.4,"signed_gap":0.06074652075767517,"absolute_gap":0.06074652075767517,"min_confidence":0.432340532541275,"max_confidence":0.5033619403839111},{"index":1,"count":6,"confidence":0.5689657727877299,"empirical_correctness":0.6666666666666666,"signed_gap":-0.09770089387893677,"absolute_gap":0.09770089387893677,"min_confidence":0.5063885450363159,"max_confidence":0.6233711242675781},{"index":2,"count":5,"confidence":0.6446181654930114,"empirical_correctness":0.0,"signed_gap":0.6446181654930114,"absolute_gap":0.6446181654930114,"min_confidence":0.6321187615394592,"max_confidence":0.6686741709709167},{"index":3,"count":6,"confidence":0.7405971884727478,"empirical_correctness":0.16666666666666666,"signed_gap":0.5739305218060812,"absolute_gap":0.5739305218060812,"min_confidence":0.6842954158782959,"max_confidence":0.7876046895980835},{"index":4,"count":6,"confidence":0.8098526398340861,"empirical_correctness":1.0,"signed_gap":-0.19014736016591394,"absolute_gap":0.19014736016591394,"min_confidence":0.7909502387046814,"max_confidence":0.8276995420455933},{"index":5,"count":5,"confidence":0.8499115705490112,"empirical_correctness":1.0,"signed_gap":-0.15008842945098877,"absolute_gap":0.15008842945098877,"min_confidence":0.8353533148765564,"max_confidence":0.8676842451095581},{"index":6,"count":6,"confidence":0.8971382180849711,"empirical_correctness":1.0,"signed_gap":-0.10286178191502893,"absolute_gap":0.10286178191502893,"min_confidence":0.8739129304885864,"max_confidence":0.9138301610946655},{"index":7,"count":5,"confidence":0.9291700839996337,"empirical_correctness":1.0,"signed_gap":-0.07082991600036626,"absolute_gap":0.07082991600036626,"min_confidence":0.920804500579834,"max_confidence":0.9420198798179626},{"index":8,"count":6,"confidence":0.9542747437953949,"empirical_correctness":1.0,"signed_gap":-0.0457252562046051,"absolute_gap":0.0457252562046051,"min_confidence":0.9426603317260742,"max_confidence":0.9676852822303772},{"index":9,"count":6,"confidence":0.9722794890403748,"empirical_correctness":1.0,"signed_gap":-0.027720510959625244,"absolute_gap":0.027720510959625244,"min_confidence":0.9692531824111938,"max_confidence":0.9763886332511902}]}},"calibration_singleton_labels":{"equal_width":{"n":56,"bin_count":10,"ece":0.20512112921902112,"signed_mean_gap":0.05501571457300867,"absolute_mean_gap":0.05501571457300867,"bins":[{"index":0,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.0,"upper":0.1,"upper_inclusive":false},{"index":1,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.1,"upper":0.2,"upper_inclusive":false},{"index":2,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.2,"upper":0.3,"upper_inclusive":false},{"index":3,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.3,"upper":0.4,"upper_inclusive":false},{"index":4,"count":4,"confidence":0.4500926658511162,"empirical_correctness":0.25,"signed_gap":0.20009266585111618,"absolute_gap":0.20009266585111618,"lower":0.4,"upper":0.5,"upper_inclusive":false},{"index":5,"count":5,"confidence":0.5372989892959594,"empirical_correctness":0.8,"signed_gap":-0.2627010107040406,"absolute_gap":0.2627010107040406,"lower":0.5,"upper":0.6,"upper_inclusive":false},{"index":6,"count":8,"confidence":0.6422559842467308,"empirical_correctness":0.125,"signed_gap":0.5172559842467308,"absolute_gap":0.5172559842467308,"lower":0.6,"upper":0.7,"upper_inclusive":false},{"index":7,"count":7,"confidence":0.7636304412569318,"empirical_correctness":0.42857142857142855,"signed_gap":0.33505901268550325,"absolute_gap":0.33505901268550325,"lower":0.7,"upper":0.8,"upper_inclusive":false},{"index":8,"count":12,"confidence":0.8474684655666351,"empirical_correctness":1.0,"signed_gap":-0.15253153443336487,"absolute_gap":0.15253153443336487,"lower":0.8,"upper":0.9,"upper_inclusive":false},{"index":9,"count":20,"confidence":0.9470465928316116,"empirical_correctness":1.0,"signed_gap":-0.05295340716838837,"absolute_gap":0.05295340716838837,"lower":0.9,"upper":1.0,"upper_inclusive":true}]},"equal_mass":{"n":56,"bin_count":10,"ece":0.19392737693020276,"signed_mean_gap":0.055015714573008655,"absolute_mean_gap":0.055015714573008655,"bins":[{"index":0,"count":5,"confidence":0.4607465207576752,"empirical_correctness":0.4,"signed_gap":0.06074652075767517,"absolute_gap":0.06074652075767517,"min_confidence":0.432340532541275,"max_confidence":0.5033619403839111},{"index":1,"count":6,"confidence":0.5689657727877299,"empirical_correctness":0.6666666666666666,"signed_gap":-0.09770089387893677,"absolute_gap":0.09770089387893677,"min_confidence":0.5063885450363159,"max_confidence":0.6233711242675781},{"index":2,"count":5,"confidence":0.6446181654930114,"empirical_correctness":0.0,"signed_gap":0.6446181654930114,"absolute_gap":0.6446181654930114,"min_confidence":0.6321187615394592,"max_confidence":0.6686741709709167},{"index":3,"count":6,"confidence":0.7405971884727478,"empirical_correctness":0.16666666666666666,"signed_gap":0.5739305218060812,"absolute_gap":0.5739305218060812,"min_confidence":0.6842954158782959,"max_confidence":0.7876046895980835},{"index":4,"count":6,"confidence":0.8098526398340861,"empirical_correctness":1.0,"signed_gap":-0.19014736016591394,"absolute_gap":0.19014736016591394,"min_confidence":0.7909502387046814,"max_confidence":0.8276995420455933},{"index":5,"count":5,"confidence":0.8499115705490112,"empirical_correctness":1.0,"signed_gap":-0.15008842945098877,"absolute_gap":0.15008842945098877,"min_confidence":0.8353533148765564,"max_confidence":0.8676842451095581},{"index":6,"count":6,"confidence":0.8971382180849711,"empirical_correctness":1.0,"signed_gap":-0.10286178191502893,"absolute_gap":0.10286178191502893,"min_confidence":0.8739129304885864,"max_confidence":0.9138301610946655},{"index":7,"count":5,"confidence":0.9291700839996337,"empirical_correctness":1.0,"signed_gap":-0.07082991600036626,"absolute_gap":0.07082991600036626,"min_confidence":0.920804500579834,"max_confidence":0.9420198798179626},{"index":8,"count":6,"confidence":0.9542747437953949,"empirical_correctness":1.0,"signed_gap":-0.0457252562046051,"absolute_gap":0.0457252562046051,"min_confidence":0.9426603317260742,"max_confidence":0.9676852822303772},{"index":9,"count":6,"confidence":0.9722794890403748,"empirical_correctness":1.0,"signed_gap":-0.027720510959625244,"absolute_gap":0.027720510959625244,"min_confidence":0.9692531824111938,"max_confidence":0.9763886332511902}]}},"latency_ms":{"n":56,"median":36.29336800076999,"p95":39.514588002930395},"per_action":{"keep":{"n":8,"raw":{"n":8,"acceptable":8,"unsafe":0,"review":0,"automated_count":8,"automated_acceptable":8,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":1.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.6755924351161198,"upper":1.0,"confidence_level":0.95}},"effective":{"n":8,"acceptable":5,"unsafe":0,"review":3,"automated_count":5,"automated_acceptable":5,"automated_unsafe":0,"unnecessary_reviews":3,"accuracy":0.625,"unsafe_rate":0.0,"review_rate":0.375,"automated_coverage":0.625,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.375,"accuracy_wilson_95":{"lower":0.30574239460262737,"upper":0.8631557141764026,"confidence_level":0.95}}},"drop":{"n":22,"raw":{"n":22,"acceptable":14,"unsafe":8,"review":0,"automated_count":22,"automated_acceptable":14,"automated_unsafe":8,"unnecessary_reviews":0,"accuracy":0.6363636363636364,"unsafe_rate":0.36363636363636365,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.6363636363636364,"conditional_unsafe":0.36363636363636365,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.42951708090131013,"upper":0.8026679532963387,"confidence_level":0.95}},"effective":{"n":22,"acceptable":14,"unsafe":0,"review":8,"automated_count":14,"automated_acceptable":14,"automated_unsafe":0,"unnecessary_reviews":8,"accuracy":0.6363636363636364,"unsafe_rate":0.0,"review_rate":0.36363636363636365,"automated_coverage":0.6363636363636364,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.36363636363636365,"accuracy_wilson_95":{"lower":0.42951708090131013,"upper":0.8026679532963387,"confidence_level":0.95}}},"mask":{"n":8,"raw":{"n":8,"acceptable":8,"unsafe":0,"review":0,"automated_count":8,"automated_acceptable":8,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":1.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.6755924351161198,"upper":1.0,"confidence_level":0.95}},"effective":{"n":8,"acceptable":8,"unsafe":0,"review":0,"automated_count":8,"automated_acceptable":8,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":1.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.6755924351161198,"upper":1.0,"confidence_level":0.95}}},"pseudonymize":{"n":8,"raw":{"n":8,"acceptable":6,"unsafe":2,"review":0,"automated_count":8,"automated_acceptable":6,"automated_unsafe":2,"unnecessary_reviews":0,"accuracy":0.75,"unsafe_rate":0.25,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.75,"conditional_unsafe":0.25,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.40927543031016883,"upper":0.9285207872478909,"confidence_level":0.95}},"effective":{"n":8,"acceptable":0,"unsafe":0,"review":8,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":8,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.32440756488388023,"confidence_level":0.95}}},"generalize":{"n":8,"raw":{"n":8,"acceptable":5,"unsafe":3,"review":0,"automated_count":8,"automated_acceptable":5,"automated_unsafe":3,"unnecessary_reviews":0,"accuracy":0.625,"unsafe_rate":0.375,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.625,"conditional_unsafe":0.375,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.30574239460262737,"upper":0.8631557141764026,"confidence_level":0.95}},"effective":{"n":8,"acceptable":5,"unsafe":0,"review":3,"automated_count":5,"automated_acceptable":5,"automated_unsafe":0,"unnecessary_reviews":3,"accuracy":0.625,"unsafe_rate":0.0,"review_rate":0.375,"automated_coverage":0.625,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.375,"accuracy_wilson_95":{"lower":0.30574239460262737,"upper":0.8631557141764026,"confidence_level":0.95}}},"review":{"n":2,"raw":{"n":2,"acceptable":0,"unsafe":2,"review":0,"automated_count":2,"automated_acceptable":0,"automated_unsafe":2,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":1.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":1.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.6576197724933469,"confidence_level":0.95}},"effective":{"n":2,"acceptable":2,"unsafe":0,"review":2,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":1.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.34238022750665303,"upper":1.0,"confidence_level":0.95}}}}}},"composite":{"n":198,"scope":"descriptive sum across four correlated panels","raw":{"n":198,"acceptable":146,"unsafe":23,"review":0,"automated_count":198,"automated_acceptable":146,"automated_unsafe":23,"unnecessary_reviews":0,"accuracy":0.7373737373737373,"unsafe_rate":0.11616161616161616,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.7373737373737373,"conditional_unsafe":0.11616161616161616,"unnecessary_review_rate":0.0},"effective":{"n":198,"acceptable":128,"unsafe":1,"review":47,"automated_count":151,"automated_acceptable":123,"automated_unsafe":1,"unnecessary_reviews":42,"accuracy":0.6464646464646465,"unsafe_rate":0.005050505050505051,"review_rate":0.23737373737373738,"automated_coverage":0.7626262626262627,"conditional_correctness":0.8145695364238411,"conditional_unsafe":0.006622516556291391,"unnecessary_review_rate":0.21212121212121213}}},{"id":"Mapika--decider-4b-nf4","platform":"Local","model_id":"Mapika/decider-4b","revision":"eb5fbdfc9448473ec25e399882912863afbdb70e","loading":"bfloat16 / nf4","scoring_kind":"Native decision readout","settings":{"dtype":"bfloat16","quantization":{"method":"bitsandbytes_nf4","bits":4,"double_quant":true,"compute_dtype":"bfloat16","weight_storage":"uint8"},"max_tokens":4096,"effective_max_tokens":4096,"native_temperature":1.099,"trust_remote_code":false,"local_files_only":true,"use_safetensors":true,"guard_threshold":0.45,"seed":0,"tf32":false},"scoring_metadata":{"backend":"decider","protocol":"hf-local-sweep-backends-v1","probability_source":"native_choice_probability_unrounded","latency_scope":"synchronous_forward_scoring_including_transfer_excluding_tokenization","precision":"bfloat16","equal_mass_convention":"rank bins with floor boundaries at floor(i*n/k), k=min(10,n//2); minimum two rows"},"load_ms":7623.638966993894,"peak_allocated_bytes":3269912576,"peak_reserved_bytes":3336568832,"panels":{"diagnostic43-no_samples":{"n":43,"raw":{"n":43,"acceptable":35,"unsafe":0,"review":0,"automated_count":43,"automated_acceptable":35,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":0.813953488372093,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.813953488372093,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.6738299634272562,"upper":0.9025824857100377,"confidence_level":0.95}},"effective":{"n":43,"acceptable":33,"unsafe":0,"review":2,"automated_count":41,"automated_acceptable":33,"automated_unsafe":0,"unnecessary_reviews":2,"accuracy":0.7674418604651163,"unsafe_rate":0.0,"review_rate":0.046511627906976744,"automated_coverage":0.9534883720930233,"conditional_correctness":0.8048780487804879,"conditional_unsafe":0.0,"unnecessary_review_rate":0.046511627906976744,"accuracy_wilson_95":{"lower":0.6225547970219101,"upper":0.8684632152061551,"confidence_level":0.95}},"confusion":{"raw":{"keep":{"keep":5,"drop":7,"mask":0,"pseudonymize":0,"generalize":0,"review":0},"drop":{"keep":0,"drop":15,"mask":1,"pseudonymize":0,"generalize":0,"review":0},"mask":{"keep":0,"drop":0,"mask":2,"pseudonymize":0,"generalize":0,"review":0},"pseudonymize":{"keep":0,"drop":0,"mask":0,"pseudonymize":2,"generalize":0,"review":0},"generalize":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":6,"review":0},"review":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0}},"effective":{"keep":{"keep":4,"drop":7,"mask":0,"pseudonymize":0,"generalize":0,"review":1},"drop":{"keep":0,"drop":15,"mask":1,"pseudonymize":0,"generalize":0,"review":0},"mask":{"keep":0,"drop":0,"mask":2,"pseudonymize":0,"generalize":0,"review":0},"pseudonymize":{"keep":0,"drop":0,"mask":0,"pseudonymize":2,"generalize":0,"review":0},"generalize":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":5,"review":1},"review":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0}}},"probability_metrics":{"singleton_n":38,"multiacceptable_n":5,"multiacceptable_coverage":0.11627906976744186,"nll":0.7642354374418084,"nll_zero_mass_n":0,"brier":0.3403674084280044,"acceptable_set_n":43,"acceptable_set_nll":0.6754702141794634,"acceptable_set_zero_mass_n":0},"calibration_all_labels":{"equal_width":{"n":43,"bin_count":10,"ece":0.15751942229825397,"signed_mean_gap":0.09698781994886177,"absolute_mean_gap":0.09698781994886177,"bins":[{"index":0,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.0,"upper":0.1,"upper_inclusive":false},{"index":1,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.1,"upper":0.2,"upper_inclusive":false},{"index":2,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.2,"upper":0.3,"upper_inclusive":false},{"index":3,"count":1,"confidence":0.3039882183074951,"empirical_correctness":1.0,"signed_gap":-0.6960117816925049,"absolute_gap":0.6960117816925049,"lower":0.3,"upper":0.4,"upper_inclusive":false},{"index":4,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.4,"upper":0.5,"upper_inclusive":false},{"index":5,"count":2,"confidence":0.5395779609680176,"empirical_correctness":0.5,"signed_gap":0.03957796096801758,"absolute_gap":0.03957796096801758,"lower":0.5,"upper":0.6,"upper_inclusive":false},{"index":6,"count":2,"confidence":0.616543173789978,"empirical_correctness":0.0,"signed_gap":0.616543173789978,"absolute_gap":0.616543173789978,"lower":0.6,"upper":0.7,"upper_inclusive":false},{"index":7,"count":2,"confidence":0.7673878073692322,"empirical_correctness":0.5,"signed_gap":0.2673878073692322,"absolute_gap":0.2673878073692322,"lower":0.7,"upper":0.8,"upper_inclusive":false},{"index":8,"count":4,"confidence":0.8486455827951431,"empirical_correctness":1.0,"signed_gap":-0.15135441720485687,"absolute_gap":0.15135441720485687,"lower":0.8,"upper":0.9,"upper_inclusive":false},{"index":9,"count":32,"confidence":0.9882777445018291,"empirical_correctness":0.875,"signed_gap":0.11327774450182915,"absolute_gap":0.11327774450182915,"lower":0.9,"upper":1.0,"upper_inclusive":true}]},"equal_mass":{"n":43,"bin_count":10,"ece":0.12258449404738668,"signed_mean_gap":0.09698781994886177,"absolute_mean_gap":0.09698781994886177,"bins":[{"index":0,"count":4,"confidence":0.4972091168165207,"empirical_correctness":0.5,"signed_gap":-0.002790883183479309,"absolute_gap":0.002790883183479309,"min_confidence":0.3039882183074951,"max_confidence":0.6056923270225525},{"index":1,"count":4,"confidence":0.7442334890365601,"empirical_correctness":0.5,"signed_gap":0.24423348903656006,"absolute_gap":0.24423348903656006,"min_confidence":0.6273940205574036,"max_confidence":0.8147643208503723},{"index":2,"count":4,"confidence":0.8749253004789352,"empirical_correctness":1.0,"signed_gap":-0.12507469952106476,"absolute_gap":0.12507469952106476,"min_confidence":0.8206768035888672,"max_confidence":0.9198831915855408},{"index":3,"count":5,"confidence":0.9672110438346863,"empirical_correctness":0.8,"signed_gap":0.1672110438346862,"absolute_gap":0.1672110438346862,"min_confidence":0.9504697918891907,"max_confidence":0.9739570021629333},{"index":4,"count":4,"confidence":0.979610487818718,"empirical_correctness":0.5,"signed_gap":0.47961048781871796,"absolute_gap":0.47961048781871796,"min_confidence":0.9742864370346069,"max_confidence":0.9874216318130493},{"index":5,"count":4,"confidence":0.9907324612140656,"empirical_correctness":1.0,"signed_gap":-0.009267538785934448,"absolute_gap":0.009267538785934448,"min_confidence":0.9880768060684204,"max_confidence":0.9927922487258911},{"index":6,"count":5,"confidence":0.9978747248649598,"empirical_correctness":0.8,"signed_gap":0.19787472486495972,"absolute_gap":0.19787472486495972,"min_confidence":0.994785487651825,"max_confidence":0.9993969202041626},{"index":7,"count":4,"confidence":0.9997822642326355,"empirical_correctness":1.0,"signed_gap":-0.00021773576736450195,"absolute_gap":0.00021773576736450195,"min_confidence":0.999754011631012,"max_confidence":0.9998206496238708},{"index":8,"count":4,"confidence":0.9998562932014465,"empirical_correctness":1.0,"signed_gap":-0.0001437067985534668,"absolute_gap":0.0001437067985534668,"min_confidence":0.9998321533203125,"max_confidence":0.9998944997787476},{"index":9,"count":5,"confidence":0.99992995262146,"empirical_correctness":1.0,"signed_gap":-7.004737853999465e-05,"absolute_gap":7.004737853999465e-05,"min_confidence":0.9998996257781982,"max_confidence":0.9999477863311768}]}},"calibration_singleton_labels":{"equal_width":{"n":38,"bin_count":10,"ece":0.17664249790342235,"signed_mean_gap":0.1139167217831863,"absolute_mean_gap":0.1139167217831863,"bins":[{"index":0,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.0,"upper":0.1,"upper_inclusive":false},{"index":1,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.1,"upper":0.2,"upper_inclusive":false},{"index":2,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.2,"upper":0.3,"upper_inclusive":false},{"index":3,"count":1,"confidence":0.3039882183074951,"empirical_correctness":1.0,"signed_gap":-0.6960117816925049,"absolute_gap":0.6960117816925049,"lower":0.3,"upper":0.4,"upper_inclusive":false},{"index":4,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.4,"upper":0.5,"upper_inclusive":false},{"index":5,"count":2,"confidence":0.5395779609680176,"empirical_correctness":0.5,"signed_gap":0.03957796096801758,"absolute_gap":0.03957796096801758,"lower":0.5,"upper":0.6,"upper_inclusive":false},{"index":6,"count":2,"confidence":0.616543173789978,"empirical_correctness":0.0,"signed_gap":0.616543173789978,"absolute_gap":0.616543173789978,"lower":0.6,"upper":0.7,"upper_inclusive":false},{"index":7,"count":2,"confidence":0.7673878073692322,"empirical_correctness":0.5,"signed_gap":0.2673878073692322,"absolute_gap":0.2673878073692322,"lower":0.7,"upper":0.8,"upper_inclusive":false},{"index":8,"count":3,"confidence":0.83474067846934,"empirical_correctness":1.0,"signed_gap":-0.16525932153066003,"absolute_gap":0.16525932153066003,"lower":0.8,"upper":0.9,"upper_inclusive":false},{"index":9,"count":28,"confidence":0.9883431174925396,"empirical_correctness":0.8571428571428571,"signed_gap":0.13120026034968246,"absolute_gap":0.13120026034968246,"lower":0.9,"upper":1.0,"upper_inclusive":true}]},"equal_mass":{"n":38,"bin_count":10,"ece":0.17698405761467784,"signed_mean_gap":0.11391672178318626,"absolute_mean_gap":0.11391672178318626,"bins":[{"index":0,"count":3,"confidence":0.46104804674784344,"empirical_correctness":0.6666666666666666,"signed_gap":-0.2056186199188232,"absolute_gap":0.2056186199188232,"min_confidence":0.3039882183074951,"max_confidence":0.5583910346031189},{"index":1,"count":4,"confidence":0.6919654905796051,"empirical_correctness":0.25,"signed_gap":0.4419654905796051,"absolute_gap":0.4419654905796051,"min_confidence":0.6056923270225525,"max_confidence":0.7977376580238342},{"index":2,"count":4,"confidence":0.8560263067483902,"empirical_correctness":1.0,"signed_gap":-0.1439736932516098,"absolute_gap":0.1439736932516098,"min_confidence":0.8147643208503723,"max_confidence":0.9198831915855408},{"index":3,"count":4,"confidence":0.9655245542526245,"empirical_correctness":0.75,"signed_gap":0.2155245542526245,"absolute_gap":0.2155245542526245,"min_confidence":0.9504697918891907,"max_confidence":0.9725958108901978},{"index":4,"count":4,"confidence":0.9797742813825607,"empirical_correctness":0.5,"signed_gap":0.47977428138256073,"absolute_gap":0.47977428138256073,"min_confidence":0.9742864370346069,"max_confidence":0.9880768060684204},{"index":5,"count":3,"confidence":0.9926858345667521,"empirical_correctness":0.6666666666666666,"signed_gap":0.32601916790008545,"absolute_gap":0.32601916790008545,"min_confidence":0.9904797673225403,"max_confidence":0.994785487651825},{"index":6,"count":4,"confidence":0.9990053176879883,"empirical_correctness":1.0,"signed_gap":-0.0009946823120117188,"absolute_gap":0.0009946823120117188,"min_confidence":0.9978598952293396,"max_confidence":0.999754011631012},{"index":7,"count":4,"confidence":0.9998017996549606,"empirical_correctness":1.0,"signed_gap":-0.00019820034503936768,"absolute_gap":0.00019820034503936768,"min_confidence":0.9997648596763611,"max_confidence":0.9998321533203125},{"index":8,"count":4,"confidence":0.999873161315918,"empirical_correctness":1.0,"signed_gap":-0.00012683868408203125,"absolute_gap":0.00012683868408203125,"min_confidence":0.9998397827148438,"max_confidence":0.9998996257781982},{"index":9,"count":4,"confidence":0.9999375343322754,"empirical_correctness":1.0,"signed_gap":-6.246566772460938e-05,"absolute_gap":6.246566772460938e-05,"min_confidence":0.999927282333374,"max_confidence":0.9999477863311768}]}},"latency_ms":{"n":43,"median":97.37587097333744,"p95":100.48148801433854},"per_action":{"keep":{"n":12,"raw":{"n":12,"acceptable":5,"unsafe":0,"review":0,"automated_count":12,"automated_acceptable":5,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":0.4166666666666667,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.4166666666666667,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.19326031365875665,"upper":0.6804886874504503,"confidence_level":0.95}},"effective":{"n":12,"acceptable":4,"unsafe":0,"review":1,"automated_count":11,"automated_acceptable":4,"automated_unsafe":0,"unnecessary_reviews":1,"accuracy":0.3333333333333333,"unsafe_rate":0.0,"review_rate":0.08333333333333333,"automated_coverage":0.9166666666666666,"conditional_correctness":0.36363636363636365,"conditional_unsafe":0.0,"unnecessary_review_rate":0.08333333333333333,"accuracy_wilson_95":{"lower":0.13812009109121315,"upper":0.6093779111272004,"confidence_level":0.95}}},"drop":{"n":16,"raw":{"n":16,"acceptable":15,"unsafe":0,"review":0,"automated_count":16,"automated_acceptable":15,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":0.9375,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.9375,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.7167126242970107,"upper":0.9888806552353576,"confidence_level":0.95}},"effective":{"n":16,"acceptable":15,"unsafe":0,"review":0,"automated_count":16,"automated_acceptable":15,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":0.9375,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.9375,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.7167126242970107,"upper":0.9888806552353576,"confidence_level":0.95}}},"mask":{"n":2,"raw":{"n":2,"acceptable":2,"unsafe":0,"review":0,"automated_count":2,"automated_acceptable":2,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":1.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.34238022750665303,"upper":1.0,"confidence_level":0.95}},"effective":{"n":2,"acceptable":2,"unsafe":0,"review":0,"automated_count":2,"automated_acceptable":2,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":1.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.34238022750665303,"upper":1.0,"confidence_level":0.95}}},"pseudonymize":{"n":2,"raw":{"n":2,"acceptable":2,"unsafe":0,"review":0,"automated_count":2,"automated_acceptable":2,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":1.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.34238022750665303,"upper":1.0,"confidence_level":0.95}},"effective":{"n":2,"acceptable":2,"unsafe":0,"review":0,"automated_count":2,"automated_acceptable":2,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":1.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.34238022750665303,"upper":1.0,"confidence_level":0.95}}},"generalize":{"n":6,"raw":{"n":6,"acceptable":6,"unsafe":0,"review":0,"automated_count":6,"automated_acceptable":6,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":1.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.6096657120978346,"upper":1.0,"confidence_level":0.95}},"effective":{"n":6,"acceptable":5,"unsafe":0,"review":1,"automated_count":5,"automated_acceptable":5,"automated_unsafe":0,"unnecessary_reviews":1,"accuracy":0.8333333333333334,"unsafe_rate":0.0,"review_rate":0.16666666666666666,"automated_coverage":0.8333333333333334,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.16666666666666666,"accuracy_wilson_95":{"lower":0.43649717781352987,"upper":0.9699466302516935,"confidence_level":0.95}}},"review":{"n":0,"raw":{"n":0,"acceptable":0,"unsafe":0,"review":0,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":null,"unsafe_rate":null,"review_rate":null,"automated_coverage":null,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":null},"effective":{"n":0,"acceptable":0,"unsafe":0,"review":0,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":null,"unsafe_rate":null,"review_rate":null,"automated_coverage":null,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":null}}}},"diagnostic43-compact_no_samples":{"n":43,"raw":{"n":43,"acceptable":36,"unsafe":0,"review":0,"automated_count":43,"automated_acceptable":36,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":0.8372093023255814,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.8372093023255814,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.7002759410625283,"upper":0.91883372652938,"confidence_level":0.95}},"effective":{"n":43,"acceptable":35,"unsafe":0,"review":1,"automated_count":42,"automated_acceptable":35,"automated_unsafe":0,"unnecessary_reviews":1,"accuracy":0.813953488372093,"unsafe_rate":0.0,"review_rate":0.023255813953488372,"automated_coverage":0.9767441860465116,"conditional_correctness":0.8333333333333334,"conditional_unsafe":0.0,"unnecessary_review_rate":0.023255813953488372,"accuracy_wilson_95":{"lower":0.6738299634272562,"upper":0.9025824857100377,"confidence_level":0.95}},"confusion":{"raw":{"keep":{"keep":5,"drop":7,"mask":0,"pseudonymize":0,"generalize":0,"review":0},"drop":{"keep":0,"drop":16,"mask":0,"pseudonymize":0,"generalize":0,"review":0},"mask":{"keep":0,"drop":0,"mask":2,"pseudonymize":0,"generalize":0,"review":0},"pseudonymize":{"keep":0,"drop":0,"mask":0,"pseudonymize":2,"generalize":0,"review":0},"generalize":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":6,"review":0},"review":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0}},"effective":{"keep":{"keep":4,"drop":7,"mask":0,"pseudonymize":0,"generalize":0,"review":1},"drop":{"keep":0,"drop":16,"mask":0,"pseudonymize":0,"generalize":0,"review":0},"mask":{"keep":0,"drop":0,"mask":2,"pseudonymize":0,"generalize":0,"review":0},"pseudonymize":{"keep":0,"drop":0,"mask":0,"pseudonymize":2,"generalize":0,"review":0},"generalize":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":6,"review":0},"review":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0}}},"probability_metrics":{"singleton_n":38,"multiacceptable_n":5,"multiacceptable_coverage":0.11627906976744186,"nll":0.8152448191007535,"nll_zero_mass_n":0,"brier":0.3124299617687159,"acceptable_set_n":43,"acceptable_set_nll":0.7207853351677066,"acceptable_set_zero_mass_n":0},"calibration_all_labels":{"equal_width":{"n":43,"bin_count":10,"ece":0.15439320580903873,"signed_mean_gap":0.07672083239222682,"absolute_mean_gap":0.07672083239222682,"bins":[{"index":0,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.0,"upper":0.1,"upper_inclusive":false},{"index":1,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.1,"upper":0.2,"upper_inclusive":false},{"index":2,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.2,"upper":0.3,"upper_inclusive":false},{"index":3,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.3,"upper":0.4,"upper_inclusive":false},{"index":4,"count":2,"confidence":0.45410317182540894,"empirical_correctness":1.0,"signed_gap":-0.5458968281745911,"absolute_gap":0.5458968281745911,"lower":0.4,"upper":0.5,"upper_inclusive":false},{"index":5,"count":1,"confidence":0.5411762595176697,"empirical_correctness":0.0,"signed_gap":0.5411762595176697,"absolute_gap":0.5411762595176697,"lower":0.5,"upper":0.6,"upper_inclusive":false},{"index":6,"count":1,"confidence":0.6695094704627991,"empirical_correctness":0.0,"signed_gap":0.6695094704627991,"absolute_gap":0.6695094704627991,"lower":0.6,"upper":0.7,"upper_inclusive":false},{"index":7,"count":3,"confidence":0.7601130207379659,"empirical_correctness":0.6666666666666666,"signed_gap":0.09344635407129931,"absolute_gap":0.09344635407129931,"lower":0.7,"upper":0.8,"upper_inclusive":false},{"index":8,"count":4,"confidence":0.8554594069719315,"empirical_correctness":1.0,"signed_gap":-0.14454059302806854,"absolute_gap":0.14454059302806854,"lower":0.8,"upper":0.9,"upper_inclusive":false},{"index":9,"count":32,"confidence":0.9836852196604013,"empirical_correctness":0.875,"signed_gap":0.10868521966040134,"absolute_gap":0.10868521966040134,"lower":0.9,"upper":1.0,"upper_inclusive":true}]},"equal_mass":{"n":43,"bin_count":10,"ece":0.10727936861126922,"signed_mean_gap":0.07672083239222682,"absolute_mean_gap":0.07672083239222682,"bins":[{"index":0,"count":4,"confidence":0.5297230184078217,"empirical_correctness":0.5,"signed_gap":0.029723018407821655,"absolute_gap":0.029723018407821655,"min_confidence":0.4449760317802429,"max_confidence":0.6695094704627991},{"index":1,"count":4,"confidence":0.7786205112934113,"empirical_correctness":0.75,"signed_gap":0.028620511293411255,"absolute_gap":0.028620511293411255,"min_confidence":0.7146139740943909,"max_confidence":0.8341429829597473},{"index":2,"count":4,"confidence":0.8750410974025726,"empirical_correctness":1.0,"signed_gap":-0.12495890259742737,"absolute_gap":0.12495890259742737,"min_confidence":0.8393296599388123,"max_confidence":0.912469744682312},{"index":3,"count":5,"confidence":0.9466462969779968,"empirical_correctness":0.6,"signed_gap":0.34664629697799687,"absolute_gap":0.34664629697799687,"min_confidence":0.9387103915214539,"max_confidence":0.9553647637367249},{"index":4,"count":4,"confidence":0.973565086722374,"empirical_correctness":1.0,"signed_gap":-0.026434913277626038,"absolute_gap":0.026434913277626038,"min_confidence":0.9674431085586548,"max_confidence":0.981613278388977},{"index":5,"count":4,"confidence":0.9880080372095108,"empirical_correctness":1.0,"signed_gap":-0.011991962790489197,"absolute_gap":0.011991962790489197,"min_confidence":0.9827321767807007,"max_confidence":0.9919970631599426},{"index":6,"count":5,"confidence":0.9978797435760498,"empirical_correctness":0.6,"signed_gap":0.3978797435760498,"absolute_gap":0.3978797435760498,"min_confidence":0.9955642223358154,"max_confidence":0.999019980430603},{"index":7,"count":4,"confidence":0.9995671808719635,"empirical_correctness":1.0,"signed_gap":-0.000432819128036499,"absolute_gap":0.000432819128036499,"min_confidence":0.9993454813957214,"max_confidence":0.9996854066848755},{"index":8,"count":4,"confidence":0.9997562170028687,"empirical_correctness":1.0,"signed_gap":-0.00024378299713134766,"absolute_gap":0.00024378299713134766,"min_confidence":0.9997183680534363,"max_confidence":0.9997712969779968},{"index":9,"count":5,"confidence":0.999848198890686,"empirical_correctness":1.0,"signed_gap":-0.00015180110931400925,"absolute_gap":0.00015180110931400925,"min_confidence":0.9998133778572083,"max_confidence":0.9998899698257446}]}},"calibration_singleton_labels":{"equal_width":{"n":38,"bin_count":10,"ece":0.1752428083043349,"signed_mean_gap":0.10137948707530368,"absolute_mean_gap":0.10137948707530368,"bins":[{"index":0,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.0,"upper":0.1,"upper_inclusive":false},{"index":1,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.1,"upper":0.2,"upper_inclusive":false},{"index":2,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.2,"upper":0.3,"upper_inclusive":false},{"index":3,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.3,"upper":0.4,"upper_inclusive":false},{"index":4,"count":2,"confidence":0.45410317182540894,"empirical_correctness":1.0,"signed_gap":-0.5458968281745911,"absolute_gap":0.5458968281745911,"lower":0.4,"upper":0.5,"upper_inclusive":false},{"index":5,"count":1,"confidence":0.5411762595176697,"empirical_correctness":0.0,"signed_gap":0.5411762595176697,"absolute_gap":0.5411762595176697,"lower":0.5,"upper":0.6,"upper_inclusive":false},{"index":6,"count":1,"confidence":0.6695094704627991,"empirical_correctness":0.0,"signed_gap":0.6695094704627991,"absolute_gap":0.6695094704627991,"lower":0.6,"upper":0.7,"upper_inclusive":false},{"index":7,"count":2,"confidence":0.7450884580612183,"empirical_correctness":0.5,"signed_gap":0.24508845806121826,"absolute_gap":0.24508845806121826,"lower":0.7,"upper":0.8,"upper_inclusive":false},{"index":8,"count":2,"confidence":0.8441952764987946,"empirical_correctness":1.0,"signed_gap":-0.15580472350120544,"absolute_gap":0.15580472350120544,"lower":0.8,"upper":0.9,"upper_inclusive":false},{"index":9,"count":30,"confidence":0.9851653655370076,"empirical_correctness":0.8666666666666667,"signed_gap":0.11849869887034092,"absolute_gap":0.11849869887034092,"lower":0.9,"upper":1.0,"upper_inclusive":true}]},"equal_mass":{"n":38,"bin_count":10,"ece":0.13357843066516675,"signed_mean_gap":0.10137948707530373,"absolute_mean_gap":0.10137948707530373,"bins":[{"index":0,"count":3,"confidence":0.48312753438949585,"empirical_correctness":0.6666666666666666,"signed_gap":-0.18353913227717078,"absolute_gap":0.18353913227717078,"min_confidence":0.4449760317802429,"max_confidence":0.5411762595176697},{"index":1,"count":4,"confidence":0.7484573423862457,"empirical_correctness":0.5,"signed_gap":0.24845734238624573,"absolute_gap":0.24845734238624573,"min_confidence":0.6695094704627991,"max_confidence":0.8341429829597473},{"index":2,"count":4,"confidence":0.9119456261396408,"empirical_correctness":0.75,"signed_gap":0.1619456261396408,"absolute_gap":0.1619456261396408,"min_confidence":0.8542475700378418,"max_confidence":0.9423547983169556},{"index":3,"count":4,"confidence":0.96046182513237,"empirical_correctness":0.75,"signed_gap":0.21046182513237,"absolute_gap":0.21046182513237,"min_confidence":0.9433321952819824,"max_confidence":0.9776026606559753},{"index":4,"count":4,"confidence":0.9854120910167694,"empirical_correctness":1.0,"signed_gap":-0.01458790898323059,"absolute_gap":0.01458790898323059,"min_confidence":0.981613278388977,"max_confidence":0.9905819892883301},{"index":5,"count":3,"confidence":0.9950362841288248,"empirical_correctness":0.6666666666666666,"signed_gap":0.3283696174621582,"absolute_gap":0.3283696174621582,"min_confidence":0.9919970631599426,"max_confidence":0.9975475668907166},{"index":6,"count":4,"confidence":0.9989081025123596,"empirical_correctness":0.75,"signed_gap":0.24890810251235962,"absolute_gap":0.24890810251235962,"min_confidence":0.9985758066177368,"max_confidence":0.9993454813957214},{"index":7,"count":4,"confidence":0.9996604025363922,"empirical_correctness":1.0,"signed_gap":-0.0003395974636077881,"absolute_gap":0.0003395974636077881,"min_confidence":0.9995872378349304,"max_confidence":0.9997183680534363},{"index":8,"count":4,"confidence":0.9997799694538116,"empirical_correctness":1.0,"signed_gap":-0.0002200305461883545,"absolute_gap":0.0002200305461883545,"min_confidence":0.9997642636299133,"max_confidence":0.9998133778572083},{"index":9,"count":4,"confidence":0.9998569041490555,"empirical_correctness":1.0,"signed_gap":-0.00014309585094451904,"absolute_gap":0.00014309585094451904,"min_confidence":0.9998254179954529,"max_confidence":0.9998899698257446}]}},"latency_ms":{"n":43,"median":97.75348997209221,"p95":102.64221495599486},"per_action":{"keep":{"n":12,"raw":{"n":12,"acceptable":5,"unsafe":0,"review":0,"automated_count":12,"automated_acceptable":5,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":0.4166666666666667,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.4166666666666667,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.19326031365875665,"upper":0.6804886874504503,"confidence_level":0.95}},"effective":{"n":12,"acceptable":4,"unsafe":0,"review":1,"automated_count":11,"automated_acceptable":4,"automated_unsafe":0,"unnecessary_reviews":1,"accuracy":0.3333333333333333,"unsafe_rate":0.0,"review_rate":0.08333333333333333,"automated_coverage":0.9166666666666666,"conditional_correctness":0.36363636363636365,"conditional_unsafe":0.0,"unnecessary_review_rate":0.08333333333333333,"accuracy_wilson_95":{"lower":0.13812009109121315,"upper":0.6093779111272004,"confidence_level":0.95}}},"drop":{"n":16,"raw":{"n":16,"acceptable":16,"unsafe":0,"review":0,"automated_count":16,"automated_acceptable":16,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":1.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.8063923194655637,"upper":1.0,"confidence_level":0.95}},"effective":{"n":16,"acceptable":16,"unsafe":0,"review":0,"automated_count":16,"automated_acceptable":16,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":1.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.8063923194655637,"upper":1.0,"confidence_level":0.95}}},"mask":{"n":2,"raw":{"n":2,"acceptable":2,"unsafe":0,"review":0,"automated_count":2,"automated_acceptable":2,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":1.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.34238022750665303,"upper":1.0,"confidence_level":0.95}},"effective":{"n":2,"acceptable":2,"unsafe":0,"review":0,"automated_count":2,"automated_acceptable":2,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":1.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.34238022750665303,"upper":1.0,"confidence_level":0.95}}},"pseudonymize":{"n":2,"raw":{"n":2,"acceptable":2,"unsafe":0,"review":0,"automated_count":2,"automated_acceptable":2,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":1.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.34238022750665303,"upper":1.0,"confidence_level":0.95}},"effective":{"n":2,"acceptable":2,"unsafe":0,"review":0,"automated_count":2,"automated_acceptable":2,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":1.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.34238022750665303,"upper":1.0,"confidence_level":0.95}}},"generalize":{"n":6,"raw":{"n":6,"acceptable":6,"unsafe":0,"review":0,"automated_count":6,"automated_acceptable":6,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":1.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.6096657120978346,"upper":1.0,"confidence_level":0.95}},"effective":{"n":6,"acceptable":6,"unsafe":0,"review":0,"automated_count":6,"automated_acceptable":6,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":1.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.6096657120978346,"upper":1.0,"confidence_level":0.95}}},"review":{"n":0,"raw":{"n":0,"acceptable":0,"unsafe":0,"review":0,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":null,"unsafe_rate":null,"review_rate":null,"automated_coverage":null,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":null},"effective":{"n":0,"acceptable":0,"unsafe":0,"review":0,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":null,"unsafe_rate":null,"review_rate":null,"automated_coverage":null,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":null}}}},"challenge56-no_samples":{"n":56,"raw":{"n":56,"acceptable":54,"unsafe":0,"review":0,"automated_count":56,"automated_acceptable":54,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":0.9642857142857143,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.9642857142857143,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.8788121981950989,"upper":0.9901505748517636,"confidence_level":0.95}},"effective":{"n":56,"acceptable":44,"unsafe":0,"review":10,"automated_count":46,"automated_acceptable":44,"automated_unsafe":0,"unnecessary_reviews":10,"accuracy":0.7857142857142857,"unsafe_rate":0.0,"review_rate":0.17857142857142858,"automated_coverage":0.8214285714285714,"conditional_correctness":0.9565217391304348,"conditional_unsafe":0.0,"unnecessary_review_rate":0.17857142857142858,"accuracy_wilson_95":{"lower":0.6618053643659488,"upper":0.8729409575090435,"confidence_level":0.95}},"confusion":{"raw":{"keep":{"keep":8,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0},"drop":{"keep":0,"drop":22,"mask":0,"pseudonymize":0,"generalize":0,"review":0},"mask":{"keep":0,"drop":0,"mask":8,"pseudonymize":0,"generalize":0,"review":0},"pseudonymize":{"keep":0,"drop":0,"mask":0,"pseudonymize":8,"generalize":0,"review":0},"generalize":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":8,"review":0},"review":{"keep":0,"drop":0,"mask":2,"pseudonymize":0,"generalize":0,"review":0}},"effective":{"keep":{"keep":5,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":3},"drop":{"keep":0,"drop":22,"mask":0,"pseudonymize":0,"generalize":0,"review":0},"mask":{"keep":0,"drop":0,"mask":8,"pseudonymize":0,"generalize":0,"review":0},"pseudonymize":{"keep":0,"drop":0,"mask":0,"pseudonymize":1,"generalize":0,"review":7},"generalize":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":8,"review":0},"review":{"keep":0,"drop":0,"mask":2,"pseudonymize":0,"generalize":0,"review":0}}},"probability_metrics":{"singleton_n":56,"multiacceptable_n":0,"multiacceptable_coverage":0.0,"nll":0.09510843953031559,"nll_zero_mass_n":0,"brier":0.04482784968272453,"acceptable_set_n":56,"acceptable_set_nll":0.09510843953031559,"acceptable_set_zero_mass_n":0},"calibration_all_labels":{"equal_width":{"n":56,"bin_count":10,"ece":0.04251080857855929,"signed_mean_gap":0.002897159329482527,"absolute_mean_gap":0.002897159329482527,"bins":[{"index":0,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.0,"upper":0.1,"upper_inclusive":false},{"index":1,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.1,"upper":0.2,"upper_inclusive":false},{"index":2,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.2,"upper":0.3,"upper_inclusive":false},{"index":3,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.3,"upper":0.4,"upper_inclusive":false},{"index":4,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.4,"upper":0.5,"upper_inclusive":false},{"index":5,"count":1,"confidence":0.5991616249084473,"empirical_correctness":0.0,"signed_gap":0.5991616249084473,"absolute_gap":0.5991616249084473,"lower":0.5,"upper":0.6,"upper_inclusive":false},{"index":6,"count":1,"confidence":0.6722614765167236,"empirical_correctness":0.0,"signed_gap":0.6722614765167236,"absolute_gap":0.6722614765167236,"lower":0.6,"upper":0.7,"upper_inclusive":false},{"index":7,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.7,"upper":0.8,"upper_inclusive":false},{"index":8,"count":2,"confidence":0.8567938804626465,"empirical_correctness":1.0,"signed_gap":-0.14320611953735352,"absolute_gap":0.14320611953735352,"lower":0.8,"upper":0.9,"upper_inclusive":false},{"index":9,"count":52,"confidence":0.98417750115578,"empirical_correctness":1.0,"signed_gap":-0.015822498844220045,"absolute_gap":0.015822498844220045,"lower":0.9,"upper":1.0,"upper_inclusive":true}]},"equal_mass":{"n":56,"bin_count":10,"ece":0.029460065066814423,"signed_mean_gap":0.002897159329482497,"absolute_mean_gap":0.002897159329482497,"bins":[{"index":0,"count":5,"confidence":0.7812004566192627,"empirical_correctness":0.6,"signed_gap":0.18120045661926276,"absolute_gap":0.18120045661926276,"min_confidence":0.5991616249084473,"max_confidence":0.9209914207458496},{"index":1,"count":6,"confidence":0.9468954006830851,"empirical_correctness":1.0,"signed_gap":-0.05310459931691491,"absolute_gap":0.05310459931691491,"min_confidence":0.93880695104599,"max_confidence":0.9556841254234314},{"index":2,"count":5,"confidence":0.9661910176277161,"empirical_correctness":1.0,"signed_gap":-0.03380898237228391,"absolute_gap":0.03380898237228391,"min_confidence":0.9622503519058228,"max_confidence":0.975496232509613},{"index":3,"count":6,"confidence":0.9818458457787832,"empirical_correctness":1.0,"signed_gap":-0.0181541542212168,"absolute_gap":0.0181541542212168,"min_confidence":0.9784013628959656,"max_confidence":0.9876948595046997},{"index":4,"count":6,"confidence":0.9894671936829885,"empirical_correctness":1.0,"signed_gap":-0.010532806317011478,"absolute_gap":0.010532806317011478,"min_confidence":0.9883855581283569,"max_confidence":0.9903458952903748},{"index":5,"count":5,"confidence":0.992543077468872,"empirical_correctness":1.0,"signed_gap":-0.007456922531127952,"absolute_gap":0.007456922531127952,"min_confidence":0.9909020066261292,"max_confidence":0.9934280514717102},{"index":6,"count":6,"confidence":0.9945486386617025,"empirical_correctness":1.0,"signed_gap":-0.005451361338297489,"absolute_gap":0.005451361338297489,"min_confidence":0.9934746623039246,"max_confidence":0.9955783486366272},{"index":7,"count":5,"confidence":0.9977863550186157,"empirical_correctness":1.0,"signed_gap":-0.0022136449813843218,"absolute_gap":0.0022136449813843218,"min_confidence":0.9961156845092773,"max_confidence":0.9988560676574707},{"index":8,"count":6,"confidence":0.9995784958203634,"empirical_correctness":1.0,"signed_gap":-0.00042150417963660036,"absolute_gap":0.00042150417963660036,"min_confidence":0.9992672801017761,"max_confidence":0.9999239444732666},{"index":9,"count":6,"confidence":0.9999371568361918,"empirical_correctness":1.0,"signed_gap":-6.284316380822386e-05,"absolute_gap":6.284316380822386e-05,"min_confidence":0.999924898147583,"max_confidence":0.9999486207962036}]}},"calibration_singleton_labels":{"equal_width":{"n":56,"bin_count":10,"ece":0.04251080857855929,"signed_mean_gap":0.002897159329482527,"absolute_mean_gap":0.002897159329482527,"bins":[{"index":0,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.0,"upper":0.1,"upper_inclusive":false},{"index":1,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.1,"upper":0.2,"upper_inclusive":false},{"index":2,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.2,"upper":0.3,"upper_inclusive":false},{"index":3,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.3,"upper":0.4,"upper_inclusive":false},{"index":4,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.4,"upper":0.5,"upper_inclusive":false},{"index":5,"count":1,"confidence":0.5991616249084473,"empirical_correctness":0.0,"signed_gap":0.5991616249084473,"absolute_gap":0.5991616249084473,"lower":0.5,"upper":0.6,"upper_inclusive":false},{"index":6,"count":1,"confidence":0.6722614765167236,"empirical_correctness":0.0,"signed_gap":0.6722614765167236,"absolute_gap":0.6722614765167236,"lower":0.6,"upper":0.7,"upper_inclusive":false},{"index":7,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.7,"upper":0.8,"upper_inclusive":false},{"index":8,"count":2,"confidence":0.8567938804626465,"empirical_correctness":1.0,"signed_gap":-0.14320611953735352,"absolute_gap":0.14320611953735352,"lower":0.8,"upper":0.9,"upper_inclusive":false},{"index":9,"count":52,"confidence":0.98417750115578,"empirical_correctness":1.0,"signed_gap":-0.015822498844220045,"absolute_gap":0.015822498844220045,"lower":0.9,"upper":1.0,"upper_inclusive":true}]},"equal_mass":{"n":56,"bin_count":10,"ece":0.029460065066814423,"signed_mean_gap":0.002897159329482497,"absolute_mean_gap":0.002897159329482497,"bins":[{"index":0,"count":5,"confidence":0.7812004566192627,"empirical_correctness":0.6,"signed_gap":0.18120045661926276,"absolute_gap":0.18120045661926276,"min_confidence":0.5991616249084473,"max_confidence":0.9209914207458496},{"index":1,"count":6,"confidence":0.9468954006830851,"empirical_correctness":1.0,"signed_gap":-0.05310459931691491,"absolute_gap":0.05310459931691491,"min_confidence":0.93880695104599,"max_confidence":0.9556841254234314},{"index":2,"count":5,"confidence":0.9661910176277161,"empirical_correctness":1.0,"signed_gap":-0.03380898237228391,"absolute_gap":0.03380898237228391,"min_confidence":0.9622503519058228,"max_confidence":0.975496232509613},{"index":3,"count":6,"confidence":0.9818458457787832,"empirical_correctness":1.0,"signed_gap":-0.0181541542212168,"absolute_gap":0.0181541542212168,"min_confidence":0.9784013628959656,"max_confidence":0.9876948595046997},{"index":4,"count":6,"confidence":0.9894671936829885,"empirical_correctness":1.0,"signed_gap":-0.010532806317011478,"absolute_gap":0.010532806317011478,"min_confidence":0.9883855581283569,"max_confidence":0.9903458952903748},{"index":5,"count":5,"confidence":0.992543077468872,"empirical_correctness":1.0,"signed_gap":-0.007456922531127952,"absolute_gap":0.007456922531127952,"min_confidence":0.9909020066261292,"max_confidence":0.9934280514717102},{"index":6,"count":6,"confidence":0.9945486386617025,"empirical_correctness":1.0,"signed_gap":-0.005451361338297489,"absolute_gap":0.005451361338297489,"min_confidence":0.9934746623039246,"max_confidence":0.9955783486366272},{"index":7,"count":5,"confidence":0.9977863550186157,"empirical_correctness":1.0,"signed_gap":-0.0022136449813843218,"absolute_gap":0.0022136449813843218,"min_confidence":0.9961156845092773,"max_confidence":0.9988560676574707},{"index":8,"count":6,"confidence":0.9995784958203634,"empirical_correctness":1.0,"signed_gap":-0.00042150417963660036,"absolute_gap":0.00042150417963660036,"min_confidence":0.9992672801017761,"max_confidence":0.9999239444732666},{"index":9,"count":6,"confidence":0.9999371568361918,"empirical_correctness":1.0,"signed_gap":-6.284316380822386e-05,"absolute_gap":6.284316380822386e-05,"min_confidence":0.999924898147583,"max_confidence":0.9999486207962036}]}},"latency_ms":{"n":56,"median":86.19590051239356,"p95":91.60264676029328},"per_action":{"keep":{"n":8,"raw":{"n":8,"acceptable":8,"unsafe":0,"review":0,"automated_count":8,"automated_acceptable":8,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":1.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.6755924351161198,"upper":1.0,"confidence_level":0.95}},"effective":{"n":8,"acceptable":5,"unsafe":0,"review":3,"automated_count":5,"automated_acceptable":5,"automated_unsafe":0,"unnecessary_reviews":3,"accuracy":0.625,"unsafe_rate":0.0,"review_rate":0.375,"automated_coverage":0.625,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.375,"accuracy_wilson_95":{"lower":0.30574239460262737,"upper":0.8631557141764026,"confidence_level":0.95}}},"drop":{"n":22,"raw":{"n":22,"acceptable":22,"unsafe":0,"review":0,"automated_count":22,"automated_acceptable":22,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":1.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.8513451253913791,"upper":1.0,"confidence_level":0.95}},"effective":{"n":22,"acceptable":22,"unsafe":0,"review":0,"automated_count":22,"automated_acceptable":22,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":1.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.8513451253913791,"upper":1.0,"confidence_level":0.95}}},"mask":{"n":8,"raw":{"n":8,"acceptable":8,"unsafe":0,"review":0,"automated_count":8,"automated_acceptable":8,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":1.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.6755924351161198,"upper":1.0,"confidence_level":0.95}},"effective":{"n":8,"acceptable":8,"unsafe":0,"review":0,"automated_count":8,"automated_acceptable":8,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":1.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.6755924351161198,"upper":1.0,"confidence_level":0.95}}},"pseudonymize":{"n":8,"raw":{"n":8,"acceptable":8,"unsafe":0,"review":0,"automated_count":8,"automated_acceptable":8,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":1.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.6755924351161198,"upper":1.0,"confidence_level":0.95}},"effective":{"n":8,"acceptable":1,"unsafe":0,"review":7,"automated_count":1,"automated_acceptable":1,"automated_unsafe":0,"unnecessary_reviews":7,"accuracy":0.125,"unsafe_rate":0.0,"review_rate":0.875,"automated_coverage":0.125,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.875,"accuracy_wilson_95":{"lower":0.02241749145005667,"upper":0.4708881822128535,"confidence_level":0.95}}},"generalize":{"n":8,"raw":{"n":8,"acceptable":8,"unsafe":0,"review":0,"automated_count":8,"automated_acceptable":8,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":1.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.6755924351161198,"upper":1.0,"confidence_level":0.95}},"effective":{"n":8,"acceptable":8,"unsafe":0,"review":0,"automated_count":8,"automated_acceptable":8,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":1.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.6755924351161198,"upper":1.0,"confidence_level":0.95}}},"review":{"n":2,"raw":{"n":2,"acceptable":0,"unsafe":0,"review":0,"automated_count":2,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.6576197724933469,"confidence_level":0.95}},"effective":{"n":2,"acceptable":0,"unsafe":0,"review":0,"automated_count":2,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.6576197724933469,"confidence_level":0.95}}}}},"challenge56-compact_no_samples":{"n":56,"raw":{"n":56,"acceptable":56,"unsafe":0,"review":2,"automated_count":54,"automated_acceptable":54,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":1.0,"unsafe_rate":0.0,"review_rate":0.03571428571428571,"automated_coverage":0.9642857142857143,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.9358060632812366,"upper":1.0,"confidence_level":0.95}},"effective":{"n":56,"acceptable":46,"unsafe":0,"review":12,"automated_count":44,"automated_acceptable":44,"automated_unsafe":0,"unnecessary_reviews":10,"accuracy":0.8214285714285714,"unsafe_rate":0.0,"review_rate":0.21428571428571427,"automated_coverage":0.7857142857142857,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.17857142857142858,"accuracy_wilson_95":{"lower":0.7015881796334894,"upper":0.900001432475877,"confidence_level":0.95}},"confusion":{"raw":{"keep":{"keep":8,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0},"drop":{"keep":0,"drop":22,"mask":0,"pseudonymize":0,"generalize":0,"review":0},"mask":{"keep":0,"drop":0,"mask":8,"pseudonymize":0,"generalize":0,"review":0},"pseudonymize":{"keep":0,"drop":0,"mask":0,"pseudonymize":8,"generalize":0,"review":0},"generalize":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":8,"review":0},"review":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":2}},"effective":{"keep":{"keep":5,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":3},"drop":{"keep":0,"drop":22,"mask":0,"pseudonymize":0,"generalize":0,"review":0},"mask":{"keep":0,"drop":0,"mask":8,"pseudonymize":0,"generalize":0,"review":0},"pseudonymize":{"keep":0,"drop":0,"mask":0,"pseudonymize":1,"generalize":0,"review":7},"generalize":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":8,"review":0},"review":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":2}}},"probability_metrics":{"singleton_n":56,"multiacceptable_n":0,"multiacceptable_coverage":0.0,"nll":0.07440699875278564,"nll_zero_mass_n":0,"brier":0.021573848419655258,"acceptable_set_n":56,"acceptable_set_nll":0.07440699875278564,"acceptable_set_zero_mass_n":0},"calibration_all_labels":{"equal_width":{"n":56,"bin_count":10,"ece":0.064325320401362,"signed_mean_gap":-0.064325320401362,"absolute_mean_gap":0.064325320401362,"bins":[{"index":0,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.0,"upper":0.1,"upper_inclusive":false},{"index":1,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.1,"upper":0.2,"upper_inclusive":false},{"index":2,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.2,"upper":0.3,"upper_inclusive":false},{"index":3,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.3,"upper":0.4,"upper_inclusive":false},{"index":4,"count":1,"confidence":0.4948805868625641,"empirical_correctness":1.0,"signed_gap":-0.5051194131374359,"absolute_gap":0.5051194131374359,"lower":0.4,"upper":0.5,"upper_inclusive":false},{"index":5,"count":1,"confidence":0.5000926852226257,"empirical_correctness":1.0,"signed_gap":-0.49990731477737427,"absolute_gap":0.49990731477737427,"lower":0.5,"upper":0.6,"upper_inclusive":false},{"index":6,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.6,"upper":0.7,"upper_inclusive":false},{"index":7,"count":1,"confidence":0.7798301577568054,"empirical_correctness":1.0,"signed_gap":-0.22016984224319458,"absolute_gap":0.22016984224319458,"lower":0.7,"upper":0.8,"upper_inclusive":false},{"index":8,"count":11,"confidence":0.8622114820913835,"empirical_correctness":1.0,"signed_gap":-0.13778851790861646,"absolute_gap":0.13778851790861646,"lower":0.8,"upper":0.9,"upper_inclusive":false},{"index":9,"count":42,"confidence":0.9794917220161075,"empirical_correctness":1.0,"signed_gap":-0.020508277983892542,"absolute_gap":0.020508277983892542,"lower":0.9,"upper":1.0,"upper_inclusive":true}]},"equal_mass":{"n":56,"bin_count":10,"ece":0.06432532040136203,"signed_mean_gap":-0.06432532040136203,"absolute_mean_gap":0.06432532040136203,"bins":[{"index":0,"count":5,"confidence":0.6807185709476471,"empirical_correctness":1.0,"signed_gap":-0.3192814290523529,"absolute_gap":0.3192814290523529,"min_confidence":0.4948805868625641,"max_confidence":0.8231001496315002},{"index":1,"count":6,"confidence":0.864449123541514,"empirical_correctness":1.0,"signed_gap":-0.13555087645848596,"absolute_gap":0.13555087645848596,"min_confidence":0.8331397175788879,"max_confidence":0.8844354748725891},{"index":2,"count":5,"confidence":0.8975049138069153,"empirical_correctness":1.0,"signed_gap":-0.10249508619308467,"absolute_gap":0.10249508619308467,"min_confidence":0.8852273225784302,"max_confidence":0.910389244556427},{"index":3,"count":6,"confidence":0.9431725839773814,"empirical_correctness":1.0,"signed_gap":-0.05682741602261865,"absolute_gap":0.05682741602261865,"min_confidence":0.9148154854774475,"max_confidence":0.9596772789955139},{"index":4,"count":6,"confidence":0.9721120099226633,"empirical_correctness":1.0,"signed_gap":-0.027887990077336666,"absolute_gap":0.027887990077336666,"min_confidence":0.9631859660148621,"max_confidence":0.9772664308547974},{"index":5,"count":5,"confidence":0.9845947384834289,"empirical_correctness":1.0,"signed_gap":-0.015405261516571067,"absolute_gap":0.015405261516571067,"min_confidence":0.9802189469337463,"max_confidence":0.9875097274780273},{"index":6,"count":6,"confidence":0.9907881120840708,"empirical_correctness":1.0,"signed_gap":-0.009211887915929196,"absolute_gap":0.009211887915929196,"min_confidence":0.9887527227401733,"max_confidence":0.9918495416641235},{"index":7,"count":5,"confidence":0.9937666535377503,"empirical_correctness":1.0,"signed_gap":-0.0062333464622497115,"absolute_gap":0.0062333464622497115,"min_confidence":0.9922490119934082,"max_confidence":0.9949474334716797},{"index":8,"count":6,"confidence":0.9988062481085459,"empirical_correctness":1.0,"signed_gap":-0.0011937518914540979,"absolute_gap":0.0011937518914540979,"min_confidence":0.9978383183479309,"max_confidence":0.9997383952140808},{"index":9,"count":6,"confidence":0.9998148679733276,"empirical_correctness":1.0,"signed_gap":-0.00018513202667236328,"absolute_gap":0.00018513202667236328,"min_confidence":0.9997691512107849,"max_confidence":0.9998660087585449}]}},"calibration_singleton_labels":{"equal_width":{"n":56,"bin_count":10,"ece":0.064325320401362,"signed_mean_gap":-0.064325320401362,"absolute_mean_gap":0.064325320401362,"bins":[{"index":0,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.0,"upper":0.1,"upper_inclusive":false},{"index":1,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.1,"upper":0.2,"upper_inclusive":false},{"index":2,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.2,"upper":0.3,"upper_inclusive":false},{"index":3,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.3,"upper":0.4,"upper_inclusive":false},{"index":4,"count":1,"confidence":0.4948805868625641,"empirical_correctness":1.0,"signed_gap":-0.5051194131374359,"absolute_gap":0.5051194131374359,"lower":0.4,"upper":0.5,"upper_inclusive":false},{"index":5,"count":1,"confidence":0.5000926852226257,"empirical_correctness":1.0,"signed_gap":-0.49990731477737427,"absolute_gap":0.49990731477737427,"lower":0.5,"upper":0.6,"upper_inclusive":false},{"index":6,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.6,"upper":0.7,"upper_inclusive":false},{"index":7,"count":1,"confidence":0.7798301577568054,"empirical_correctness":1.0,"signed_gap":-0.22016984224319458,"absolute_gap":0.22016984224319458,"lower":0.7,"upper":0.8,"upper_inclusive":false},{"index":8,"count":11,"confidence":0.8622114820913835,"empirical_correctness":1.0,"signed_gap":-0.13778851790861646,"absolute_gap":0.13778851790861646,"lower":0.8,"upper":0.9,"upper_inclusive":false},{"index":9,"count":42,"confidence":0.9794917220161075,"empirical_correctness":1.0,"signed_gap":-0.020508277983892542,"absolute_gap":0.020508277983892542,"lower":0.9,"upper":1.0,"upper_inclusive":true}]},"equal_mass":{"n":56,"bin_count":10,"ece":0.06432532040136203,"signed_mean_gap":-0.06432532040136203,"absolute_mean_gap":0.06432532040136203,"bins":[{"index":0,"count":5,"confidence":0.6807185709476471,"empirical_correctness":1.0,"signed_gap":-0.3192814290523529,"absolute_gap":0.3192814290523529,"min_confidence":0.4948805868625641,"max_confidence":0.8231001496315002},{"index":1,"count":6,"confidence":0.864449123541514,"empirical_correctness":1.0,"signed_gap":-0.13555087645848596,"absolute_gap":0.13555087645848596,"min_confidence":0.8331397175788879,"max_confidence":0.8844354748725891},{"index":2,"count":5,"confidence":0.8975049138069153,"empirical_correctness":1.0,"signed_gap":-0.10249508619308467,"absolute_gap":0.10249508619308467,"min_confidence":0.8852273225784302,"max_confidence":0.910389244556427},{"index":3,"count":6,"confidence":0.9431725839773814,"empirical_correctness":1.0,"signed_gap":-0.05682741602261865,"absolute_gap":0.05682741602261865,"min_confidence":0.9148154854774475,"max_confidence":0.9596772789955139},{"index":4,"count":6,"confidence":0.9721120099226633,"empirical_correctness":1.0,"signed_gap":-0.027887990077336666,"absolute_gap":0.027887990077336666,"min_confidence":0.9631859660148621,"max_confidence":0.9772664308547974},{"index":5,"count":5,"confidence":0.9845947384834289,"empirical_correctness":1.0,"signed_gap":-0.015405261516571067,"absolute_gap":0.015405261516571067,"min_confidence":0.9802189469337463,"max_confidence":0.9875097274780273},{"index":6,"count":6,"confidence":0.9907881120840708,"empirical_correctness":1.0,"signed_gap":-0.009211887915929196,"absolute_gap":0.009211887915929196,"min_confidence":0.9887527227401733,"max_confidence":0.9918495416641235},{"index":7,"count":5,"confidence":0.9937666535377503,"empirical_correctness":1.0,"signed_gap":-0.0062333464622497115,"absolute_gap":0.0062333464622497115,"min_confidence":0.9922490119934082,"max_confidence":0.9949474334716797},{"index":8,"count":6,"confidence":0.9988062481085459,"empirical_correctness":1.0,"signed_gap":-0.0011937518914540979,"absolute_gap":0.0011937518914540979,"min_confidence":0.9978383183479309,"max_confidence":0.9997383952140808},{"index":9,"count":6,"confidence":0.9998148679733276,"empirical_correctness":1.0,"signed_gap":-0.00018513202667236328,"absolute_gap":0.00018513202667236328,"min_confidence":0.9997691512107849,"max_confidence":0.9998660087585449}]}},"latency_ms":{"n":56,"median":97.62890648562461,"p95":104.32477548602037},"per_action":{"keep":{"n":8,"raw":{"n":8,"acceptable":8,"unsafe":0,"review":0,"automated_count":8,"automated_acceptable":8,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":1.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.6755924351161198,"upper":1.0,"confidence_level":0.95}},"effective":{"n":8,"acceptable":5,"unsafe":0,"review":3,"automated_count":5,"automated_acceptable":5,"automated_unsafe":0,"unnecessary_reviews":3,"accuracy":0.625,"unsafe_rate":0.0,"review_rate":0.375,"automated_coverage":0.625,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.375,"accuracy_wilson_95":{"lower":0.30574239460262737,"upper":0.8631557141764026,"confidence_level":0.95}}},"drop":{"n":22,"raw":{"n":22,"acceptable":22,"unsafe":0,"review":0,"automated_count":22,"automated_acceptable":22,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":1.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.8513451253913791,"upper":1.0,"confidence_level":0.95}},"effective":{"n":22,"acceptable":22,"unsafe":0,"review":0,"automated_count":22,"automated_acceptable":22,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":1.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.8513451253913791,"upper":1.0,"confidence_level":0.95}}},"mask":{"n":8,"raw":{"n":8,"acceptable":8,"unsafe":0,"review":0,"automated_count":8,"automated_acceptable":8,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":1.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.6755924351161198,"upper":1.0,"confidence_level":0.95}},"effective":{"n":8,"acceptable":8,"unsafe":0,"review":0,"automated_count":8,"automated_acceptable":8,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":1.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.6755924351161198,"upper":1.0,"confidence_level":0.95}}},"pseudonymize":{"n":8,"raw":{"n":8,"acceptable":8,"unsafe":0,"review":0,"automated_count":8,"automated_acceptable":8,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":1.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.6755924351161198,"upper":1.0,"confidence_level":0.95}},"effective":{"n":8,"acceptable":1,"unsafe":0,"review":7,"automated_count":1,"automated_acceptable":1,"automated_unsafe":0,"unnecessary_reviews":7,"accuracy":0.125,"unsafe_rate":0.0,"review_rate":0.875,"automated_coverage":0.125,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.875,"accuracy_wilson_95":{"lower":0.02241749145005667,"upper":0.4708881822128535,"confidence_level":0.95}}},"generalize":{"n":8,"raw":{"n":8,"acceptable":8,"unsafe":0,"review":0,"automated_count":8,"automated_acceptable":8,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":1.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.6755924351161198,"upper":1.0,"confidence_level":0.95}},"effective":{"n":8,"acceptable":8,"unsafe":0,"review":0,"automated_count":8,"automated_acceptable":8,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":1.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.6755924351161198,"upper":1.0,"confidence_level":0.95}}},"review":{"n":2,"raw":{"n":2,"acceptable":2,"unsafe":0,"review":2,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":1.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.34238022750665303,"upper":1.0,"confidence_level":0.95}},"effective":{"n":2,"acceptable":2,"unsafe":0,"review":2,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":1.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.34238022750665303,"upper":1.0,"confidence_level":0.95}}}}}},"composite":{"n":198,"scope":"descriptive sum across four correlated panels","raw":{"n":198,"acceptable":181,"unsafe":0,"review":2,"automated_count":196,"automated_acceptable":179,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":0.9141414141414141,"unsafe_rate":0.0,"review_rate":0.010101010101010102,"automated_coverage":0.98989898989899,"conditional_correctness":0.9132653061224489,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0},"effective":{"n":198,"acceptable":158,"unsafe":0,"review":25,"automated_count":173,"automated_acceptable":156,"automated_unsafe":0,"unnecessary_reviews":23,"accuracy":0.797979797979798,"unsafe_rate":0.0,"review_rate":0.12626262626262627,"automated_coverage":0.8737373737373737,"conditional_correctness":0.9017341040462428,"conditional_unsafe":0.0,"unnecessary_review_rate":0.11616161616161616}}},{"id":"internlm--Intern-Decision-0.8B","platform":"Local","model_id":"internlm/Intern-Decision-0.8B","revision":"85a0cc5a99d67ea8d56dfe98115689212867171d","loading":"bfloat16 / none","scoring_kind":"Native decision readout","settings":{"dtype":"bfloat16","quantization":"none","max_tokens":4096,"effective_max_tokens":4096,"temperature":2.747760550703,"cpu_offload":false,"scoring_batch_size":1,"trust_remote_code":false,"local_files_only":true,"use_safetensors":true,"guard_threshold":0.45,"seed":0,"tf32":false},"scoring_metadata":{"backend":"intern_decision","protocol":"hf-local-sweep-native-v1","readout":"builtin_logits_immediately_before_decision","probability_source":"native_choice_published_temperature_not_privacy_calibrated","latency_scope":"synchronous_forward_scoring_including_transfer_excluding_tokenization","precision":"bfloat16","equal_mass_convention":"rank bins with floor boundaries at floor(i*n/k), k=min(10,n//2); minimum two rows"},"load_ms":1920.8080870448612,"peak_allocated_bytes":1842024960,"peak_reserved_bytes":1908408320,"panels":{"diagnostic43-no_samples":{"n":43,"raw":{"n":43,"acceptable":12,"unsafe":2,"review":0,"automated_count":43,"automated_acceptable":12,"automated_unsafe":2,"unnecessary_reviews":0,"accuracy":0.27906976744186046,"unsafe_rate":0.046511627906976744,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.27906976744186046,"conditional_unsafe":0.046511627906976744,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.16746613786237488,"upper":0.42691028681878873,"confidence_level":0.95}},"effective":{"n":43,"acceptable":6,"unsafe":0,"review":36,"automated_count":7,"automated_acceptable":2,"automated_unsafe":0,"unnecessary_reviews":32,"accuracy":0.13953488372093023,"unsafe_rate":0.0,"review_rate":0.8372093023255814,"automated_coverage":0.16279069767441862,"conditional_correctness":0.2857142857142857,"conditional_unsafe":0.0,"unnecessary_review_rate":0.7441860465116279,"accuracy_wilson_95":{"lower":0.06555723427964878,"upper":0.2726358796738286,"confidence_level":0.95}},"confusion":{"raw":{"keep":{"keep":0,"drop":0,"mask":11,"pseudonymize":1,"generalize":0,"review":0},"drop":{"keep":0,"drop":4,"mask":10,"pseudonymize":2,"generalize":0,"review":0},"mask":{"keep":0,"drop":0,"mask":2,"pseudonymize":0,"generalize":0,"review":0},"pseudonymize":{"keep":0,"drop":0,"mask":1,"pseudonymize":1,"generalize":0,"review":0},"generalize":{"keep":0,"drop":1,"mask":3,"pseudonymize":2,"generalize":0,"review":0},"review":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0}},"effective":{"keep":{"keep":0,"drop":0,"mask":3,"pseudonymize":0,"generalize":0,"review":9},"drop":{"keep":0,"drop":0,"mask":2,"pseudonymize":0,"generalize":0,"review":14},"mask":{"keep":0,"drop":0,"mask":1,"pseudonymize":0,"generalize":0,"review":1},"pseudonymize":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":2},"generalize":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":6},"review":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0}}},"probability_metrics":{"singleton_n":38,"multiacceptable_n":5,"multiacceptable_coverage":0.11627906976744186,"nll":1.8938744587199878,"nll_zero_mass_n":0,"brier":0.8798577716871068,"acceptable_set_n":43,"acceptable_set_nll":1.7075763229421135,"acceptable_set_zero_mass_n":0},"calibration_all_labels":{"equal_width":{"n":43,"bin_count":10,"ece":0.14491679648166536,"signed_mean_gap":0.14491679648166536,"absolute_mean_gap":0.14491679648166536,"bins":[{"index":0,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.0,"upper":0.1,"upper_inclusive":false},{"index":1,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.1,"upper":0.2,"upper_inclusive":false},{"index":2,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.2,"upper":0.3,"upper_inclusive":false},{"index":3,"count":22,"confidence":0.370740476700451,"empirical_correctness":0.22727272727272727,"signed_gap":0.14346774942772375,"absolute_gap":0.14346774942772375,"lower":0.3,"upper":0.4,"upper_inclusive":false},{"index":4,"count":14,"confidence":0.41839478012695436,"empirical_correctness":0.35714285714285715,"signed_gap":0.06125192298409721,"absolute_gap":0.06125192298409721,"lower":0.4,"upper":0.5,"upper_inclusive":false},{"index":5,"count":3,"confidence":0.5744439591582983,"empirical_correctness":0.0,"signed_gap":0.5744439591582983,"absolute_gap":0.5744439591582983,"lower":0.5,"upper":0.6,"upper_inclusive":false},{"index":6,"count":4,"confidence":0.6235682405123582,"empirical_correctness":0.5,"signed_gap":0.12356824051235815,"absolute_gap":0.12356824051235815,"lower":0.6,"upper":0.7,"upper_inclusive":false},{"index":7,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.7,"upper":0.8,"upper_inclusive":false},{"index":8,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.8,"upper":0.9,"upper_inclusive":false},{"index":9,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.9,"upper":1.0,"upper_inclusive":true}]},"equal_mass":{"n":43,"bin_count":10,"ece":0.1762700193029871,"signed_mean_gap":0.14491679648166536,"absolute_mean_gap":0.14491679648166536,"bins":[{"index":0,"count":4,"confidence":0.34165640399988334,"empirical_correctness":0.25,"signed_gap":0.09165640399988334,"absolute_gap":0.09165640399988334,"min_confidence":0.3081991047727647,"max_confidence":0.3560927215703785},{"index":1,"count":4,"confidence":0.3582936237290222,"empirical_correctness":0.25,"signed_gap":0.10829362372902218,"absolute_gap":0.10829362372902218,"min_confidence":0.35731882515883273,"max_confidence":0.35918374134885406},{"index":2,"count":4,"confidence":0.36930650410595095,"empirical_correctness":0.25,"signed_gap":0.11930650410595095,"absolute_gap":0.11930650410595095,"min_confidence":0.36061068265362384,"max_confidence":0.37628675230664294},{"index":3,"count":5,"confidence":0.3816576735968713,"empirical_correctness":0.0,"signed_gap":0.3816576735968713,"absolute_gap":0.3816576735968713,"min_confidence":0.37665565883610225,"max_confidence":0.39113875101571793},{"index":4,"count":4,"confidence":0.39330589330564036,"empirical_correctness":0.25,"signed_gap":0.14330589330564036,"absolute_gap":0.14330589330564036,"min_confidence":0.3927322217387423,"max_confidence":0.393949716823783},{"index":5,"count":4,"confidence":0.40143159971875686,"empirical_correctness":0.5,"signed_gap":-0.09856840028124314,"absolute_gap":0.09856840028124314,"min_confidence":0.39775241886357837,"max_confidence":0.4042640852369733},{"index":6,"count":5,"confidence":0.4096395130806071,"empirical_correctness":0.0,"signed_gap":0.4096395130806071,"absolute_gap":0.4096395130806071,"min_confidence":0.40688892712948505,"max_confidence":0.4128003203168438},{"index":7,"count":4,"confidence":0.43004482761663876,"empirical_correctness":0.5,"signed_gap":-0.06995517238336124,"absolute_gap":0.06995517238336124,"min_confidence":0.4147872748143301,"max_confidence":0.4385391860892131},{"index":8,"count":4,"confidence":0.5041971154871098,"empirical_correctness":0.5,"signed_gap":0.0041971154871097704,"absolute_gap":0.0041971154871097704,"min_confidence":0.4389370220465934,"max_confidence":0.5839725347556307},{"index":9,"count":5,"confidence":0.6163984886944419,"empirical_correctness":0.4,"signed_gap":0.2163984886944419,"absolute_gap":0.2163984886944419,"min_confidence":0.5877194814227773,"max_confidence":0.6510943344189141}]}},"calibration_singleton_labels":{"equal_width":{"n":38,"bin_count":10,"ece":0.2366220369581915,"signed_mean_gap":0.2366220369581915,"absolute_mean_gap":0.2366220369581915,"bins":[{"index":0,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.0,"upper":0.1,"upper_inclusive":false},{"index":1,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.1,"upper":0.2,"upper_inclusive":false},{"index":2,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.2,"upper":0.3,"upper_inclusive":false},{"index":3,"count":20,"confidence":0.3702271916852362,"empirical_correctness":0.15,"signed_gap":0.22022719168523622,"absolute_gap":0.22022719168523622,"lower":0.3,"upper":0.4,"upper_inclusive":false},{"index":4,"count":12,"confidence":0.41491813101792757,"empirical_correctness":0.25,"signed_gap":0.16491813101792757,"absolute_gap":0.16491813101792757,"lower":0.4,"upper":0.5,"upper_inclusive":false},{"index":5,"count":3,"confidence":0.5744439591582983,"empirical_correctness":0.0,"signed_gap":0.5744439591582983,"absolute_gap":0.5744439591582983,"lower":0.5,"upper":0.6,"upper_inclusive":false},{"index":6,"count":3,"confidence":0.6282480403388424,"empirical_correctness":0.3333333333333333,"signed_gap":0.2949147070055091,"absolute_gap":0.2949147070055091,"lower":0.6,"upper":0.7,"upper_inclusive":false},{"index":7,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.7,"upper":0.8,"upper_inclusive":false},{"index":8,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.8,"upper":0.9,"upper_inclusive":false},{"index":9,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.9,"upper":1.0,"upper_inclusive":true}]},"equal_mass":{"n":38,"bin_count":10,"ece":0.2366220369581915,"signed_mean_gap":0.2366220369581915,"absolute_mean_gap":0.2366220369581915,"bins":[{"index":0,"count":3,"confidence":0.33684429814305156,"empirical_correctness":0.0,"signed_gap":0.33684429814305156,"absolute_gap":0.33684429814305156,"min_confidence":0.3081991047727647,"max_confidence":0.35303795961224294},{"index":1,"count":4,"confidence":0.3576359046459668,"empirical_correctness":0.25,"signed_gap":0.10763590464596678,"absolute_gap":0.10763590464596678,"min_confidence":0.3560927215703785,"max_confidence":0.35918374134885406},{"index":2,"count":4,"confidence":0.36930650410595095,"empirical_correctness":0.25,"signed_gap":0.11930650410595095,"absolute_gap":0.11930650410595095,"min_confidence":0.36061068265362384,"max_confidence":0.37628675230664294},{"index":3,"count":4,"confidence":0.3792874042421596,"empirical_correctness":0.0,"signed_gap":0.3792874042421596,"absolute_gap":0.3792874042421596,"min_confidence":0.37665565883610225,"max_confidence":0.3825846157909363},{"index":4,"count":4,"confidence":0.3928348171089204,"empirical_correctness":0.0,"signed_gap":0.3928348171089204,"absolute_gap":0.3928348171089204,"min_confidence":0.39113875101571793,"max_confidence":0.393949716823783},{"index":5,"count":3,"confidence":0.4004874378793514,"empirical_correctness":0.3333333333333333,"signed_gap":0.06715410454601811,"absolute_gap":0.06715410454601811,"min_confidence":0.39775241886357837,"max_confidence":0.4036556316858386},{"index":6,"count":4,"confidence":0.4073158469418973,"empirical_correctness":0.25,"signed_gap":0.15731584694189732,"absolute_gap":0.15731584694189732,"min_confidence":0.4042640852369733,"max_confidence":0.4093464130457236},{"index":7,"count":4,"confidence":0.4171420203843148,"empirical_correctness":0.25,"signed_gap":0.1671420203843148,"absolute_gap":0.1671420203843148,"min_confidence":0.4103979425555756,"max_confidence":0.4305825438505098},{"index":8,"count":4,"confidence":0.503272151046981,"empirical_correctness":0.25,"signed_gap":0.253272151046981,"absolute_gap":0.253272151046981,"min_confidence":0.4385391860892131,"max_confidence":0.5839725347556307},{"index":9,"count":4,"confidence":0.6181159006098261,"empirical_correctness":0.25,"signed_gap":0.36811590060982613,"absolute_gap":0.36811590060982613,"min_confidence":0.5877194814227773,"max_confidence":0.6510943344189141}]}},"latency_ms":{"n":43,"median":32.652827037964016,"p95":33.327805798035115},"per_action":{"keep":{"n":12,"raw":{"n":12,"acceptable":0,"unsafe":0,"review":0,"automated_count":12,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":2.7755575615628914e-17,"upper":0.24249400665524085,"confidence_level":0.95}},"effective":{"n":12,"acceptable":0,"unsafe":0,"review":9,"automated_count":3,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":9,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":0.75,"automated_coverage":0.25,"conditional_correctness":0.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.75,"accuracy_wilson_95":{"lower":2.7755575615628914e-17,"upper":0.24249400665524085,"confidence_level":0.95}}},"drop":{"n":16,"raw":{"n":16,"acceptable":4,"unsafe":2,"review":0,"automated_count":16,"automated_acceptable":4,"automated_unsafe":2,"unnecessary_reviews":0,"accuracy":0.25,"unsafe_rate":0.125,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.25,"conditional_unsafe":0.125,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.10182067491213048,"upper":0.49498316535508774,"confidence_level":0.95}},"effective":{"n":16,"acceptable":0,"unsafe":0,"review":14,"automated_count":2,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":14,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":0.875,"automated_coverage":0.125,"conditional_correctness":0.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.875,"accuracy_wilson_95":{"lower":0.0,"upper":0.1936076805344365,"confidence_level":0.95}}},"mask":{"n":2,"raw":{"n":2,"acceptable":2,"unsafe":0,"review":0,"automated_count":2,"automated_acceptable":2,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":1.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.34238022750665303,"upper":1.0,"confidence_level":0.95}},"effective":{"n":2,"acceptable":1,"unsafe":0,"review":1,"automated_count":1,"automated_acceptable":1,"automated_unsafe":0,"unnecessary_reviews":1,"accuracy":0.5,"unsafe_rate":0.0,"review_rate":0.5,"automated_coverage":0.5,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.5,"accuracy_wilson_95":{"lower":0.09453120573423074,"upper":0.9054687942657693,"confidence_level":0.95}}},"pseudonymize":{"n":2,"raw":{"n":2,"acceptable":1,"unsafe":0,"review":0,"automated_count":2,"automated_acceptable":1,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":0.5,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.5,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.09453120573423074,"upper":0.9054687942657693,"confidence_level":0.95}},"effective":{"n":2,"acceptable":0,"unsafe":0,"review":2,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":2,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.6576197724933469,"confidence_level":0.95}}},"generalize":{"n":6,"raw":{"n":6,"acceptable":0,"unsafe":0,"review":0,"automated_count":6,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":2.7755575615628914e-17,"upper":0.3903342879021653,"confidence_level":0.95}},"effective":{"n":6,"acceptable":0,"unsafe":0,"review":6,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":6,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":2.7755575615628914e-17,"upper":0.3903342879021653,"confidence_level":0.95}}},"review":{"n":0,"raw":{"n":0,"acceptable":0,"unsafe":0,"review":0,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":null,"unsafe_rate":null,"review_rate":null,"automated_coverage":null,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":null},"effective":{"n":0,"acceptable":0,"unsafe":0,"review":0,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":null,"unsafe_rate":null,"review_rate":null,"automated_coverage":null,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":null}}}},"diagnostic43-compact_no_samples":{"n":43,"raw":{"n":43,"acceptable":18,"unsafe":0,"review":0,"automated_count":43,"automated_acceptable":18,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":0.4186046511627907,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.4186046511627907,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.28384550927849195,"upper":0.5667142261303578,"confidence_level":0.95}},"effective":{"n":43,"acceptable":13,"unsafe":0,"review":24,"automated_count":19,"automated_acceptable":13,"automated_unsafe":0,"unnecessary_reviews":24,"accuracy":0.3023255813953488,"unsafe_rate":0.0,"review_rate":0.5581395348837209,"automated_coverage":0.4418604651162791,"conditional_correctness":0.6842105263157895,"conditional_unsafe":0.0,"unnecessary_review_rate":0.5581395348837209,"accuracy_wilson_95":{"lower":0.1860200013584818,"upper":0.45105364177729607,"confidence_level":0.95}},"confusion":{"raw":{"keep":{"keep":0,"drop":5,"mask":7,"pseudonymize":0,"generalize":0,"review":0},"drop":{"keep":0,"drop":10,"mask":6,"pseudonymize":0,"generalize":0,"review":0},"mask":{"keep":0,"drop":0,"mask":2,"pseudonymize":0,"generalize":0,"review":0},"pseudonymize":{"keep":0,"drop":0,"mask":2,"pseudonymize":0,"generalize":0,"review":0},"generalize":{"keep":0,"drop":2,"mask":3,"pseudonymize":0,"generalize":1,"review":0},"review":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0}},"effective":{"keep":{"keep":0,"drop":0,"mask":3,"pseudonymize":0,"generalize":0,"review":9},"drop":{"keep":0,"drop":6,"mask":3,"pseudonymize":0,"generalize":0,"review":7},"mask":{"keep":0,"drop":0,"mask":2,"pseudonymize":0,"generalize":0,"review":0},"pseudonymize":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":2},"generalize":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":6},"review":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0}}},"probability_metrics":{"singleton_n":38,"multiacceptable_n":5,"multiacceptable_coverage":0.11627906976744186,"nll":1.6557143547783193,"nll_zero_mass_n":0,"brier":0.7811068194769613,"acceptable_set_n":43,"acceptable_set_nll":1.482645373498523,"acceptable_set_zero_mass_n":0},"calibration_all_labels":{"equal_width":{"n":43,"bin_count":10,"ece":0.10203111926028034,"signed_mean_gap":0.03759211672096008,"absolute_mean_gap":0.03759211672096008,"bins":[{"index":0,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.0,"upper":0.1,"upper_inclusive":false},{"index":1,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.1,"upper":0.2,"upper_inclusive":false},{"index":2,"count":1,"confidence":0.2716809157590573,"empirical_correctness":0.0,"signed_gap":0.2716809157590573,"absolute_gap":0.2716809157590573,"lower":0.2,"upper":0.3,"upper_inclusive":false},{"index":3,"count":14,"confidence":0.36195548216621376,"empirical_correctness":0.2857142857142857,"signed_gap":0.07624119645192806,"absolute_gap":0.07624119645192806,"lower":0.3,"upper":0.4,"upper_inclusive":false},{"index":4,"count":14,"confidence":0.44095931126538984,"empirical_correctness":0.35714285714285715,"signed_gap":0.08381645412253269,"absolute_gap":0.08381645412253269,"lower":0.4,"upper":0.5,"upper_inclusive":false},{"index":5,"count":10,"confidence":0.5489411549795161,"empirical_correctness":0.5,"signed_gap":0.04894115497951612,"absolute_gap":0.04894115497951612,"lower":0.5,"upper":0.6,"upper_inclusive":false},{"index":6,"count":4,"confidence":0.6536403613511536,"empirical_correctness":1.0,"signed_gap":-0.34635963864884645,"absolute_gap":0.34635963864884645,"lower":0.6,"upper":0.7,"upper_inclusive":false},{"index":7,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.7,"upper":0.8,"upper_inclusive":false},{"index":8,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.8,"upper":0.9,"upper_inclusive":false},{"index":9,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.9,"upper":1.0,"upper_inclusive":true}]},"equal_mass":{"n":43,"bin_count":10,"ece":0.1744725969733483,"signed_mean_gap":0.03759211672096005,"absolute_mean_gap":0.03759211672096005,"bins":[{"index":0,"count":4,"confidence":0.30726530437460786,"empirical_correctness":0.0,"signed_gap":0.30726530437460786,"absolute_gap":0.30726530437460786,"min_confidence":0.2716809157590573,"max_confidence":0.32377513051319334},{"index":1,"count":4,"confidence":0.3524604797729874,"empirical_correctness":0.25,"signed_gap":0.10246047977298739,"absolute_gap":0.10246047977298739,"min_confidence":0.3397896335429749,"max_confidence":0.36809962256371814},{"index":2,"count":4,"confidence":0.3810632824549975,"empirical_correctness":0.5,"signed_gap":-0.11893671754500251,"absolute_gap":0.11893671754500251,"min_confidence":0.3747808377623644,"max_confidence":0.38539306598609535},{"index":3,"count":5,"confidence":0.3994714057894636,"empirical_correctness":0.2,"signed_gap":0.1994714057894636,"absolute_gap":0.1994714057894636,"min_confidence":0.3878059992954111,"max_confidence":0.41177460236861285},{"index":4,"count":4,"confidence":0.4192950026881372,"empirical_correctness":0.0,"signed_gap":0.4192950026881372,"absolute_gap":0.4192950026881372,"min_confidence":0.4132130375707514,"max_confidence":0.4275044041893568},{"index":5,"count":4,"confidence":0.4391745548290511,"empirical_correctness":0.5,"signed_gap":-0.0608254451709489,"absolute_gap":0.0608254451709489,"min_confidence":0.42951439531169033,"max_confidence":0.4538293215207111},{"index":6,"count":5,"confidence":0.4852670765802921,"empirical_correctness":0.6,"signed_gap":-0.11473292341970787,"absolute_gap":0.11473292341970787,"min_confidence":0.4590578855877576,"max_confidence":0.5082388845263952},{"index":7,"count":4,"confidence":0.5357140158058251,"empirical_correctness":0.75,"signed_gap":-0.21428598419417488,"absolute_gap":0.21428598419417488,"min_confidence":0.532239557446525,"max_confidence":0.5394912513324479},{"index":8,"count":4,"confidence":0.5614877920343455,"empirical_correctness":0.5,"signed_gap":0.06148779203434551,"absolute_gap":0.06148779203434551,"min_confidence":0.5429275550484758,"max_confidence":0.5896127115555514},{"index":9,"count":5,"confidence":0.6413853758625394,"empirical_correctness":0.8,"signed_gap":-0.15861462413746064,"absolute_gap":0.15861462413746064,"min_confidence":0.5923654339080828,"max_confidence":0.694652683188134}]}},"calibration_singleton_labels":{"equal_width":{"n":38,"bin_count":10,"ece":0.15708402645201439,"signed_mean_gap":0.10041324246849488,"absolute_mean_gap":0.10041324246849488,"bins":[{"index":0,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.0,"upper":0.1,"upper_inclusive":false},{"index":1,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.1,"upper":0.2,"upper_inclusive":false},{"index":2,"count":1,"confidence":0.2716809157590573,"empirical_correctness":0.0,"signed_gap":0.2716809157590573,"absolute_gap":0.2716809157590573,"lower":0.2,"upper":0.3,"upper_inclusive":false},{"index":3,"count":14,"confidence":0.36195548216621376,"empirical_correctness":0.2857142857142857,"signed_gap":0.07624119645192806,"absolute_gap":0.07624119645192806,"lower":0.3,"upper":0.4,"upper_inclusive":false},{"index":4,"count":13,"confidence":0.43725044768031057,"empirical_correctness":0.3076923076923077,"signed_gap":0.12955813998800286,"absolute_gap":0.12955813998800286,"lower":0.4,"upper":0.5,"upper_inclusive":false},{"index":5,"count":7,"confidence":0.5527335176513698,"empirical_correctness":0.2857142857142857,"signed_gap":0.2670192319370841,"absolute_gap":0.2670192319370841,"lower":0.5,"upper":0.6,"upper_inclusive":false},{"index":6,"count":3,"confidence":0.6410850347710432,"empirical_correctness":1.0,"signed_gap":-0.35891496522895683,"absolute_gap":0.35891496522895683,"lower":0.6,"upper":0.7,"upper_inclusive":false},{"index":7,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.7,"upper":0.8,"upper_inclusive":false},{"index":8,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.8,"upper":0.9,"upper_inclusive":false},{"index":9,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.9,"upper":1.0,"upper_inclusive":true}]},"equal_mass":{"n":38,"bin_count":10,"ece":0.16024709965400757,"signed_mean_gap":0.10041324246849487,"absolute_mean_gap":0.10041324246849487,"bins":[{"index":0,"count":3,"confidence":0.30176202899507937,"empirical_correctness":0.0,"signed_gap":0.30176202899507937,"absolute_gap":0.30176202899507937,"min_confidence":0.2716809157590573,"max_confidence":0.3198238459440615},{"index":1,"count":4,"confidence":0.3413793567603562,"empirical_correctness":0.25,"signed_gap":0.09137935676035619,"absolute_gap":0.09137935676035619,"min_confidence":0.32377513051319334,"max_confidence":0.3565776249395765},{"index":2,"count":4,"confidence":0.37673992159940317,"empirical_correctness":0.25,"signed_gap":0.12673992159940317,"absolute_gap":0.12673992159940317,"min_confidence":0.36809962256371814,"max_confidence":0.38298519539998066},{"index":3,"count":4,"confidence":0.39032361641544366,"empirical_correctness":0.5,"signed_gap":-0.10967638358455634,"absolute_gap":0.10967638358455634,"min_confidence":0.38539306598609535,"max_confidence":0.39518763855082323},{"index":4,"count":4,"confidence":0.41210443057755525,"empirical_correctness":0.0,"signed_gap":0.41210443057755525,"absolute_gap":0.41210443057755525,"min_confidence":0.40968102690302594,"max_confidence":0.41374905546783075},{"index":5,"count":3,"confidence":0.426577437675219,"empirical_correctness":0.3333333333333333,"signed_gap":0.09324410434188568,"absolute_gap":0.09324410434188568,"min_confidence":0.42271351352460984,"max_confidence":0.42951439531169033},{"index":6,"count":4,"confidence":0.4465604273980679,"empirical_correctness":0.5,"signed_gap":-0.053439572601932106,"absolute_gap":0.053439572601932106,"min_confidence":0.4353217599423549,"max_confidence":0.4590578855877576},{"index":7,"count":4,"confidence":0.5026805309646352,"empirical_correctness":0.25,"signed_gap":0.25268053096463516,"absolute_gap":0.25268053096463516,"min_confidence":0.4743446476001639,"max_confidence":0.5326191644162578},{"index":8,"count":4,"confidence":0.5589777851772131,"empirical_correctness":0.5,"signed_gap":0.05897778517721308,"absolute_gap":0.05897778517721308,"min_confidence":0.5394912513324479,"max_confidence":0.5896127115555514},{"index":9,"count":4,"confidence":0.6289051345553031,"empirical_correctness":0.75,"signed_gap":-0.12109486544469694,"absolute_gap":0.12109486544469694,"min_confidence":0.5923654339080828,"max_confidence":0.694652683188134}]}},"latency_ms":{"n":43,"median":35.4109620093368,"p95":35.781973000848666},"per_action":{"keep":{"n":12,"raw":{"n":12,"acceptable":0,"unsafe":0,"review":0,"automated_count":12,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":2.7755575615628914e-17,"upper":0.24249400665524085,"confidence_level":0.95}},"effective":{"n":12,"acceptable":0,"unsafe":0,"review":9,"automated_count":3,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":9,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":0.75,"automated_coverage":0.25,"conditional_correctness":0.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.75,"accuracy_wilson_95":{"lower":2.7755575615628914e-17,"upper":0.24249400665524085,"confidence_level":0.95}}},"drop":{"n":16,"raw":{"n":16,"acceptable":10,"unsafe":0,"review":0,"automated_count":16,"automated_acceptable":10,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":0.625,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.625,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.38641040545502714,"upper":0.8151876744113636,"confidence_level":0.95}},"effective":{"n":16,"acceptable":6,"unsafe":0,"review":7,"automated_count":9,"automated_acceptable":6,"automated_unsafe":0,"unnecessary_reviews":7,"accuracy":0.375,"unsafe_rate":0.0,"review_rate":0.4375,"automated_coverage":0.5625,"conditional_correctness":0.6666666666666666,"conditional_unsafe":0.0,"unnecessary_review_rate":0.4375,"accuracy_wilson_95":{"lower":0.18481232558863633,"upper":0.6135895945449727,"confidence_level":0.95}}},"mask":{"n":2,"raw":{"n":2,"acceptable":2,"unsafe":0,"review":0,"automated_count":2,"automated_acceptable":2,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":1.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.34238022750665303,"upper":1.0,"confidence_level":0.95}},"effective":{"n":2,"acceptable":2,"unsafe":0,"review":0,"automated_count":2,"automated_acceptable":2,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":1.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.34238022750665303,"upper":1.0,"confidence_level":0.95}}},"pseudonymize":{"n":2,"raw":{"n":2,"acceptable":0,"unsafe":0,"review":0,"automated_count":2,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.6576197724933469,"confidence_level":0.95}},"effective":{"n":2,"acceptable":0,"unsafe":0,"review":2,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":2,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.6576197724933469,"confidence_level":0.95}}},"generalize":{"n":6,"raw":{"n":6,"acceptable":1,"unsafe":0,"review":0,"automated_count":6,"automated_acceptable":1,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":0.16666666666666666,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.16666666666666666,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.030053369748306635,"upper":0.5635028221864702,"confidence_level":0.95}},"effective":{"n":6,"acceptable":0,"unsafe":0,"review":6,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":6,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":2.7755575615628914e-17,"upper":0.3903342879021653,"confidence_level":0.95}}},"review":{"n":0,"raw":{"n":0,"acceptable":0,"unsafe":0,"review":0,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":null,"unsafe_rate":null,"review_rate":null,"automated_coverage":null,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":null},"effective":{"n":0,"acceptable":0,"unsafe":0,"review":0,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":null,"unsafe_rate":null,"review_rate":null,"automated_coverage":null,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":null}}}},"challenge56-no_samples":{"n":56,"raw":{"n":56,"acceptable":54,"unsafe":0,"review":2,"automated_count":54,"automated_acceptable":52,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":0.9642857142857143,"unsafe_rate":0.0,"review_rate":0.03571428571428571,"automated_coverage":0.9642857142857143,"conditional_correctness":0.9629629629629629,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.8788121981950989,"upper":0.9901505748517636,"confidence_level":0.95}},"effective":{"n":56,"acceptable":31,"unsafe":0,"review":26,"automated_count":30,"automated_acceptable":29,"automated_unsafe":0,"unnecessary_reviews":24,"accuracy":0.5535714285714286,"unsafe_rate":0.0,"review_rate":0.4642857142857143,"automated_coverage":0.5357142857142857,"conditional_correctness":0.9666666666666667,"conditional_unsafe":0.0,"unnecessary_review_rate":0.42857142857142855,"accuracy_wilson_95":{"lower":0.4241321478993947,"upper":0.6761327874521665,"confidence_level":0.95}},"confusion":{"raw":{"keep":{"keep":8,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0},"drop":{"keep":0,"drop":22,"mask":0,"pseudonymize":0,"generalize":0,"review":0},"mask":{"keep":0,"drop":0,"mask":8,"pseudonymize":0,"generalize":0,"review":0},"pseudonymize":{"keep":0,"drop":0,"mask":0,"pseudonymize":8,"generalize":0,"review":0},"generalize":{"keep":0,"drop":2,"mask":0,"pseudonymize":0,"generalize":6,"review":0},"review":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":2}},"effective":{"keep":{"keep":2,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":6},"drop":{"keep":0,"drop":17,"mask":0,"pseudonymize":0,"generalize":0,"review":5},"mask":{"keep":0,"drop":0,"mask":8,"pseudonymize":0,"generalize":0,"review":0},"pseudonymize":{"keep":0,"drop":0,"mask":0,"pseudonymize":1,"generalize":0,"review":7},"generalize":{"keep":0,"drop":1,"mask":0,"pseudonymize":0,"generalize":1,"review":6},"review":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":2}}},"probability_metrics":{"singleton_n":56,"multiacceptable_n":0,"multiacceptable_coverage":0.0,"nll":0.6326890769944529,"nll_zero_mass_n":0,"brier":0.28628276626973737,"acceptable_set_n":56,"acceptable_set_nll":0.6326890769944529,"acceptable_set_zero_mass_n":0},"calibration_all_labels":{"equal_width":{"n":56,"bin_count":10,"ece":0.4042233130700779,"signed_mean_gap":-0.4042233130700779,"absolute_mean_gap":0.4042233130700779,"bins":[{"index":0,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.0,"upper":0.1,"upper_inclusive":false},{"index":1,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.1,"upper":0.2,"upper_inclusive":false},{"index":2,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.2,"upper":0.3,"upper_inclusive":false},{"index":3,"count":7,"confidence":0.37857851447675284,"empirical_correctness":0.8571428571428571,"signed_gap":-0.47856434266610426,"absolute_gap":0.47856434266610426,"lower":0.3,"upper":0.4,"upper_inclusive":false},{"index":4,"count":21,"confidence":0.44925109141660446,"empirical_correctness":0.9523809523809523,"signed_gap":-0.5031298609643479,"absolute_gap":0.5031298609643479,"lower":0.4,"upper":0.5,"upper_inclusive":false},{"index":5,"count":7,"confidence":0.5246958033933754,"empirical_correctness":1.0,"signed_gap":-0.4753041966066246,"absolute_gap":0.4753041966066246,"lower":0.5,"upper":0.6,"upper_inclusive":false},{"index":6,"count":10,"confidence":0.6857137333576085,"empirical_correctness":1.0,"signed_gap":-0.31428626664239145,"absolute_gap":0.31428626664239145,"lower":0.6,"upper":0.7,"upper_inclusive":false},{"index":7,"count":4,"confidence":0.7495452606240378,"empirical_correctness":1.0,"signed_gap":-0.2504547393759622,"absolute_gap":0.2504547393759622,"lower":0.7,"upper":0.8,"upper_inclusive":false},{"index":8,"count":7,"confidence":0.8215689924519726,"empirical_correctness":1.0,"signed_gap":-0.17843100754802743,"absolute_gap":0.17843100754802743,"lower":0.8,"upper":0.9,"upper_inclusive":false},{"index":9,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.9,"upper":1.0,"upper_inclusive":true}]},"equal_mass":{"n":56,"bin_count":10,"ece":0.4042233130700779,"signed_mean_gap":-0.4042233130700779,"absolute_mean_gap":0.4042233130700779,"bins":[{"index":0,"count":5,"confidence":0.37225169724065277,"empirical_correctness":0.8,"signed_gap":-0.4277483027593473,"absolute_gap":0.4277483027593473,"min_confidence":0.3628965707811716,"max_confidence":0.38757202326360096},{"index":1,"count":6,"confidence":0.4058183502128025,"empirical_correctness":1.0,"signed_gap":-0.5941816497871975,"absolute_gap":0.5941816497871975,"min_confidence":0.39177731051764264,"max_confidence":0.4151394395287293},{"index":2,"count":5,"confidence":0.42905065110161145,"empirical_correctness":1.0,"signed_gap":-0.5709493488983886,"absolute_gap":0.5709493488983886,"min_confidence":0.4227599436330814,"max_confidence":0.4413597398600358},{"index":3,"count":6,"confidence":0.4565027233027647,"empirical_correctness":1.0,"signed_gap":-0.5434972766972352,"absolute_gap":0.5434972766972352,"min_confidence":0.44495386555143746,"max_confidence":0.46838718896847314},{"index":4,"count":6,"confidence":0.4839807230468732,"empirical_correctness":0.8333333333333334,"signed_gap":-0.3493526102864602,"absolute_gap":0.3493526102864602,"min_confidence":0.4755328547193531,"max_confidence":0.4979269204760343},{"index":5,"count":5,"confidence":0.5158433305678568,"empirical_correctness":1.0,"signed_gap":-0.4841566694321432,"absolute_gap":0.4841566694321432,"min_confidence":0.5050984925240184,"max_confidence":0.5237142787149366},{"index":6,"count":6,"confidence":0.6330564648089152,"empirical_correctness":1.0,"signed_gap":-0.3669435351910848,"absolute_gap":0.3669435351910848,"min_confidence":0.5263331603121703,"max_confidence":0.6882554052274023},{"index":7,"count":5,"confidence":0.6915084217679495,"empirical_correctness":1.0,"signed_gap":-0.30849157823205053,"absolute_gap":0.30849157823205053,"min_confidence":0.688831318828003,"max_confidence":0.6938906751674236},{"index":8,"count":6,"confidence":0.7507426190467551,"empirical_correctness":1.0,"signed_gap":-0.2492573809532449,"absolute_gap":0.2492573809532449,"min_confidence":0.6949104067971908,"max_confidence":0.8113642649871885},{"index":9,"count":6,"confidence":0.8232697803627699,"empirical_correctness":1.0,"signed_gap":-0.17673021963723012,"absolute_gap":0.17673021963723012,"min_confidence":0.8123713352695208,"max_confidence":0.8334716239822783}]}},"calibration_singleton_labels":{"equal_width":{"n":56,"bin_count":10,"ece":0.4042233130700779,"signed_mean_gap":-0.4042233130700779,"absolute_mean_gap":0.4042233130700779,"bins":[{"index":0,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.0,"upper":0.1,"upper_inclusive":false},{"index":1,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.1,"upper":0.2,"upper_inclusive":false},{"index":2,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.2,"upper":0.3,"upper_inclusive":false},{"index":3,"count":7,"confidence":0.37857851447675284,"empirical_correctness":0.8571428571428571,"signed_gap":-0.47856434266610426,"absolute_gap":0.47856434266610426,"lower":0.3,"upper":0.4,"upper_inclusive":false},{"index":4,"count":21,"confidence":0.44925109141660446,"empirical_correctness":0.9523809523809523,"signed_gap":-0.5031298609643479,"absolute_gap":0.5031298609643479,"lower":0.4,"upper":0.5,"upper_inclusive":false},{"index":5,"count":7,"confidence":0.5246958033933754,"empirical_correctness":1.0,"signed_gap":-0.4753041966066246,"absolute_gap":0.4753041966066246,"lower":0.5,"upper":0.6,"upper_inclusive":false},{"index":6,"count":10,"confidence":0.6857137333576085,"empirical_correctness":1.0,"signed_gap":-0.31428626664239145,"absolute_gap":0.31428626664239145,"lower":0.6,"upper":0.7,"upper_inclusive":false},{"index":7,"count":4,"confidence":0.7495452606240378,"empirical_correctness":1.0,"signed_gap":-0.2504547393759622,"absolute_gap":0.2504547393759622,"lower":0.7,"upper":0.8,"upper_inclusive":false},{"index":8,"count":7,"confidence":0.8215689924519726,"empirical_correctness":1.0,"signed_gap":-0.17843100754802743,"absolute_gap":0.17843100754802743,"lower":0.8,"upper":0.9,"upper_inclusive":false},{"index":9,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.9,"upper":1.0,"upper_inclusive":true}]},"equal_mass":{"n":56,"bin_count":10,"ece":0.4042233130700779,"signed_mean_gap":-0.4042233130700779,"absolute_mean_gap":0.4042233130700779,"bins":[{"index":0,"count":5,"confidence":0.37225169724065277,"empirical_correctness":0.8,"signed_gap":-0.4277483027593473,"absolute_gap":0.4277483027593473,"min_confidence":0.3628965707811716,"max_confidence":0.38757202326360096},{"index":1,"count":6,"confidence":0.4058183502128025,"empirical_correctness":1.0,"signed_gap":-0.5941816497871975,"absolute_gap":0.5941816497871975,"min_confidence":0.39177731051764264,"max_confidence":0.4151394395287293},{"index":2,"count":5,"confidence":0.42905065110161145,"empirical_correctness":1.0,"signed_gap":-0.5709493488983886,"absolute_gap":0.5709493488983886,"min_confidence":0.4227599436330814,"max_confidence":0.4413597398600358},{"index":3,"count":6,"confidence":0.4565027233027647,"empirical_correctness":1.0,"signed_gap":-0.5434972766972352,"absolute_gap":0.5434972766972352,"min_confidence":0.44495386555143746,"max_confidence":0.46838718896847314},{"index":4,"count":6,"confidence":0.4839807230468732,"empirical_correctness":0.8333333333333334,"signed_gap":-0.3493526102864602,"absolute_gap":0.3493526102864602,"min_confidence":0.4755328547193531,"max_confidence":0.4979269204760343},{"index":5,"count":5,"confidence":0.5158433305678568,"empirical_correctness":1.0,"signed_gap":-0.4841566694321432,"absolute_gap":0.4841566694321432,"min_confidence":0.5050984925240184,"max_confidence":0.5237142787149366},{"index":6,"count":6,"confidence":0.6330564648089152,"empirical_correctness":1.0,"signed_gap":-0.3669435351910848,"absolute_gap":0.3669435351910848,"min_confidence":0.5263331603121703,"max_confidence":0.6882554052274023},{"index":7,"count":5,"confidence":0.6915084217679495,"empirical_correctness":1.0,"signed_gap":-0.30849157823205053,"absolute_gap":0.30849157823205053,"min_confidence":0.688831318828003,"max_confidence":0.6938906751674236},{"index":8,"count":6,"confidence":0.7507426190467551,"empirical_correctness":1.0,"signed_gap":-0.2492573809532449,"absolute_gap":0.2492573809532449,"min_confidence":0.6949104067971908,"max_confidence":0.8113642649871885},{"index":9,"count":6,"confidence":0.8232697803627699,"empirical_correctness":1.0,"signed_gap":-0.17673021963723012,"absolute_gap":0.17673021963723012,"min_confidence":0.8123713352695208,"max_confidence":0.8334716239822783}]}},"latency_ms":{"n":56,"median":28.95996702136472,"p95":29.241091484436765},"per_action":{"keep":{"n":8,"raw":{"n":8,"acceptable":8,"unsafe":0,"review":0,"automated_count":8,"automated_acceptable":8,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":1.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.6755924351161198,"upper":1.0,"confidence_level":0.95}},"effective":{"n":8,"acceptable":2,"unsafe":0,"review":6,"automated_count":2,"automated_acceptable":2,"automated_unsafe":0,"unnecessary_reviews":6,"accuracy":0.25,"unsafe_rate":0.0,"review_rate":0.75,"automated_coverage":0.25,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.75,"accuracy_wilson_95":{"lower":0.071479212752109,"upper":0.5907245696898311,"confidence_level":0.95}}},"drop":{"n":22,"raw":{"n":22,"acceptable":22,"unsafe":0,"review":0,"automated_count":22,"automated_acceptable":22,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":1.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.8513451253913791,"upper":1.0,"confidence_level":0.95}},"effective":{"n":22,"acceptable":17,"unsafe":0,"review":5,"automated_count":17,"automated_acceptable":17,"automated_unsafe":0,"unnecessary_reviews":5,"accuracy":0.7727272727272727,"unsafe_rate":0.0,"review_rate":0.22727272727272727,"automated_coverage":0.7727272727272727,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.22727272727272727,"accuracy_wilson_95":{"lower":0.5656004682477829,"upper":0.8987696001475147,"confidence_level":0.95}}},"mask":{"n":8,"raw":{"n":8,"acceptable":8,"unsafe":0,"review":0,"automated_count":8,"automated_acceptable":8,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":1.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.6755924351161198,"upper":1.0,"confidence_level":0.95}},"effective":{"n":8,"acceptable":8,"unsafe":0,"review":0,"automated_count":8,"automated_acceptable":8,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":1.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.6755924351161198,"upper":1.0,"confidence_level":0.95}}},"pseudonymize":{"n":8,"raw":{"n":8,"acceptable":8,"unsafe":0,"review":0,"automated_count":8,"automated_acceptable":8,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":1.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.6755924351161198,"upper":1.0,"confidence_level":0.95}},"effective":{"n":8,"acceptable":1,"unsafe":0,"review":7,"automated_count":1,"automated_acceptable":1,"automated_unsafe":0,"unnecessary_reviews":7,"accuracy":0.125,"unsafe_rate":0.0,"review_rate":0.875,"automated_coverage":0.125,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.875,"accuracy_wilson_95":{"lower":0.02241749145005667,"upper":0.4708881822128535,"confidence_level":0.95}}},"generalize":{"n":8,"raw":{"n":8,"acceptable":6,"unsafe":0,"review":0,"automated_count":8,"automated_acceptable":6,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":0.75,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.75,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.40927543031016883,"upper":0.9285207872478909,"confidence_level":0.95}},"effective":{"n":8,"acceptable":1,"unsafe":0,"review":6,"automated_count":2,"automated_acceptable":1,"automated_unsafe":0,"unnecessary_reviews":6,"accuracy":0.125,"unsafe_rate":0.0,"review_rate":0.75,"automated_coverage":0.25,"conditional_correctness":0.5,"conditional_unsafe":0.0,"unnecessary_review_rate":0.75,"accuracy_wilson_95":{"lower":0.02241749145005667,"upper":0.4708881822128535,"confidence_level":0.95}}},"review":{"n":2,"raw":{"n":2,"acceptable":2,"unsafe":0,"review":2,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":1.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.34238022750665303,"upper":1.0,"confidence_level":0.95}},"effective":{"n":2,"acceptable":2,"unsafe":0,"review":2,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":1.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.34238022750665303,"upper":1.0,"confidence_level":0.95}}}}},"challenge56-compact_no_samples":{"n":56,"raw":{"n":56,"acceptable":53,"unsafe":0,"review":2,"automated_count":54,"automated_acceptable":51,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":0.9464285714285714,"unsafe_rate":0.0,"review_rate":0.03571428571428571,"automated_coverage":0.9642857142857143,"conditional_correctness":0.9444444444444444,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.8539269093117847,"upper":0.9816142186178909,"confidence_level":0.95}},"effective":{"n":56,"acceptable":40,"unsafe":0,"review":18,"automated_count":38,"automated_acceptable":38,"automated_unsafe":0,"unnecessary_reviews":16,"accuracy":0.7142857142857143,"unsafe_rate":0.0,"review_rate":0.32142857142857145,"automated_coverage":0.6785714285714286,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.2857142857142857,"accuracy_wilson_95":{"lower":0.5852474928448334,"upper":0.8158122485614109,"confidence_level":0.95}},"confusion":{"raw":{"keep":{"keep":7,"drop":1,"mask":0,"pseudonymize":0,"generalize":0,"review":0},"drop":{"keep":0,"drop":22,"mask":0,"pseudonymize":0,"generalize":0,"review":0},"mask":{"keep":0,"drop":0,"mask":8,"pseudonymize":0,"generalize":0,"review":0},"pseudonymize":{"keep":0,"drop":0,"mask":0,"pseudonymize":8,"generalize":0,"review":0},"generalize":{"keep":0,"drop":2,"mask":0,"pseudonymize":0,"generalize":6,"review":0},"review":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":2}},"effective":{"keep":{"keep":3,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":5},"drop":{"keep":0,"drop":22,"mask":0,"pseudonymize":0,"generalize":0,"review":0},"mask":{"keep":0,"drop":0,"mask":8,"pseudonymize":0,"generalize":0,"review":0},"pseudonymize":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":8},"generalize":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":5,"review":3},"review":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":2}}},"probability_metrics":{"singleton_n":56,"multiacceptable_n":0,"multiacceptable_coverage":0.0,"nll":0.6729844857897822,"nll_zero_mass_n":0,"brier":0.3047166819060941,"acceptable_set_n":56,"acceptable_set_nll":0.6729844857897822,"acceptable_set_zero_mass_n":0},"calibration_all_labels":{"equal_width":{"n":56,"bin_count":10,"ece":0.39257460508751535,"signed_mean_gap":-0.39257460508751535,"absolute_mean_gap":0.39257460508751535,"bins":[{"index":0,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.0,"upper":0.1,"upper_inclusive":false},{"index":1,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.1,"upper":0.2,"upper_inclusive":false},{"index":2,"count":2,"confidence":0.28269738661791555,"empirical_correctness":1.0,"signed_gap":-0.7173026133820845,"absolute_gap":0.7173026133820845,"lower":0.2,"upper":0.3,"upper_inclusive":false},{"index":3,"count":9,"confidence":0.3490394402572424,"empirical_correctness":0.7777777777777778,"signed_gap":-0.4287383375205354,"absolute_gap":0.4287383375205354,"lower":0.3,"upper":0.4,"upper_inclusive":false},{"index":4,"count":13,"confidence":0.4553390922154786,"empirical_correctness":0.9230769230769231,"signed_gap":-0.46773783086144455,"absolute_gap":0.46773783086144455,"lower":0.4,"upper":0.5,"upper_inclusive":false},{"index":5,"count":9,"confidence":0.5263764449264817,"empirical_correctness":1.0,"signed_gap":-0.4736235550735183,"absolute_gap":0.4736235550735183,"lower":0.5,"upper":0.6,"upper_inclusive":false},{"index":6,"count":9,"confidence":0.667402510650026,"empirical_correctness":1.0,"signed_gap":-0.332597489349974,"absolute_gap":0.332597489349974,"lower":0.6,"upper":0.7,"upper_inclusive":false},{"index":7,"count":13,"confidence":0.757190840681598,"empirical_correctness":1.0,"signed_gap":-0.24280915931840197,"absolute_gap":0.24280915931840197,"lower":0.7,"upper":0.8,"upper_inclusive":false},{"index":8,"count":1,"confidence":0.8021726516975664,"empirical_correctness":1.0,"signed_gap":-0.19782734830243365,"absolute_gap":0.19782734830243365,"lower":0.8,"upper":0.9,"upper_inclusive":false},{"index":9,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.9,"upper":1.0,"upper_inclusive":true}]},"equal_mass":{"n":56,"bin_count":10,"ece":0.3925746050875153,"signed_mean_gap":-0.3925746050875153,"absolute_mean_gap":0.3925746050875153,"bins":[{"index":0,"count":5,"confidence":0.30702626024888163,"empirical_correctness":1.0,"signed_gap":-0.6929737397511184,"absolute_gap":0.6929737397511184,"min_confidence":0.2705986044494918,"max_confidence":0.33029484061660425},{"index":1,"count":6,"confidence":0.3619364057177674,"empirical_correctness":0.6666666666666666,"signed_gap":-0.3047302609488992,"absolute_gap":0.3047302609488992,"min_confidence":0.3356520171773779,"max_confidence":0.38670572867859243},{"index":2,"count":5,"confidence":0.42753955755178924,"empirical_correctness":0.8,"signed_gap":-0.3724604424482108,"absolute_gap":0.3724604424482108,"min_confidence":0.40043656862055305,"max_confidence":0.4518940505530784},{"index":3,"count":6,"confidence":0.46634283035480256,"empirical_correctness":1.0,"signed_gap":-0.5336571696451975,"absolute_gap":0.5336571696451975,"min_confidence":0.45429669890598695,"max_confidence":0.4795022974229463},{"index":4,"count":6,"confidence":0.5020506179410129,"empirical_correctness":1.0,"signed_gap":-0.4979493820589871,"absolute_gap":0.4979493820589871,"min_confidence":0.4859403339878747,"max_confidence":0.5158116358146482},{"index":5,"count":5,"confidence":0.5417475451211435,"empirical_correctness":1.0,"signed_gap":-0.45825245487885646,"absolute_gap":0.45825245487885646,"min_confidence":0.5209617079625253,"max_confidence":0.5648847577649965},{"index":6,"count":6,"confidence":0.658370844866594,"empirical_correctness":1.0,"signed_gap":-0.34162915513340597,"absolute_gap":0.34162915513340597,"min_confidence":0.6091060625273915,"max_confidence":0.6731241989940535},{"index":7,"count":5,"confidence":0.6943974709330805,"empirical_correctness":1.0,"signed_gap":-0.30560252906691954,"absolute_gap":0.30560252906691954,"min_confidence":0.6734517117651359,"max_confidence":0.7110733260521914},{"index":8,"count":6,"confidence":0.7483944367452633,"empirical_correctness":1.0,"signed_gap":-0.2516055632547367,"absolute_gap":0.2516055632547367,"min_confidence":0.7289342972700266,"max_confidence":0.7708069288049455},{"index":9,"count":6,"confidence":0.7899495220120046,"empirical_correctness":1.0,"signed_gap":-0.21005047798799537,"absolute_gap":0.21005047798799537,"min_confidence":0.7737773065189293,"max_confidence":0.8021726516975664}]}},"calibration_singleton_labels":{"equal_width":{"n":56,"bin_count":10,"ece":0.39257460508751535,"signed_mean_gap":-0.39257460508751535,"absolute_mean_gap":0.39257460508751535,"bins":[{"index":0,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.0,"upper":0.1,"upper_inclusive":false},{"index":1,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.1,"upper":0.2,"upper_inclusive":false},{"index":2,"count":2,"confidence":0.28269738661791555,"empirical_correctness":1.0,"signed_gap":-0.7173026133820845,"absolute_gap":0.7173026133820845,"lower":0.2,"upper":0.3,"upper_inclusive":false},{"index":3,"count":9,"confidence":0.3490394402572424,"empirical_correctness":0.7777777777777778,"signed_gap":-0.4287383375205354,"absolute_gap":0.4287383375205354,"lower":0.3,"upper":0.4,"upper_inclusive":false},{"index":4,"count":13,"confidence":0.4553390922154786,"empirical_correctness":0.9230769230769231,"signed_gap":-0.46773783086144455,"absolute_gap":0.46773783086144455,"lower":0.4,"upper":0.5,"upper_inclusive":false},{"index":5,"count":9,"confidence":0.5263764449264817,"empirical_correctness":1.0,"signed_gap":-0.4736235550735183,"absolute_gap":0.4736235550735183,"lower":0.5,"upper":0.6,"upper_inclusive":false},{"index":6,"count":9,"confidence":0.667402510650026,"empirical_correctness":1.0,"signed_gap":-0.332597489349974,"absolute_gap":0.332597489349974,"lower":0.6,"upper":0.7,"upper_inclusive":false},{"index":7,"count":13,"confidence":0.757190840681598,"empirical_correctness":1.0,"signed_gap":-0.24280915931840197,"absolute_gap":0.24280915931840197,"lower":0.7,"upper":0.8,"upper_inclusive":false},{"index":8,"count":1,"confidence":0.8021726516975664,"empirical_correctness":1.0,"signed_gap":-0.19782734830243365,"absolute_gap":0.19782734830243365,"lower":0.8,"upper":0.9,"upper_inclusive":false},{"index":9,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.9,"upper":1.0,"upper_inclusive":true}]},"equal_mass":{"n":56,"bin_count":10,"ece":0.3925746050875153,"signed_mean_gap":-0.3925746050875153,"absolute_mean_gap":0.3925746050875153,"bins":[{"index":0,"count":5,"confidence":0.30702626024888163,"empirical_correctness":1.0,"signed_gap":-0.6929737397511184,"absolute_gap":0.6929737397511184,"min_confidence":0.2705986044494918,"max_confidence":0.33029484061660425},{"index":1,"count":6,"confidence":0.3619364057177674,"empirical_correctness":0.6666666666666666,"signed_gap":-0.3047302609488992,"absolute_gap":0.3047302609488992,"min_confidence":0.3356520171773779,"max_confidence":0.38670572867859243},{"index":2,"count":5,"confidence":0.42753955755178924,"empirical_correctness":0.8,"signed_gap":-0.3724604424482108,"absolute_gap":0.3724604424482108,"min_confidence":0.40043656862055305,"max_confidence":0.4518940505530784},{"index":3,"count":6,"confidence":0.46634283035480256,"empirical_correctness":1.0,"signed_gap":-0.5336571696451975,"absolute_gap":0.5336571696451975,"min_confidence":0.45429669890598695,"max_confidence":0.4795022974229463},{"index":4,"count":6,"confidence":0.5020506179410129,"empirical_correctness":1.0,"signed_gap":-0.4979493820589871,"absolute_gap":0.4979493820589871,"min_confidence":0.4859403339878747,"max_confidence":0.5158116358146482},{"index":5,"count":5,"confidence":0.5417475451211435,"empirical_correctness":1.0,"signed_gap":-0.45825245487885646,"absolute_gap":0.45825245487885646,"min_confidence":0.5209617079625253,"max_confidence":0.5648847577649965},{"index":6,"count":6,"confidence":0.658370844866594,"empirical_correctness":1.0,"signed_gap":-0.34162915513340597,"absolute_gap":0.34162915513340597,"min_confidence":0.6091060625273915,"max_confidence":0.6731241989940535},{"index":7,"count":5,"confidence":0.6943974709330805,"empirical_correctness":1.0,"signed_gap":-0.30560252906691954,"absolute_gap":0.30560252906691954,"min_confidence":0.6734517117651359,"max_confidence":0.7110733260521914},{"index":8,"count":6,"confidence":0.7483944367452633,"empirical_correctness":1.0,"signed_gap":-0.2516055632547367,"absolute_gap":0.2516055632547367,"min_confidence":0.7289342972700266,"max_confidence":0.7708069288049455},{"index":9,"count":6,"confidence":0.7899495220120046,"empirical_correctness":1.0,"signed_gap":-0.21005047798799537,"absolute_gap":0.21005047798799537,"min_confidence":0.7737773065189293,"max_confidence":0.8021726516975664}]}},"latency_ms":{"n":56,"median":32.87161950720474,"p95":33.2370089890901},"per_action":{"keep":{"n":8,"raw":{"n":8,"acceptable":7,"unsafe":0,"review":0,"automated_count":8,"automated_acceptable":7,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":0.875,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.875,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.5291118177871464,"upper":0.9775825085499433,"confidence_level":0.95}},"effective":{"n":8,"acceptable":3,"unsafe":0,"review":5,"automated_count":3,"automated_acceptable":3,"automated_unsafe":0,"unnecessary_reviews":5,"accuracy":0.375,"unsafe_rate":0.0,"review_rate":0.625,"automated_coverage":0.375,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.625,"accuracy_wilson_95":{"lower":0.13684428582359742,"upper":0.6942576053973727,"confidence_level":0.95}}},"drop":{"n":22,"raw":{"n":22,"acceptable":22,"unsafe":0,"review":0,"automated_count":22,"automated_acceptable":22,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":1.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.8513451253913791,"upper":1.0,"confidence_level":0.95}},"effective":{"n":22,"acceptable":22,"unsafe":0,"review":0,"automated_count":22,"automated_acceptable":22,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":1.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.8513451253913791,"upper":1.0,"confidence_level":0.95}}},"mask":{"n":8,"raw":{"n":8,"acceptable":8,"unsafe":0,"review":0,"automated_count":8,"automated_acceptable":8,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":1.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.6755924351161198,"upper":1.0,"confidence_level":0.95}},"effective":{"n":8,"acceptable":8,"unsafe":0,"review":0,"automated_count":8,"automated_acceptable":8,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":1.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.6755924351161198,"upper":1.0,"confidence_level":0.95}}},"pseudonymize":{"n":8,"raw":{"n":8,"acceptable":8,"unsafe":0,"review":0,"automated_count":8,"automated_acceptable":8,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":1.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.6755924351161198,"upper":1.0,"confidence_level":0.95}},"effective":{"n":8,"acceptable":0,"unsafe":0,"review":8,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":8,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.32440756488388023,"confidence_level":0.95}}},"generalize":{"n":8,"raw":{"n":8,"acceptable":6,"unsafe":0,"review":0,"automated_count":8,"automated_acceptable":6,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":0.75,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.75,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.40927543031016883,"upper":0.9285207872478909,"confidence_level":0.95}},"effective":{"n":8,"acceptable":5,"unsafe":0,"review":3,"automated_count":5,"automated_acceptable":5,"automated_unsafe":0,"unnecessary_reviews":3,"accuracy":0.625,"unsafe_rate":0.0,"review_rate":0.375,"automated_coverage":0.625,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.375,"accuracy_wilson_95":{"lower":0.30574239460262737,"upper":0.8631557141764026,"confidence_level":0.95}}},"review":{"n":2,"raw":{"n":2,"acceptable":2,"unsafe":0,"review":2,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":1.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.34238022750665303,"upper":1.0,"confidence_level":0.95}},"effective":{"n":2,"acceptable":2,"unsafe":0,"review":2,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":1.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.34238022750665303,"upper":1.0,"confidence_level":0.95}}}}}},"composite":{"n":198,"scope":"descriptive sum across four correlated panels","raw":{"n":198,"acceptable":137,"unsafe":2,"review":4,"automated_count":194,"automated_acceptable":133,"automated_unsafe":2,"unnecessary_reviews":0,"accuracy":0.6919191919191919,"unsafe_rate":0.010101010101010102,"review_rate":0.020202020202020204,"automated_coverage":0.9797979797979798,"conditional_correctness":0.6855670103092784,"conditional_unsafe":0.010309278350515464,"unnecessary_review_rate":0.0},"effective":{"n":198,"acceptable":90,"unsafe":0,"review":104,"automated_count":94,"automated_acceptable":82,"automated_unsafe":0,"unnecessary_reviews":96,"accuracy":0.45454545454545453,"unsafe_rate":0.0,"review_rate":0.5252525252525253,"automated_coverage":0.47474747474747475,"conditional_correctness":0.8723404255319149,"conditional_unsafe":0.0,"unnecessary_review_rate":0.48484848484848486}}},{"id":"internlm--Intern-Decision-4B-nf4","platform":"Local","model_id":"internlm/Intern-Decision-4B","revision":"0e5e6aa7d6d750e2b1504ba11a8136cb58aeb3cd","loading":"bfloat16 / nf4","scoring_kind":"Native decision readout","settings":{"dtype":"bfloat16","quantization":"nf4","max_tokens":4096,"effective_max_tokens":4096,"temperature":1.99241824,"cpu_offload":false,"scoring_batch_size":1,"trust_remote_code":false,"local_files_only":true,"use_safetensors":true,"guard_threshold":0.45,"seed":0,"tf32":false},"scoring_metadata":{"backend":"intern_decision","protocol":"hf-local-sweep-native-v1","readout":"builtin_logits_immediately_before_decision","probability_source":"native_choice_published_temperature_not_privacy_calibrated","latency_scope":"synchronous_forward_scoring_including_transfer_excluding_tokenization","precision":"bfloat16","equal_mass_convention":"rank bins with floor boundaries at floor(i*n/k), k=min(10,n//2); minimum two rows"},"load_ms":7315.420111990534,"peak_allocated_bytes":3477287936,"peak_reserved_bytes":3539992576,"panels":{"diagnostic43-no_samples":{"n":43,"raw":{"n":43,"acceptable":28,"unsafe":1,"review":0,"automated_count":43,"automated_acceptable":28,"automated_unsafe":1,"unnecessary_reviews":0,"accuracy":0.6511627906976745,"unsafe_rate":0.023255813953488372,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.6511627906976745,"conditional_unsafe":0.023255813953488372,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.5017178199285205,"upper":0.775814100026473,"confidence_level":0.95}},"effective":{"n":43,"acceptable":28,"unsafe":1,"review":7,"automated_count":36,"automated_acceptable":28,"automated_unsafe":1,"unnecessary_reviews":7,"accuracy":0.6511627906976745,"unsafe_rate":0.023255813953488372,"review_rate":0.16279069767441862,"automated_coverage":0.8372093023255814,"conditional_correctness":0.7777777777777778,"conditional_unsafe":0.027777777777777776,"unnecessary_review_rate":0.16279069767441862,"accuracy_wilson_95":{"lower":0.5017178199285205,"upper":0.775814100026473,"confidence_level":0.95}},"confusion":{"raw":{"keep":{"keep":0,"drop":2,"mask":6,"pseudonymize":3,"generalize":1,"review":0},"drop":{"keep":0,"drop":13,"mask":1,"pseudonymize":1,"generalize":1,"review":0},"mask":{"keep":0,"drop":0,"mask":2,"pseudonymize":0,"generalize":0,"review":0},"pseudonymize":{"keep":0,"drop":0,"mask":0,"pseudonymize":2,"generalize":0,"review":0},"generalize":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":6,"review":0},"review":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0}},"effective":{"keep":{"keep":0,"drop":0,"mask":6,"pseudonymize":0,"generalize":0,"review":6},"drop":{"keep":0,"drop":13,"mask":1,"pseudonymize":1,"generalize":0,"review":1},"mask":{"keep":0,"drop":0,"mask":2,"pseudonymize":0,"generalize":0,"review":0},"pseudonymize":{"keep":0,"drop":0,"mask":0,"pseudonymize":2,"generalize":0,"review":0},"generalize":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":6,"review":0},"review":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0}}},"probability_metrics":{"singleton_n":38,"multiacceptable_n":5,"multiacceptable_coverage":0.11627906976744186,"nll":1.3215986564190876,"nll_zero_mass_n":0,"brier":0.5560454619301702,"acceptable_set_n":43,"acceptable_set_nll":1.1725050975957914,"acceptable_set_zero_mass_n":0},"calibration_all_labels":{"equal_width":{"n":43,"bin_count":10,"ece":0.13825172140458067,"signed_mean_gap":0.06528430419499438,"absolute_mean_gap":0.06528430419499438,"bins":[{"index":0,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.0,"upper":0.1,"upper_inclusive":false},{"index":1,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.1,"upper":0.2,"upper_inclusive":false},{"index":2,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.2,"upper":0.3,"upper_inclusive":false},{"index":3,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.3,"upper":0.4,"upper_inclusive":false},{"index":4,"count":4,"confidence":0.45199441565915066,"empirical_correctness":0.25,"signed_gap":0.20199441565915066,"absolute_gap":0.20199441565915066,"lower":0.4,"upper":0.5,"upper_inclusive":false},{"index":5,"count":9,"confidence":0.557550644395555,"empirical_correctness":0.5555555555555556,"signed_gap":0.001995088839999437,"absolute_gap":0.001995088839999437,"lower":0.5,"upper":0.6,"upper_inclusive":false},{"index":6,"count":3,"confidence":0.6665732030543413,"empirical_correctness":0.3333333333333333,"signed_gap":0.333239869721008,"absolute_gap":0.333239869721008,"lower":0.6,"upper":0.7,"upper_inclusive":false},{"index":7,"count":10,"confidence":0.7550371479031242,"empirical_correctness":0.5,"signed_gap":0.25503714790312415,"absolute_gap":0.25503714790312415,"lower":0.7,"upper":0.8,"upper_inclusive":false},{"index":8,"count":14,"confidence":0.8330136702816596,"empirical_correctness":0.9285714285714286,"signed_gap":-0.09555775828976898,"absolute_gap":0.09555775828976898,"lower":0.8,"upper":0.9,"upper_inclusive":false},{"index":9,"count":3,"confidence":0.9230030486835535,"empirical_correctness":1.0,"signed_gap":-0.07699695131644646,"absolute_gap":0.07699695131644646,"lower":0.9,"upper":1.0,"upper_inclusive":true}]},"equal_mass":{"n":43,"bin_count":10,"ece":0.11680395507099099,"signed_mean_gap":0.06528430419499438,"absolute_mean_gap":0.06528430419499438,"bins":[{"index":0,"count":4,"confidence":0.45199441565915066,"empirical_correctness":0.25,"signed_gap":0.20199441565915066,"absolute_gap":0.20199441565915066,"min_confidence":0.4206501647204102,"max_confidence":0.4948150169611263},{"index":1,"count":4,"confidence":0.5361566386180385,"empirical_correctness":0.5,"signed_gap":0.03615663861803853,"absolute_gap":0.03615663861803853,"min_confidence":0.5173741867367083,"max_confidence":0.5549778612214851},{"index":2,"count":4,"confidence":0.5696625864220953,"empirical_correctness":0.5,"signed_gap":0.06966258642209533,"absolute_gap":0.06966258642209533,"min_confidence":0.5617868071360469,"max_confidence":0.5762321412496465},{"index":3,"count":5,"confidence":0.6619120297226162,"empirical_correctness":0.6,"signed_gap":0.061912029722616224,"absolute_gap":0.061912029722616224,"min_confidence":0.5946788993994594,"max_confidence":0.715161640050598},{"index":4,"count":4,"confidence":0.7382427894953796,"empirical_correctness":0.5,"signed_gap":0.23824278949537958,"absolute_gap":0.23824278949537958,"min_confidence":0.7199763577304358,"max_confidence":0.7596982060979095},{"index":5,"count":4,"confidence":0.7729813340640814,"empirical_correctness":0.5,"signed_gap":0.2729813340640814,"absolute_gap":0.2729813340640814,"min_confidence":0.7630553624704511,"max_confidence":0.7853273535324975},{"index":6,"count":5,"confidence":0.8019571461912097,"empirical_correctness":0.8,"signed_gap":0.00195714619120968,"absolute_gap":0.00195714619120968,"min_confidence":0.790313344742799,"max_confidence":0.8066391253186229},{"index":7,"count":4,"confidence":0.8298501594036434,"empirical_correctness":0.75,"signed_gap":0.07985015940364337,"absolute_gap":0.07985015940364337,"min_confidence":0.8283721666461699,"max_confidence":0.8317093781127357},{"index":8,"count":4,"confidence":0.8457171477114623,"empirical_correctness":1.0,"signed_gap":-0.15428285228853766,"absolute_gap":0.15428285228853766,"min_confidence":0.8408115165839404,"max_confidence":0.8536745046843833},{"index":9,"count":5,"confidence":0.9018917830640447,"empirical_correctness":1.0,"signed_gap":-0.09810821693595528,"absolute_gap":0.09810821693595528,"min_confidence":0.870169824477564,"max_confidence":0.9428342076637269}]}},"calibration_singleton_labels":{"equal_width":{"n":38,"bin_count":10,"ece":0.18350514848495392,"signed_mean_gap":0.11379346765412135,"absolute_mean_gap":0.11379346765412135,"bins":[{"index":0,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.0,"upper":0.1,"upper_inclusive":false},{"index":1,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.1,"upper":0.2,"upper_inclusive":false},{"index":2,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.2,"upper":0.3,"upper_inclusive":false},{"index":3,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.3,"upper":0.4,"upper_inclusive":false},{"index":4,"count":3,"confidence":0.43772088189182545,"empirical_correctness":0.0,"signed_gap":0.43772088189182545,"absolute_gap":0.43772088189182545,"lower":0.4,"upper":0.5,"upper_inclusive":false},{"index":5,"count":8,"confidence":0.5625727016029108,"empirical_correctness":0.5,"signed_gap":0.06257270160291084,"absolute_gap":0.06257270160291084,"lower":0.5,"upper":0.6,"upper_inclusive":false},{"index":6,"count":3,"confidence":0.6665732030543413,"empirical_correctness":0.3333333333333333,"signed_gap":0.333239869721008,"absolute_gap":0.333239869721008,"lower":0.6,"upper":0.7,"upper_inclusive":false},{"index":7,"count":9,"confidence":0.7594677598867381,"empirical_correctness":0.4444444444444444,"signed_gap":0.31502331544229367,"absolute_gap":0.31502331544229367,"lower":0.7,"upper":0.8,"upper_inclusive":false},{"index":8,"count":13,"confidence":0.8322820525174122,"empirical_correctness":0.9230769230769231,"signed_gap":-0.09079487055951097,"absolute_gap":0.09079487055951097,"lower":0.8,"upper":0.9,"upper_inclusive":false},{"index":9,"count":2,"confidence":0.9279056907439118,"empirical_correctness":1.0,"signed_gap":-0.07209430925608817,"absolute_gap":0.07209430925608817,"lower":0.9,"upper":1.0,"upper_inclusive":true}]},"equal_mass":{"n":38,"bin_count":10,"ece":0.16809270878574795,"signed_mean_gap":0.11379346765412139,"absolute_mean_gap":0.11379346765412139,"bins":[{"index":0,"count":3,"confidence":0.43772088189182545,"empirical_correctness":0.0,"signed_gap":0.43772088189182545,"absolute_gap":0.43772088189182545,"min_confidence":0.4206501647204102,"max_confidence":0.4487399223702628},{"index":1,"count":4,"confidence":0.5472597937178733,"empirical_correctness":0.5,"signed_gap":0.047259793717873255,"absolute_gap":0.047259793717873255,"min_confidence":0.5287124315063948,"max_confidence":0.5617868071360469},{"index":2,"count":4,"confidence":0.5778856094879484,"empirical_correctness":0.5,"signed_gap":0.07788560948794843,"absolute_gap":0.07788560948794843,"min_confidence":0.5659381656696777,"max_confidence":0.5946788993994594},{"index":3,"count":4,"confidence":0.6799239917233649,"empirical_correctness":0.25,"signed_gap":0.42992399172336493,"absolute_gap":0.42992399172336493,"min_confidence":0.6065887229968355,"max_confidence":0.7199763577304358},{"index":4,"count":4,"confidence":0.7490125406803835,"empirical_correctness":0.5,"signed_gap":0.24901254068038348,"absolute_gap":0.24901254068038348,"min_confidence":0.7314135245526479,"max_confidence":0.7630553624704511},{"index":5,"count":3,"confidence":0.7762899912619582,"empirical_correctness":0.6666666666666666,"signed_gap":0.10962332459529156,"absolute_gap":0.10962332459529156,"min_confidence":0.7692564542996909,"max_confidence":0.7853273535324975},{"index":6,"count":4,"confidence":0.8007866514093565,"empirical_correctness":0.75,"signed_gap":0.050786651409356454,"absolute_gap":0.050786651409356454,"min_confidence":0.790313344742799,"max_confidence":0.8063214544716586},{"index":7,"count":4,"confidence":0.8235825962051151,"empirical_correctness":0.75,"signed_gap":0.07358259620511509,"absolute_gap":0.07358259620511509,"min_confidence":0.8066391253186229,"max_confidence":0.8305214212534053},{"index":8,"count":4,"confidence":0.8430133169354271,"empirical_correctness":1.0,"signed_gap":-0.15698668306457286,"absolute_gap":0.15698668306457286,"min_confidence":0.8317093781127357,"max_confidence":0.8536745046843833},{"index":9,"count":4,"confidence":0.8990652876893466,"empirical_correctness":1.0,"signed_gap":-0.10093471231065343,"absolute_gap":0.10093471231065343,"min_confidence":0.870169824477564,"max_confidence":0.9428342076637269}]}},"latency_ms":{"n":43,"median":119.39925199840218,"p95":120.43879647389986},"per_action":{"keep":{"n":12,"raw":{"n":12,"acceptable":0,"unsafe":0,"review":0,"automated_count":12,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":2.7755575615628914e-17,"upper":0.24249400665524085,"confidence_level":0.95}},"effective":{"n":12,"acceptable":0,"unsafe":0,"review":6,"automated_count":6,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":6,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":0.5,"automated_coverage":0.5,"conditional_correctness":0.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.5,"accuracy_wilson_95":{"lower":2.7755575615628914e-17,"upper":0.24249400665524085,"confidence_level":0.95}}},"drop":{"n":16,"raw":{"n":16,"acceptable":13,"unsafe":1,"review":0,"automated_count":16,"automated_acceptable":13,"automated_unsafe":1,"unnecessary_reviews":0,"accuracy":0.8125,"unsafe_rate":0.0625,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.8125,"conditional_unsafe":0.0625,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.5699111903802586,"upper":0.9340840092857187,"confidence_level":0.95}},"effective":{"n":16,"acceptable":13,"unsafe":1,"review":1,"automated_count":15,"automated_acceptable":13,"automated_unsafe":1,"unnecessary_reviews":1,"accuracy":0.8125,"unsafe_rate":0.0625,"review_rate":0.0625,"automated_coverage":0.9375,"conditional_correctness":0.8666666666666667,"conditional_unsafe":0.06666666666666667,"unnecessary_review_rate":0.0625,"accuracy_wilson_95":{"lower":0.5699111903802586,"upper":0.9340840092857187,"confidence_level":0.95}}},"mask":{"n":2,"raw":{"n":2,"acceptable":2,"unsafe":0,"review":0,"automated_count":2,"automated_acceptable":2,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":1.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.34238022750665303,"upper":1.0,"confidence_level":0.95}},"effective":{"n":2,"acceptable":2,"unsafe":0,"review":0,"automated_count":2,"automated_acceptable":2,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":1.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.34238022750665303,"upper":1.0,"confidence_level":0.95}}},"pseudonymize":{"n":2,"raw":{"n":2,"acceptable":2,"unsafe":0,"review":0,"automated_count":2,"automated_acceptable":2,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":1.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.34238022750665303,"upper":1.0,"confidence_level":0.95}},"effective":{"n":2,"acceptable":2,"unsafe":0,"review":0,"automated_count":2,"automated_acceptable":2,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":1.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.34238022750665303,"upper":1.0,"confidence_level":0.95}}},"generalize":{"n":6,"raw":{"n":6,"acceptable":6,"unsafe":0,"review":0,"automated_count":6,"automated_acceptable":6,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":1.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.6096657120978346,"upper":1.0,"confidence_level":0.95}},"effective":{"n":6,"acceptable":6,"unsafe":0,"review":0,"automated_count":6,"automated_acceptable":6,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":1.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.6096657120978346,"upper":1.0,"confidence_level":0.95}}},"review":{"n":0,"raw":{"n":0,"acceptable":0,"unsafe":0,"review":0,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":null,"unsafe_rate":null,"review_rate":null,"automated_coverage":null,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":null},"effective":{"n":0,"acceptable":0,"unsafe":0,"review":0,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":null,"unsafe_rate":null,"review_rate":null,"automated_coverage":null,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":null}}}},"diagnostic43-compact_no_samples":{"n":43,"raw":{"n":43,"acceptable":33,"unsafe":0,"review":0,"automated_count":43,"automated_acceptable":33,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":0.7674418604651163,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.7674418604651163,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.6225547970219101,"upper":0.8684632152061551,"confidence_level":0.95}},"effective":{"n":43,"acceptable":33,"unsafe":0,"review":3,"automated_count":40,"automated_acceptable":33,"automated_unsafe":0,"unnecessary_reviews":3,"accuracy":0.7674418604651163,"unsafe_rate":0.0,"review_rate":0.06976744186046512,"automated_coverage":0.9302325581395349,"conditional_correctness":0.825,"conditional_unsafe":0.0,"unnecessary_review_rate":0.06976744186046512,"accuracy_wilson_95":{"lower":0.6225547970219101,"upper":0.8684632152061551,"confidence_level":0.95}},"confusion":{"raw":{"keep":{"keep":3,"drop":2,"mask":4,"pseudonymize":3,"generalize":0,"review":0},"drop":{"keep":0,"drop":15,"mask":1,"pseudonymize":0,"generalize":0,"review":0},"mask":{"keep":0,"drop":0,"mask":2,"pseudonymize":0,"generalize":0,"review":0},"pseudonymize":{"keep":0,"drop":0,"mask":0,"pseudonymize":2,"generalize":0,"review":0},"generalize":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":6,"review":0},"review":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0}},"effective":{"keep":{"keep":3,"drop":2,"mask":4,"pseudonymize":0,"generalize":0,"review":3},"drop":{"keep":0,"drop":15,"mask":1,"pseudonymize":0,"generalize":0,"review":0},"mask":{"keep":0,"drop":0,"mask":2,"pseudonymize":0,"generalize":0,"review":0},"pseudonymize":{"keep":0,"drop":0,"mask":0,"pseudonymize":2,"generalize":0,"review":0},"generalize":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":6,"review":0},"review":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0}}},"probability_metrics":{"singleton_n":38,"multiacceptable_n":5,"multiacceptable_coverage":0.11627906976744186,"nll":0.6834168224442793,"nll_zero_mass_n":0,"brier":0.312375014033991,"acceptable_set_n":43,"acceptable_set_nll":0.6064856466208263,"acceptable_set_zero_mass_n":0},"calibration_all_labels":{"equal_width":{"n":43,"bin_count":10,"ece":0.14435420460787204,"signed_mean_gap":-0.009942553801647517,"absolute_mean_gap":0.009942553801647517,"bins":[{"index":0,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.0,"upper":0.1,"upper_inclusive":false},{"index":1,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.1,"upper":0.2,"upper_inclusive":false},{"index":2,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.2,"upper":0.3,"upper_inclusive":false},{"index":3,"count":1,"confidence":0.36168348832197245,"empirical_correctness":0.0,"signed_gap":0.36168348832197245,"absolute_gap":0.36168348832197245,"lower":0.3,"upper":0.4,"upper_inclusive":false},{"index":4,"count":3,"confidence":0.4364032426495806,"empirical_correctness":0.0,"signed_gap":0.4364032426495806,"absolute_gap":0.4364032426495806,"lower":0.4,"upper":0.5,"upper_inclusive":false},{"index":5,"count":7,"confidence":0.5635913820667726,"empirical_correctness":0.42857142857142855,"signed_gap":0.13501995349534407,"absolute_gap":0.13501995349534407,"lower":0.5,"upper":0.6,"upper_inclusive":false},{"index":6,"count":5,"confidence":0.6547635203191409,"empirical_correctness":0.6,"signed_gap":0.05476352031914089,"absolute_gap":0.05476352031914089,"lower":0.6,"upper":0.7,"upper_inclusive":false},{"index":7,"count":3,"confidence":0.7439731302642665,"empirical_correctness":1.0,"signed_gap":-0.2560268697357335,"absolute_gap":0.2560268697357335,"lower":0.7,"upper":0.8,"upper_inclusive":false},{"index":8,"count":12,"confidence":0.8571140844535688,"empirical_correctness":1.0,"signed_gap":-0.14288591554643115,"absolute_gap":0.14288591554643115,"lower":0.8,"upper":0.9,"upper_inclusive":false},{"index":9,"count":12,"confidence":0.9304442741633087,"empirical_correctness":1.0,"signed_gap":-0.06955572583669134,"absolute_gap":0.06955572583669134,"lower":0.9,"upper":1.0,"upper_inclusive":true}]},"equal_mass":{"n":43,"bin_count":10,"ece":0.15953508617009324,"signed_mean_gap":-0.009942553801647483,"absolute_mean_gap":0.009942553801647483,"bins":[{"index":0,"count":4,"confidence":0.4177233040676786,"empirical_correctness":0.0,"signed_gap":0.4177233040676786,"absolute_gap":0.4177233040676786,"min_confidence":0.36168348832197245,"max_confidence":0.46129376405471445},{"index":1,"count":4,"confidence":0.5442902532974694,"empirical_correctness":0.25,"signed_gap":0.2942902532974694,"absolute_gap":0.2942902532974694,"min_confidence":0.526680543092384,"max_confidence":0.5682537810116126},{"index":2,"count":4,"confidence":0.592046304115248,"empirical_correctness":0.5,"signed_gap":0.09204630411524795,"absolute_gap":0.09204630411524795,"min_confidence":0.584399132805777,"max_confidence":0.6002065551834608},{"index":3,"count":5,"confidence":0.6780675441951717,"empirical_correctness":0.8,"signed_gap":-0.12193245580482837,"absolute_gap":0.12193245580482837,"min_confidence":0.6180959498025065,"max_confidence":0.7167266745636146},{"index":4,"count":4,"confidence":0.7834436862565736,"empirical_correctness":1.0,"signed_gap":-0.21655631374342643,"absolute_gap":0.21655631374342643,"min_confidence":0.7339463761815718,"max_confidence":0.8164443700300013},{"index":5,"count":4,"confidence":0.8497150656446155,"empirical_correctness":1.0,"signed_gap":-0.15028493435538448,"absolute_gap":0.15028493435538448,"min_confidence":0.8352636061928096,"max_confidence":0.8610127059027644},{"index":6,"count":5,"confidence":0.8741853927119445,"empirical_correctness":1.0,"signed_gap":-0.1258146072880555,"absolute_gap":0.1258146072880555,"min_confidence":0.8625319008287703,"max_confidence":0.8912868862428885},{"index":7,"count":4,"confidence":0.9133058180262211,"empirical_correctness":1.0,"signed_gap":-0.08669418197377887,"absolute_gap":0.08669418197377887,"min_confidence":0.8969997585075322,"max_confidence":0.92282401342704},{"index":8,"count":4,"confidence":0.9270387238870541,"empirical_correctness":1.0,"signed_gap":-0.0729612761129459,"absolute_gap":0.0729612761129459,"min_confidence":0.9233258733348947,"max_confidence":0.9294024613207595},{"index":9,"count":5,"confidence":0.9401905761628273,"empirical_correctness":1.0,"signed_gap":-0.05980942383717269,"absolute_gap":0.05980942383717269,"min_confidence":0.9312539998908801,"max_confidence":0.9528309698400699}]}},"calibration_singleton_labels":{"equal_width":{"n":38,"bin_count":10,"ece":0.18122390891046783,"signed_mean_gap":0.028670766091707232,"absolute_mean_gap":0.028670766091707232,"bins":[{"index":0,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.0,"upper":0.1,"upper_inclusive":false},{"index":1,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.1,"upper":0.2,"upper_inclusive":false},{"index":2,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.2,"upper":0.3,"upper_inclusive":false},{"index":3,"count":1,"confidence":0.36168348832197245,"empirical_correctness":0.0,"signed_gap":0.36168348832197245,"absolute_gap":0.36168348832197245,"lower":0.3,"upper":0.4,"upper_inclusive":false},{"index":4,"count":3,"confidence":0.4364032426495806,"empirical_correctness":0.0,"signed_gap":0.4364032426495806,"absolute_gap":0.4364032426495806,"lower":0.4,"upper":0.5,"upper_inclusive":false},{"index":5,"count":6,"confidence":0.5601234236102719,"empirical_correctness":0.3333333333333333,"signed_gap":0.22679009027693858,"absolute_gap":0.22679009027693858,"lower":0.5,"upper":0.6,"upper_inclusive":false},{"index":6,"count":3,"confidence":0.6521216890363267,"empirical_correctness":0.3333333333333333,"signed_gap":0.3187883557029934,"absolute_gap":0.3187883557029934,"lower":0.6,"upper":0.7,"upper_inclusive":false},{"index":7,"count":2,"confidence":0.7489865073056139,"empirical_correctness":1.0,"signed_gap":-0.2510134926943861,"absolute_gap":0.2510134926943861,"lower":0.7,"upper":0.8,"upper_inclusive":false},{"index":8,"count":11,"confidence":0.8580169074429652,"empirical_correctness":1.0,"signed_gap":-0.14198309255703478,"absolute_gap":0.14198309255703478,"lower":0.8,"upper":0.9,"upper_inclusive":false},{"index":9,"count":12,"confidence":0.9304442741633087,"empirical_correctness":1.0,"signed_gap":-0.06955572583669134,"absolute_gap":0.06955572583669134,"lower":0.9,"upper":1.0,"upper_inclusive":true}]},"equal_mass":{"n":38,"bin_count":10,"ece":0.16247872407228253,"signed_mean_gap":0.028670766091707295,"absolute_mean_gap":0.028670766091707295,"bins":[{"index":0,"count":3,"confidence":0.4031998174053333,"empirical_correctness":0.0,"signed_gap":0.4031998174053333,"absolute_gap":0.4031998174053333,"min_confidence":0.36168348832197245,"max_confidence":0.4257876604383611},{"index":1,"count":4,"confidence":0.5175502490582449,"empirical_correctness":0.25,"signed_gap":0.26755024905824487,"absolute_gap":0.26755024905824487,"min_confidence":0.46129376405471445,"max_confidence":0.5468356788729898},{"index":2,"count":4,"confidence":0.5880099661667069,"empirical_correctness":0.25,"signed_gap":0.33800996616670687,"absolute_gap":0.33800996616670687,"min_confidence":0.5682537810116126,"max_confidence":0.6002065551834608},{"index":3,"count":4,"confidence":0.7135328816341868,"empirical_correctness":0.75,"signed_gap":-0.03646711836581318,"absolute_gap":0.03646711836581318,"min_confidence":0.6715739390323031,"max_confidence":0.7812463400476133},{"index":4,"count":4,"confidence":0.8273116384756498,"empirical_correctness":1.0,"signed_gap":-0.1726883615243502,"absolute_gap":0.1726883615243502,"min_confidence":0.8021376587671081,"max_confidence":0.8554009189126804},{"index":5,"count":3,"confidence":0.8636068618645839,"empirical_correctness":1.0,"signed_gap":-0.1363931381354161,"absolute_gap":0.1363931381354161,"min_confidence":0.8610127059027644,"max_confidence":0.8672759788622169},{"index":6,"count":4,"confidence":0.8845297105940669,"empirical_correctness":1.0,"signed_gap":-0.11547028940593307,"absolute_gap":0.11547028940593307,"min_confidence":0.8742815625234912,"max_confidence":0.8969997585075322},{"index":7,"count":4,"confidence":0.9198873467330617,"empirical_correctness":1.0,"signed_gap":-0.08011265326693828,"absolute_gap":0.08011265326693828,"min_confidence":0.916511217943675,"max_confidence":0.9233258733348947},{"index":8,"count":4,"confidence":0.9290207555260503,"empirical_correctness":1.0,"signed_gap":-0.07097924447394965,"absolute_gap":0.07097924447394965,"min_confidence":0.9266506203669318,"max_confidence":0.9312539998908801},{"index":9,"count":4,"confidence":0.942424720230814,"empirical_correctness":1.0,"signed_gap":-0.05757527976918597,"absolute_gap":0.05757527976918597,"min_confidence":0.9334277509912526,"max_confidence":0.9528309698400699}]}},"latency_ms":{"n":43,"median":136.78713899571449,"p95":138.02937583532184},"per_action":{"keep":{"n":12,"raw":{"n":12,"acceptable":3,"unsafe":0,"review":0,"automated_count":12,"automated_acceptable":3,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":0.25,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.25,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.08894166839405471,"upper":0.5323053349335657,"confidence_level":0.95}},"effective":{"n":12,"acceptable":3,"unsafe":0,"review":3,"automated_count":9,"automated_acceptable":3,"automated_unsafe":0,"unnecessary_reviews":3,"accuracy":0.25,"unsafe_rate":0.0,"review_rate":0.25,"automated_coverage":0.75,"conditional_correctness":0.3333333333333333,"conditional_unsafe":0.0,"unnecessary_review_rate":0.25,"accuracy_wilson_95":{"lower":0.08894166839405471,"upper":0.5323053349335657,"confidence_level":0.95}}},"drop":{"n":16,"raw":{"n":16,"acceptable":15,"unsafe":0,"review":0,"automated_count":16,"automated_acceptable":15,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":0.9375,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.9375,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.7167126242970107,"upper":0.9888806552353576,"confidence_level":0.95}},"effective":{"n":16,"acceptable":15,"unsafe":0,"review":0,"automated_count":16,"automated_acceptable":15,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":0.9375,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.9375,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.7167126242970107,"upper":0.9888806552353576,"confidence_level":0.95}}},"mask":{"n":2,"raw":{"n":2,"acceptable":2,"unsafe":0,"review":0,"automated_count":2,"automated_acceptable":2,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":1.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.34238022750665303,"upper":1.0,"confidence_level":0.95}},"effective":{"n":2,"acceptable":2,"unsafe":0,"review":0,"automated_count":2,"automated_acceptable":2,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":1.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.34238022750665303,"upper":1.0,"confidence_level":0.95}}},"pseudonymize":{"n":2,"raw":{"n":2,"acceptable":2,"unsafe":0,"review":0,"automated_count":2,"automated_acceptable":2,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":1.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.34238022750665303,"upper":1.0,"confidence_level":0.95}},"effective":{"n":2,"acceptable":2,"unsafe":0,"review":0,"automated_count":2,"automated_acceptable":2,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":1.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.34238022750665303,"upper":1.0,"confidence_level":0.95}}},"generalize":{"n":6,"raw":{"n":6,"acceptable":6,"unsafe":0,"review":0,"automated_count":6,"automated_acceptable":6,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":1.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.6096657120978346,"upper":1.0,"confidence_level":0.95}},"effective":{"n":6,"acceptable":6,"unsafe":0,"review":0,"automated_count":6,"automated_acceptable":6,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":1.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.6096657120978346,"upper":1.0,"confidence_level":0.95}}},"review":{"n":0,"raw":{"n":0,"acceptable":0,"unsafe":0,"review":0,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":null,"unsafe_rate":null,"review_rate":null,"automated_coverage":null,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":null},"effective":{"n":0,"acceptable":0,"unsafe":0,"review":0,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":null,"unsafe_rate":null,"review_rate":null,"automated_coverage":null,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":null}}}},"challenge56-no_samples":{"n":56,"raw":{"n":56,"acceptable":56,"unsafe":0,"review":2,"automated_count":54,"automated_acceptable":54,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":1.0,"unsafe_rate":0.0,"review_rate":0.03571428571428571,"automated_coverage":0.9642857142857143,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.9358060632812366,"upper":1.0,"confidence_level":0.95}},"effective":{"n":56,"acceptable":46,"unsafe":0,"review":12,"automated_count":44,"automated_acceptable":44,"automated_unsafe":0,"unnecessary_reviews":10,"accuracy":0.8214285714285714,"unsafe_rate":0.0,"review_rate":0.21428571428571427,"automated_coverage":0.7857142857142857,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.17857142857142858,"accuracy_wilson_95":{"lower":0.7015881796334894,"upper":0.900001432475877,"confidence_level":0.95}},"confusion":{"raw":{"keep":{"keep":8,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0},"drop":{"keep":0,"drop":22,"mask":0,"pseudonymize":0,"generalize":0,"review":0},"mask":{"keep":0,"drop":0,"mask":8,"pseudonymize":0,"generalize":0,"review":0},"pseudonymize":{"keep":0,"drop":0,"mask":0,"pseudonymize":8,"generalize":0,"review":0},"generalize":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":8,"review":0},"review":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":2}},"effective":{"keep":{"keep":5,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":3},"drop":{"keep":0,"drop":22,"mask":0,"pseudonymize":0,"generalize":0,"review":0},"mask":{"keep":0,"drop":0,"mask":8,"pseudonymize":0,"generalize":0,"review":0},"pseudonymize":{"keep":0,"drop":0,"mask":0,"pseudonymize":1,"generalize":0,"review":7},"generalize":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":8,"review":0},"review":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":2}}},"probability_metrics":{"singleton_n":56,"multiacceptable_n":0,"multiacceptable_coverage":0.0,"nll":0.13099212214975756,"nll_zero_mass_n":0,"brier":0.04904359484115365,"acceptable_set_n":56,"acceptable_set_nll":0.13099212214975756,"acceptable_set_zero_mass_n":0},"calibration_all_labels":{"equal_width":{"n":56,"bin_count":10,"ece":0.11024574301010041,"signed_mean_gap":-0.11024574301010041,"absolute_mean_gap":0.11024574301010041,"bins":[{"index":0,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.0,"upper":0.1,"upper_inclusive":false},{"index":1,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.1,"upper":0.2,"upper_inclusive":false},{"index":2,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.2,"upper":0.3,"upper_inclusive":false},{"index":3,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.3,"upper":0.4,"upper_inclusive":false},{"index":4,"count":1,"confidence":0.47808073010122915,"empirical_correctness":1.0,"signed_gap":-0.5219192698987709,"absolute_gap":0.5219192698987709,"lower":0.4,"upper":0.5,"upper_inclusive":false},{"index":5,"count":4,"confidence":0.5467226819036944,"empirical_correctness":1.0,"signed_gap":-0.45327731809630556,"absolute_gap":0.45327731809630556,"lower":0.5,"upper":0.6,"upper_inclusive":false},{"index":6,"count":2,"confidence":0.669444144977433,"empirical_correctness":1.0,"signed_gap":-0.330555855022567,"absolute_gap":0.330555855022567,"lower":0.6,"upper":0.7,"upper_inclusive":false},{"index":7,"count":2,"confidence":0.7232081174873803,"empirical_correctness":1.0,"signed_gap":-0.27679188251261966,"absolute_gap":0.27679188251261966,"lower":0.7,"upper":0.8,"upper_inclusive":false},{"index":8,"count":3,"confidence":0.8738781708483901,"empirical_correctness":1.0,"signed_gap":-0.12612182915160985,"absolute_gap":0.12612182915160985,"lower":0.8,"upper":0.9,"upper_inclusive":false},{"index":9,"count":44,"confidence":0.9489619976418994,"empirical_correctness":1.0,"signed_gap":-0.05103800235810063,"absolute_gap":0.05103800235810063,"lower":0.9,"upper":1.0,"upper_inclusive":true}]},"equal_mass":{"n":56,"bin_count":10,"ece":0.11024574301010046,"signed_mean_gap":-0.11024574301010046,"absolute_mean_gap":0.11024574301010046,"bins":[{"index":0,"count":5,"confidence":0.5329942915432013,"empirical_correctness":1.0,"signed_gap":-0.4670057084567987,"absolute_gap":0.4670057084567987,"min_confidence":0.47808073010122915,"max_confidence":0.5739093950973443},{"index":1,"count":6,"confidence":0.751892794932835,"empirical_correctness":1.0,"signed_gap":-0.248107205067165,"absolute_gap":0.248107205067165,"min_confidence":0.6574826419821307,"max_confidence":0.8781995079595186},{"index":2,"count":5,"confidence":0.9021738821593726,"empirical_correctness":1.0,"signed_gap":-0.09782611784062745,"absolute_gap":0.09782611784062745,"min_confidence":0.8955822678777869,"max_confidence":0.9080233552995574},{"index":3,"count":6,"confidence":0.9185855741520866,"empirical_correctness":1.0,"signed_gap":-0.08141442584791336,"absolute_gap":0.08141442584791336,"min_confidence":0.9097314096165467,"max_confidence":0.924576168434225},{"index":4,"count":6,"confidence":0.9300601602228743,"empirical_correctness":1.0,"signed_gap":-0.0699398397771257,"absolute_gap":0.0699398397771257,"min_confidence":0.9260295997709623,"max_confidence":0.9360549746852447},{"index":5,"count":5,"confidence":0.9455170678190109,"empirical_correctness":1.0,"signed_gap":-0.054482932180989097,"absolute_gap":0.054482932180989097,"min_confidence":0.9395855159740586,"max_confidence":0.9508333883227836},{"index":6,"count":6,"confidence":0.9612226191378862,"empirical_correctness":1.0,"signed_gap":-0.0387773808621138,"absolute_gap":0.0387773808621138,"min_confidence":0.9568666639371937,"max_confidence":0.9638723616528065},{"index":7,"count":5,"confidence":0.9689312045881866,"empirical_correctness":1.0,"signed_gap":-0.0310687954118134,"absolute_gap":0.0310687954118134,"min_confidence":0.9649774365811754,"max_confidence":0.9724966309243874},{"index":8,"count":6,"confidence":0.9738867833322815,"empirical_correctness":1.0,"signed_gap":-0.026113216667718486,"absolute_gap":0.026113216667718486,"min_confidence":0.9729802824428851,"max_confidence":0.9754047372322765},{"index":9,"count":6,"confidence":0.9773780950362893,"empirical_correctness":1.0,"signed_gap":-0.02262190496371075,"absolute_gap":0.02262190496371075,"min_confidence":0.9763079223685701,"max_confidence":0.9792896132349759}]}},"calibration_singleton_labels":{"equal_width":{"n":56,"bin_count":10,"ece":0.11024574301010041,"signed_mean_gap":-0.11024574301010041,"absolute_mean_gap":0.11024574301010041,"bins":[{"index":0,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.0,"upper":0.1,"upper_inclusive":false},{"index":1,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.1,"upper":0.2,"upper_inclusive":false},{"index":2,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.2,"upper":0.3,"upper_inclusive":false},{"index":3,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.3,"upper":0.4,"upper_inclusive":false},{"index":4,"count":1,"confidence":0.47808073010122915,"empirical_correctness":1.0,"signed_gap":-0.5219192698987709,"absolute_gap":0.5219192698987709,"lower":0.4,"upper":0.5,"upper_inclusive":false},{"index":5,"count":4,"confidence":0.5467226819036944,"empirical_correctness":1.0,"signed_gap":-0.45327731809630556,"absolute_gap":0.45327731809630556,"lower":0.5,"upper":0.6,"upper_inclusive":false},{"index":6,"count":2,"confidence":0.669444144977433,"empirical_correctness":1.0,"signed_gap":-0.330555855022567,"absolute_gap":0.330555855022567,"lower":0.6,"upper":0.7,"upper_inclusive":false},{"index":7,"count":2,"confidence":0.7232081174873803,"empirical_correctness":1.0,"signed_gap":-0.27679188251261966,"absolute_gap":0.27679188251261966,"lower":0.7,"upper":0.8,"upper_inclusive":false},{"index":8,"count":3,"confidence":0.8738781708483901,"empirical_correctness":1.0,"signed_gap":-0.12612182915160985,"absolute_gap":0.12612182915160985,"lower":0.8,"upper":0.9,"upper_inclusive":false},{"index":9,"count":44,"confidence":0.9489619976418994,"empirical_correctness":1.0,"signed_gap":-0.05103800235810063,"absolute_gap":0.05103800235810063,"lower":0.9,"upper":1.0,"upper_inclusive":true}]},"equal_mass":{"n":56,"bin_count":10,"ece":0.11024574301010046,"signed_mean_gap":-0.11024574301010046,"absolute_mean_gap":0.11024574301010046,"bins":[{"index":0,"count":5,"confidence":0.5329942915432013,"empirical_correctness":1.0,"signed_gap":-0.4670057084567987,"absolute_gap":0.4670057084567987,"min_confidence":0.47808073010122915,"max_confidence":0.5739093950973443},{"index":1,"count":6,"confidence":0.751892794932835,"empirical_correctness":1.0,"signed_gap":-0.248107205067165,"absolute_gap":0.248107205067165,"min_confidence":0.6574826419821307,"max_confidence":0.8781995079595186},{"index":2,"count":5,"confidence":0.9021738821593726,"empirical_correctness":1.0,"signed_gap":-0.09782611784062745,"absolute_gap":0.09782611784062745,"min_confidence":0.8955822678777869,"max_confidence":0.9080233552995574},{"index":3,"count":6,"confidence":0.9185855741520866,"empirical_correctness":1.0,"signed_gap":-0.08141442584791336,"absolute_gap":0.08141442584791336,"min_confidence":0.9097314096165467,"max_confidence":0.924576168434225},{"index":4,"count":6,"confidence":0.9300601602228743,"empirical_correctness":1.0,"signed_gap":-0.0699398397771257,"absolute_gap":0.0699398397771257,"min_confidence":0.9260295997709623,"max_confidence":0.9360549746852447},{"index":5,"count":5,"confidence":0.9455170678190109,"empirical_correctness":1.0,"signed_gap":-0.054482932180989097,"absolute_gap":0.054482932180989097,"min_confidence":0.9395855159740586,"max_confidence":0.9508333883227836},{"index":6,"count":6,"confidence":0.9612226191378862,"empirical_correctness":1.0,"signed_gap":-0.0387773808621138,"absolute_gap":0.0387773808621138,"min_confidence":0.9568666639371937,"max_confidence":0.9638723616528065},{"index":7,"count":5,"confidence":0.9689312045881866,"empirical_correctness":1.0,"signed_gap":-0.0310687954118134,"absolute_gap":0.0310687954118134,"min_confidence":0.9649774365811754,"max_confidence":0.9724966309243874},{"index":8,"count":6,"confidence":0.9738867833322815,"empirical_correctness":1.0,"signed_gap":-0.026113216667718486,"absolute_gap":0.026113216667718486,"min_confidence":0.9729802824428851,"max_confidence":0.9754047372322765},{"index":9,"count":6,"confidence":0.9773780950362893,"empirical_correctness":1.0,"signed_gap":-0.02262190496371075,"absolute_gap":0.02262190496371075,"min_confidence":0.9763079223685701,"max_confidence":0.9792896132349759}]}},"latency_ms":{"n":56,"median":113.68743199273013,"p95":114.57367203547619},"per_action":{"keep":{"n":8,"raw":{"n":8,"acceptable":8,"unsafe":0,"review":0,"automated_count":8,"automated_acceptable":8,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":1.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.6755924351161198,"upper":1.0,"confidence_level":0.95}},"effective":{"n":8,"acceptable":5,"unsafe":0,"review":3,"automated_count":5,"automated_acceptable":5,"automated_unsafe":0,"unnecessary_reviews":3,"accuracy":0.625,"unsafe_rate":0.0,"review_rate":0.375,"automated_coverage":0.625,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.375,"accuracy_wilson_95":{"lower":0.30574239460262737,"upper":0.8631557141764026,"confidence_level":0.95}}},"drop":{"n":22,"raw":{"n":22,"acceptable":22,"unsafe":0,"review":0,"automated_count":22,"automated_acceptable":22,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":1.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.8513451253913791,"upper":1.0,"confidence_level":0.95}},"effective":{"n":22,"acceptable":22,"unsafe":0,"review":0,"automated_count":22,"automated_acceptable":22,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":1.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.8513451253913791,"upper":1.0,"confidence_level":0.95}}},"mask":{"n":8,"raw":{"n":8,"acceptable":8,"unsafe":0,"review":0,"automated_count":8,"automated_acceptable":8,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":1.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.6755924351161198,"upper":1.0,"confidence_level":0.95}},"effective":{"n":8,"acceptable":8,"unsafe":0,"review":0,"automated_count":8,"automated_acceptable":8,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":1.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.6755924351161198,"upper":1.0,"confidence_level":0.95}}},"pseudonymize":{"n":8,"raw":{"n":8,"acceptable":8,"unsafe":0,"review":0,"automated_count":8,"automated_acceptable":8,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":1.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.6755924351161198,"upper":1.0,"confidence_level":0.95}},"effective":{"n":8,"acceptable":1,"unsafe":0,"review":7,"automated_count":1,"automated_acceptable":1,"automated_unsafe":0,"unnecessary_reviews":7,"accuracy":0.125,"unsafe_rate":0.0,"review_rate":0.875,"automated_coverage":0.125,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.875,"accuracy_wilson_95":{"lower":0.02241749145005667,"upper":0.4708881822128535,"confidence_level":0.95}}},"generalize":{"n":8,"raw":{"n":8,"acceptable":8,"unsafe":0,"review":0,"automated_count":8,"automated_acceptable":8,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":1.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.6755924351161198,"upper":1.0,"confidence_level":0.95}},"effective":{"n":8,"acceptable":8,"unsafe":0,"review":0,"automated_count":8,"automated_acceptable":8,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":1.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.6755924351161198,"upper":1.0,"confidence_level":0.95}}},"review":{"n":2,"raw":{"n":2,"acceptable":2,"unsafe":0,"review":2,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":1.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.34238022750665303,"upper":1.0,"confidence_level":0.95}},"effective":{"n":2,"acceptable":2,"unsafe":0,"review":2,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":1.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.34238022750665303,"upper":1.0,"confidence_level":0.95}}}}},"challenge56-compact_no_samples":{"n":56,"raw":{"n":56,"acceptable":56,"unsafe":0,"review":2,"automated_count":54,"automated_acceptable":54,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":1.0,"unsafe_rate":0.0,"review_rate":0.03571428571428571,"automated_coverage":0.9642857142857143,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.9358060632812366,"upper":1.0,"confidence_level":0.95}},"effective":{"n":56,"acceptable":44,"unsafe":0,"review":14,"automated_count":42,"automated_acceptable":42,"automated_unsafe":0,"unnecessary_reviews":12,"accuracy":0.7857142857142857,"unsafe_rate":0.0,"review_rate":0.25,"automated_coverage":0.75,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.21428571428571427,"accuracy_wilson_95":{"lower":0.6618053643659488,"upper":0.8729409575090435,"confidence_level":0.95}},"confusion":{"raw":{"keep":{"keep":8,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0},"drop":{"keep":0,"drop":22,"mask":0,"pseudonymize":0,"generalize":0,"review":0},"mask":{"keep":0,"drop":0,"mask":8,"pseudonymize":0,"generalize":0,"review":0},"pseudonymize":{"keep":0,"drop":0,"mask":0,"pseudonymize":8,"generalize":0,"review":0},"generalize":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":8,"review":0},"review":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":2}},"effective":{"keep":{"keep":5,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":3},"drop":{"keep":0,"drop":20,"mask":0,"pseudonymize":0,"generalize":0,"review":2},"mask":{"keep":0,"drop":0,"mask":8,"pseudonymize":0,"generalize":0,"review":0},"pseudonymize":{"keep":0,"drop":0,"mask":0,"pseudonymize":1,"generalize":0,"review":7},"generalize":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":8,"review":0},"review":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":2}}},"probability_metrics":{"singleton_n":56,"multiacceptable_n":0,"multiacceptable_coverage":0.0,"nll":0.14758659934851398,"nll_zero_mass_n":0,"brier":0.052332622754329164,"acceptable_set_n":56,"acceptable_set_nll":0.14758659934851398,"acceptable_set_zero_mass_n":0},"calibration_all_labels":{"equal_width":{"n":56,"bin_count":10,"ece":0.12069529050550212,"signed_mean_gap":-0.12069529050550212,"absolute_mean_gap":0.12069529050550212,"bins":[{"index":0,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.0,"upper":0.1,"upper_inclusive":false},{"index":1,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.1,"upper":0.2,"upper_inclusive":false},{"index":2,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.2,"upper":0.3,"upper_inclusive":false},{"index":3,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.3,"upper":0.4,"upper_inclusive":false},{"index":4,"count":2,"confidence":0.42553149363922915,"empirical_correctness":1.0,"signed_gap":-0.5744685063607708,"absolute_gap":0.5744685063607708,"lower":0.4,"upper":0.5,"upper_inclusive":false},{"index":5,"count":3,"confidence":0.5347627517741432,"empirical_correctness":1.0,"signed_gap":-0.46523724822585677,"absolute_gap":0.46523724822585677,"lower":0.5,"upper":0.6,"upper_inclusive":false},{"index":6,"count":2,"confidence":0.6039724253271372,"empirical_correctness":1.0,"signed_gap":-0.39602757467286276,"absolute_gap":0.39602757467286276,"lower":0.6,"upper":0.7,"upper_inclusive":false},{"index":7,"count":3,"confidence":0.734539201279155,"empirical_correctness":1.0,"signed_gap":-0.26546079872084505,"absolute_gap":0.26546079872084505,"lower":0.7,"upper":0.8,"upper_inclusive":false},{"index":8,"count":8,"confidence":0.8824225600720521,"empirical_correctness":1.0,"signed_gap":-0.11757743992794789,"absolute_gap":0.11757743992794789,"lower":0.8,"upper":0.9,"upper_inclusive":false},{"index":9,"count":38,"confidence":0.955651830369022,"empirical_correctness":1.0,"signed_gap":-0.044348169630977985,"absolute_gap":0.044348169630977985,"lower":0.9,"upper":1.0,"upper_inclusive":true}]},"equal_mass":{"n":56,"bin_count":10,"ece":0.12069529050550208,"signed_mean_gap":-0.12069529050550208,"absolute_mean_gap":0.12069529050550208,"bins":[{"index":0,"count":5,"confidence":0.4910702485201776,"empirical_correctness":1.0,"signed_gap":-0.5089297514798223,"absolute_gap":0.5089297514798223,"min_confidence":0.4146464404756929,"max_confidence":0.5527203270662902},{"index":1,"count":6,"confidence":0.7139082998660813,"empirical_correctness":1.0,"signed_gap":-0.2860917001339187,"absolute_gap":0.2860917001339187,"min_confidence":0.6014214739192408,"max_confidence":0.8718873447047482},{"index":2,"count":5,"confidence":0.8790420540299501,"empirical_correctness":1.0,"signed_gap":-0.12095794597004994,"absolute_gap":0.12095794597004994,"min_confidence":0.8735762327449692,"max_confidence":0.8919260287283401},{"index":3,"count":6,"confidence":0.9011885484713495,"empirical_correctness":1.0,"signed_gap":-0.09881145152865045,"absolute_gap":0.09881145152865045,"min_confidence":0.8929875933462647,"max_confidence":0.9100752989293123},{"index":4,"count":6,"confidence":0.9267581956936937,"empirical_correctness":1.0,"signed_gap":-0.07324180430630634,"absolute_gap":0.07324180430630634,"min_confidence":0.9123131589002922,"max_confidence":0.9318102289656415},{"index":5,"count":5,"confidence":0.9419819085745622,"empirical_correctness":1.0,"signed_gap":-0.05801809142543779,"absolute_gap":0.05801809142543779,"min_confidence":0.9324624605738346,"max_confidence":0.9608315472874815},{"index":6,"count":6,"confidence":0.9687146809031394,"empirical_correctness":1.0,"signed_gap":-0.03128531909686061,"absolute_gap":0.03128531909686061,"min_confidence":0.9650328173110401,"max_confidence":0.970006882813768},{"index":7,"count":5,"confidence":0.9727409724703445,"empirical_correctness":1.0,"signed_gap":-0.027259027529655477,"absolute_gap":0.027259027529655477,"min_confidence":0.971376288919748,"max_confidence":0.9746912713700959},{"index":8,"count":6,"confidence":0.9759598154246344,"empirical_correctness":1.0,"signed_gap":-0.02404018457536561,"absolute_gap":0.02404018457536561,"min_confidence":0.9747289150453073,"max_confidence":0.9785917532642592},{"index":9,"count":6,"confidence":0.9829517619272203,"empirical_correctness":1.0,"signed_gap":-0.017048238072779665,"absolute_gap":0.017048238072779665,"min_confidence":0.9816231841526402,"max_confidence":0.9851158385116051}]}},"calibration_singleton_labels":{"equal_width":{"n":56,"bin_count":10,"ece":0.12069529050550212,"signed_mean_gap":-0.12069529050550212,"absolute_mean_gap":0.12069529050550212,"bins":[{"index":0,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.0,"upper":0.1,"upper_inclusive":false},{"index":1,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.1,"upper":0.2,"upper_inclusive":false},{"index":2,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.2,"upper":0.3,"upper_inclusive":false},{"index":3,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.3,"upper":0.4,"upper_inclusive":false},{"index":4,"count":2,"confidence":0.42553149363922915,"empirical_correctness":1.0,"signed_gap":-0.5744685063607708,"absolute_gap":0.5744685063607708,"lower":0.4,"upper":0.5,"upper_inclusive":false},{"index":5,"count":3,"confidence":0.5347627517741432,"empirical_correctness":1.0,"signed_gap":-0.46523724822585677,"absolute_gap":0.46523724822585677,"lower":0.5,"upper":0.6,"upper_inclusive":false},{"index":6,"count":2,"confidence":0.6039724253271372,"empirical_correctness":1.0,"signed_gap":-0.39602757467286276,"absolute_gap":0.39602757467286276,"lower":0.6,"upper":0.7,"upper_inclusive":false},{"index":7,"count":3,"confidence":0.734539201279155,"empirical_correctness":1.0,"signed_gap":-0.26546079872084505,"absolute_gap":0.26546079872084505,"lower":0.7,"upper":0.8,"upper_inclusive":false},{"index":8,"count":8,"confidence":0.8824225600720521,"empirical_correctness":1.0,"signed_gap":-0.11757743992794789,"absolute_gap":0.11757743992794789,"lower":0.8,"upper":0.9,"upper_inclusive":false},{"index":9,"count":38,"confidence":0.955651830369022,"empirical_correctness":1.0,"signed_gap":-0.044348169630977985,"absolute_gap":0.044348169630977985,"lower":0.9,"upper":1.0,"upper_inclusive":true}]},"equal_mass":{"n":56,"bin_count":10,"ece":0.12069529050550208,"signed_mean_gap":-0.12069529050550208,"absolute_mean_gap":0.12069529050550208,"bins":[{"index":0,"count":5,"confidence":0.4910702485201776,"empirical_correctness":1.0,"signed_gap":-0.5089297514798223,"absolute_gap":0.5089297514798223,"min_confidence":0.4146464404756929,"max_confidence":0.5527203270662902},{"index":1,"count":6,"confidence":0.7139082998660813,"empirical_correctness":1.0,"signed_gap":-0.2860917001339187,"absolute_gap":0.2860917001339187,"min_confidence":0.6014214739192408,"max_confidence":0.8718873447047482},{"index":2,"count":5,"confidence":0.8790420540299501,"empirical_correctness":1.0,"signed_gap":-0.12095794597004994,"absolute_gap":0.12095794597004994,"min_confidence":0.8735762327449692,"max_confidence":0.8919260287283401},{"index":3,"count":6,"confidence":0.9011885484713495,"empirical_correctness":1.0,"signed_gap":-0.09881145152865045,"absolute_gap":0.09881145152865045,"min_confidence":0.8929875933462647,"max_confidence":0.9100752989293123},{"index":4,"count":6,"confidence":0.9267581956936937,"empirical_correctness":1.0,"signed_gap":-0.07324180430630634,"absolute_gap":0.07324180430630634,"min_confidence":0.9123131589002922,"max_confidence":0.9318102289656415},{"index":5,"count":5,"confidence":0.9419819085745622,"empirical_correctness":1.0,"signed_gap":-0.05801809142543779,"absolute_gap":0.05801809142543779,"min_confidence":0.9324624605738346,"max_confidence":0.9608315472874815},{"index":6,"count":6,"confidence":0.9687146809031394,"empirical_correctness":1.0,"signed_gap":-0.03128531909686061,"absolute_gap":0.03128531909686061,"min_confidence":0.9650328173110401,"max_confidence":0.970006882813768},{"index":7,"count":5,"confidence":0.9727409724703445,"empirical_correctness":1.0,"signed_gap":-0.027259027529655477,"absolute_gap":0.027259027529655477,"min_confidence":0.971376288919748,"max_confidence":0.9746912713700959},{"index":8,"count":6,"confidence":0.9759598154246344,"empirical_correctness":1.0,"signed_gap":-0.02404018457536561,"absolute_gap":0.02404018457536561,"min_confidence":0.9747289150453073,"max_confidence":0.9785917532642592},{"index":9,"count":6,"confidence":0.9829517619272203,"empirical_correctness":1.0,"signed_gap":-0.017048238072779665,"absolute_gap":0.017048238072779665,"min_confidence":0.9816231841526402,"max_confidence":0.9851158385116051}]}},"latency_ms":{"n":56,"median":119.93470950983465,"p95":121.34794828307349},"per_action":{"keep":{"n":8,"raw":{"n":8,"acceptable":8,"unsafe":0,"review":0,"automated_count":8,"automated_acceptable":8,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":1.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.6755924351161198,"upper":1.0,"confidence_level":0.95}},"effective":{"n":8,"acceptable":5,"unsafe":0,"review":3,"automated_count":5,"automated_acceptable":5,"automated_unsafe":0,"unnecessary_reviews":3,"accuracy":0.625,"unsafe_rate":0.0,"review_rate":0.375,"automated_coverage":0.625,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.375,"accuracy_wilson_95":{"lower":0.30574239460262737,"upper":0.8631557141764026,"confidence_level":0.95}}},"drop":{"n":22,"raw":{"n":22,"acceptable":22,"unsafe":0,"review":0,"automated_count":22,"automated_acceptable":22,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":1.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.8513451253913791,"upper":1.0,"confidence_level":0.95}},"effective":{"n":22,"acceptable":20,"unsafe":0,"review":2,"automated_count":20,"automated_acceptable":20,"automated_unsafe":0,"unnecessary_reviews":2,"accuracy":0.9090909090909091,"unsafe_rate":0.0,"review_rate":0.09090909090909091,"automated_coverage":0.9090909090909091,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.09090909090909091,"accuracy_wilson_95":{"lower":0.7218505315839417,"upper":0.9747045710090046,"confidence_level":0.95}}},"mask":{"n":8,"raw":{"n":8,"acceptable":8,"unsafe":0,"review":0,"automated_count":8,"automated_acceptable":8,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":1.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.6755924351161198,"upper":1.0,"confidence_level":0.95}},"effective":{"n":8,"acceptable":8,"unsafe":0,"review":0,"automated_count":8,"automated_acceptable":8,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":1.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.6755924351161198,"upper":1.0,"confidence_level":0.95}}},"pseudonymize":{"n":8,"raw":{"n":8,"acceptable":8,"unsafe":0,"review":0,"automated_count":8,"automated_acceptable":8,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":1.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.6755924351161198,"upper":1.0,"confidence_level":0.95}},"effective":{"n":8,"acceptable":1,"unsafe":0,"review":7,"automated_count":1,"automated_acceptable":1,"automated_unsafe":0,"unnecessary_reviews":7,"accuracy":0.125,"unsafe_rate":0.0,"review_rate":0.875,"automated_coverage":0.125,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.875,"accuracy_wilson_95":{"lower":0.02241749145005667,"upper":0.4708881822128535,"confidence_level":0.95}}},"generalize":{"n":8,"raw":{"n":8,"acceptable":8,"unsafe":0,"review":0,"automated_count":8,"automated_acceptable":8,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":1.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.6755924351161198,"upper":1.0,"confidence_level":0.95}},"effective":{"n":8,"acceptable":8,"unsafe":0,"review":0,"automated_count":8,"automated_acceptable":8,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":1.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.6755924351161198,"upper":1.0,"confidence_level":0.95}}},"review":{"n":2,"raw":{"n":2,"acceptable":2,"unsafe":0,"review":2,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":1.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.34238022750665303,"upper":1.0,"confidence_level":0.95}},"effective":{"n":2,"acceptable":2,"unsafe":0,"review":2,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":1.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.34238022750665303,"upper":1.0,"confidence_level":0.95}}}}}},"composite":{"n":198,"scope":"descriptive sum across four correlated panels","raw":{"n":198,"acceptable":173,"unsafe":1,"review":4,"automated_count":194,"automated_acceptable":169,"automated_unsafe":1,"unnecessary_reviews":0,"accuracy":0.8737373737373737,"unsafe_rate":0.005050505050505051,"review_rate":0.020202020202020204,"automated_coverage":0.9797979797979798,"conditional_correctness":0.8711340206185567,"conditional_unsafe":0.005154639175257732,"unnecessary_review_rate":0.0},"effective":{"n":198,"acceptable":151,"unsafe":1,"review":36,"automated_count":162,"automated_acceptable":147,"automated_unsafe":1,"unnecessary_reviews":32,"accuracy":0.7626262626262627,"unsafe_rate":0.005050505050505051,"review_rate":0.18181818181818182,"automated_coverage":0.8181818181818182,"conditional_correctness":0.9074074074074074,"conditional_unsafe":0.006172839506172839,"unnecessary_review_rate":0.16161616161616163}}},{"id":"chaoliangUNSW--Jev-Style-0.8B-Decision-v3","platform":"Local","model_id":"chaoliangUNSW/Jev-Style-0.8B-Decision-v3","revision":"633729d87c685cd9e867da1a801caf8cd4cba16c","loading":"float16 / none","scoring_kind":"Native decision readout","settings":{"dtype":"float16","quantization":"none","max_tokens":4096,"effective_max_tokens":4096,"head_max_tokens":2048,"temperature":0.8800546821789332,"cpu_offload":false,"scoring_batch_size":1,"trust_remote_code":false,"local_files_only":true,"use_safetensors":true,"guard_threshold":0.45,"seed":0,"tf32":false},"scoring_metadata":{"backend":"jev_style","protocol":"hf-local-sweep-native-v1","readout":"builtin_final_hidden_float32_dot_tied_yes_minus_no","probability_source":"native_choice_published_temperature_not_privacy_calibrated","latency_scope":"synchronous_forward_scoring_including_transfer_excluding_tokenization","precision":"float16","equal_mass_convention":"rank bins with floor boundaries at floor(i*n/k), k=min(10,n//2); minimum two rows"},"load_ms":7141.659999964759,"peak_allocated_bytes":1624459264,"peak_reserved_bytes":1688207360,"panels":{"diagnostic43-no_samples":{"n":43,"raw":{"n":43,"acceptable":16,"unsafe":6,"review":0,"automated_count":43,"automated_acceptable":16,"automated_unsafe":6,"unnecessary_reviews":0,"accuracy":0.37209302325581395,"unsafe_rate":0.13953488372093023,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.37209302325581395,"conditional_unsafe":0.13953488372093023,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.24376339009650572,"upper":0.5214019084031153,"confidence_level":0.95}},"effective":{"n":43,"acceptable":14,"unsafe":3,"review":14,"automated_count":29,"automated_acceptable":13,"automated_unsafe":3,"unnecessary_reviews":13,"accuracy":0.32558139534883723,"unsafe_rate":0.06976744186046512,"review_rate":0.32558139534883723,"automated_coverage":0.6744186046511628,"conditional_correctness":0.4482758620689655,"conditional_unsafe":0.10344827586206896,"unnecessary_review_rate":0.3023255813953488,"accuracy_wilson_95":{"lower":0.20493297126568708,"upper":0.47483789032470525,"confidence_level":0.95}},"confusion":{"raw":{"keep":{"keep":0,"drop":3,"mask":6,"pseudonymize":3,"generalize":0,"review":0},"drop":{"keep":0,"drop":8,"mask":3,"pseudonymize":5,"generalize":0,"review":0},"mask":{"keep":0,"drop":0,"mask":2,"pseudonymize":0,"generalize":0,"review":0},"pseudonymize":{"keep":0,"drop":0,"mask":0,"pseudonymize":2,"generalize":0,"review":0},"generalize":{"keep":0,"drop":3,"mask":0,"pseudonymize":3,"generalize":0,"review":0},"review":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0}},"effective":{"keep":{"keep":0,"drop":3,"mask":5,"pseudonymize":0,"generalize":0,"review":4},"drop":{"keep":0,"drop":7,"mask":2,"pseudonymize":3,"generalize":0,"review":4},"mask":{"keep":0,"drop":0,"mask":1,"pseudonymize":0,"generalize":0,"review":1},"pseudonymize":{"keep":0,"drop":0,"mask":0,"pseudonymize":1,"generalize":0,"review":1},"generalize":{"keep":0,"drop":3,"mask":0,"pseudonymize":0,"generalize":0,"review":3},"review":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0}}},"probability_metrics":{"singleton_n":38,"multiacceptable_n":5,"multiacceptable_coverage":0.11627906976744186,"nll":1.9453928165054266,"nll_zero_mass_n":0,"brier":0.8800884704035175,"acceptable_set_n":43,"acceptable_set_nll":1.7465659007039442,"acceptable_set_zero_mass_n":0},"calibration_all_labels":{"equal_width":{"n":43,"bin_count":10,"ece":0.16188928990545579,"signed_mean_gap":0.141047364364354,"absolute_mean_gap":0.141047364364354,"bins":[{"index":0,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.0,"upper":0.1,"upper_inclusive":false},{"index":1,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.1,"upper":0.2,"upper_inclusive":false},{"index":2,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.2,"upper":0.3,"upper_inclusive":false},{"index":3,"count":5,"confidence":0.3749166284989845,"empirical_correctness":0.0,"signed_gap":0.3749166284989845,"absolute_gap":0.3749166284989845,"lower":0.3,"upper":0.4,"upper_inclusive":false},{"index":4,"count":15,"confidence":0.4574820128592278,"empirical_correctness":0.26666666666666666,"signed_gap":0.19081534619256113,"absolute_gap":0.19081534619256113,"lower":0.4,"upper":0.5,"upper_inclusive":false},{"index":5,"count":14,"confidence":0.5394213286333079,"empirical_correctness":0.5714285714285714,"signed_gap":-0.03200724279526346,"absolute_gap":0.03200724279526346,"lower":0.5,"upper":0.6,"upper_inclusive":false},{"index":6,"count":9,"confidence":0.6418138590463968,"empirical_correctness":0.4444444444444444,"signed_gap":0.19736941460195234,"absolute_gap":0.19736941460195234,"lower":0.6,"upper":0.7,"upper_inclusive":false},{"index":7,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.7,"upper":0.8,"upper_inclusive":false},{"index":8,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.8,"upper":0.9,"upper_inclusive":false},{"index":9,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.9,"upper":1.0,"upper_inclusive":true}]},"equal_mass":{"n":43,"bin_count":10,"ece":0.2731908382368816,"signed_mean_gap":0.14104736436435397,"absolute_mean_gap":0.14104736436435397,"bins":[{"index":0,"count":4,"confidence":0.3703416587118384,"empirical_correctness":0.0,"signed_gap":0.3703416587118384,"absolute_gap":0.3703416587118384,"min_confidence":0.3426316908669974,"max_confidence":0.38601659335731914},{"index":1,"count":4,"confidence":0.41243867706734944,"empirical_correctness":0.5,"signed_gap":-0.08756132293265056,"absolute_gap":0.08756132293265056,"min_confidence":0.393216507647569,"max_confidence":0.42770538785514967},{"index":2,"count":4,"confidence":0.44535456947677654,"empirical_correctness":0.5,"signed_gap":-0.05464543052322346,"absolute_gap":0.05464543052322346,"min_confidence":0.4315910925766639,"max_confidence":0.4624574801014176},{"index":3,"count":5,"confidence":0.4724788020147024,"empirical_correctness":0.0,"signed_gap":0.4724788020147024,"absolute_gap":0.4724788020147024,"min_confidence":0.4633232328147902,"max_confidence":0.4835290566986487},{"index":4,"count":4,"confidence":0.4920821732848288,"empirical_correctness":0.0,"signed_gap":0.4920821732848288,"absolute_gap":0.4920821732848288,"min_confidence":0.48612140327424325,"max_confidence":0.5064489888533452},{"index":5,"count":4,"confidence":0.5161115750637087,"empirical_correctness":0.75,"signed_gap":-0.23388842493629125,"absolute_gap":0.23388842493629125,"min_confidence":0.5126096219745762,"max_confidence":0.5201965422265602},{"index":6,"count":5,"confidence":0.5326592050618635,"empirical_correctness":0.8,"signed_gap":-0.26734079493813656,"absolute_gap":0.26734079493813656,"min_confidence":0.5202842637124272,"max_confidence":0.54911582073606},{"index":7,"count":4,"confidence":0.5794268216122034,"empirical_correctness":0.25,"signed_gap":0.3294268216122034,"absolute_gap":0.3294268216122034,"min_confidence":0.5643078442119651,"max_confidence":0.5953995137719565},{"index":8,"count":4,"confidence":0.6213210084779162,"empirical_correctness":0.25,"signed_gap":0.37132100847791616,"absolute_gap":0.37132100847791616,"min_confidence":0.6047234105155851,"max_confidence":0.637842477812516},{"index":9,"count":5,"confidence":0.6582081395011812,"empirical_correctness":0.6,"signed_gap":0.0582081395011812,"absolute_gap":0.0582081395011812,"min_confidence":0.6407658761670633,"max_confidence":0.6728359948280822}]}},"calibration_singleton_labels":{"equal_width":{"n":38,"bin_count":10,"ece":0.19910756248563058,"signed_mean_gap":0.19910756248563058,"absolute_mean_gap":0.19910756248563058,"bins":[{"index":0,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.0,"upper":0.1,"upper_inclusive":false},{"index":1,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.1,"upper":0.2,"upper_inclusive":false},{"index":2,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.2,"upper":0.3,"upper_inclusive":false},{"index":3,"count":4,"confidence":0.3829878629069813,"empirical_correctness":0.0,"signed_gap":0.3829878629069813,"absolute_gap":0.3829878629069813,"lower":0.3,"upper":0.4,"upper_inclusive":false},{"index":4,"count":14,"confidence":0.4579681963979553,"empirical_correctness":0.21428571428571427,"signed_gap":0.243682482112241,"absolute_gap":0.243682482112241,"lower":0.4,"upper":0.5,"upper_inclusive":false},{"index":5,"count":12,"confidence":0.5432577030554312,"empirical_correctness":0.5,"signed_gap":0.043257703055431196,"absolute_gap":0.043257703055431196,"lower":0.5,"upper":0.6,"upper_inclusive":false},{"index":6,"count":8,"confidence":0.6379360920736861,"empirical_correctness":0.375,"signed_gap":0.26293609207368607,"absolute_gap":0.26293609207368607,"lower":0.6,"upper":0.7,"upper_inclusive":false},{"index":7,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.7,"upper":0.8,"upper_inclusive":false},{"index":8,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.8,"upper":0.9,"upper_inclusive":false},{"index":9,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.9,"upper":1.0,"upper_inclusive":true}]},"equal_mass":{"n":38,"bin_count":10,"ece":0.28622909566287197,"signed_mean_gap":0.19910756248563058,"absolute_mean_gap":0.19910756248563058,"bins":[{"index":0,"count":3,"confidence":0.37957831466011865,"empirical_correctness":0.0,"signed_gap":0.37957831466011865,"absolute_gap":0.37957831466011865,"min_confidence":0.3694524237255476,"max_confidence":0.38601659335731914},{"index":1,"count":4,"confidence":0.41243867706734944,"empirical_correctness":0.5,"signed_gap":-0.08756132293265056,"absolute_gap":0.08756132293265056,"min_confidence":0.393216507647569,"max_confidence":0.42770538785514967},{"index":2,"count":4,"confidence":0.44851651685121335,"empirical_correctness":0.25,"signed_gap":0.19851651685121335,"absolute_gap":0.19851651685121335,"min_confidence":0.4315910925766639,"max_confidence":0.4633232328147902},{"index":3,"count":4,"confidence":0.47476769431468036,"empirical_correctness":0.0,"signed_gap":0.47476769431468036,"absolute_gap":0.47476769431468036,"min_confidence":0.4674532863403889,"max_confidence":0.4835290566986487},{"index":4,"count":4,"confidence":0.4920821732848288,"empirical_correctness":0.0,"signed_gap":0.4920821732848288,"absolute_gap":0.4920821732848288,"min_confidence":0.48612140327424325,"max_confidence":0.5064489888533452},{"index":5,"count":3,"confidence":0.5173081332553752,"empirical_correctness":0.6666666666666666,"signed_gap":-0.14935853341129146,"absolute_gap":0.14935853341129146,"min_confidence":0.5128956368105805,"max_confidence":0.5202842637124272},{"index":6,"count":4,"confidence":0.5357529403992225,"empirical_correctness":0.75,"signed_gap":-0.2142470596007775,"absolute_gap":0.2142470596007775,"min_confidence":0.5261713949299682,"max_confidence":0.54911582073606},{"index":7,"count":4,"confidence":0.5794268216122034,"empirical_correctness":0.25,"signed_gap":0.3294268216122034,"absolute_gap":0.3294268216122034,"min_confidence":0.5643078442119651,"max_confidence":0.5953995137719565},{"index":8,"count":4,"confidence":0.6213210084779162,"empirical_correctness":0.25,"signed_gap":0.37132100847791616,"absolute_gap":0.37132100847791616,"min_confidence":0.6047234105155851,"max_confidence":0.637842477812516},{"index":9,"count":4,"confidence":0.654551175669456,"empirical_correctness":0.5,"signed_gap":0.15455117566945598,"absolute_gap":0.15455117566945598,"min_confidence":0.6407658761670633,"max_confidence":0.6721717332775659}]}},"latency_ms":{"n":43,"median":82.51350000500679,"p95":87.41834012325853},"per_action":{"keep":{"n":12,"raw":{"n":12,"acceptable":0,"unsafe":0,"review":0,"automated_count":12,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":2.7755575615628914e-17,"upper":0.24249400665524085,"confidence_level":0.95}},"effective":{"n":12,"acceptable":0,"unsafe":0,"review":4,"automated_count":8,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":4,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":0.3333333333333333,"automated_coverage":0.6666666666666666,"conditional_correctness":0.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.3333333333333333,"accuracy_wilson_95":{"lower":2.7755575615628914e-17,"upper":0.24249400665524085,"confidence_level":0.95}}},"drop":{"n":16,"raw":{"n":16,"acceptable":8,"unsafe":5,"review":0,"automated_count":16,"automated_acceptable":8,"automated_unsafe":5,"unnecessary_reviews":0,"accuracy":0.5,"unsafe_rate":0.3125,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.5,"conditional_unsafe":0.3125,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.27999563610326017,"upper":0.7200043638967398,"confidence_level":0.95}},"effective":{"n":16,"acceptable":7,"unsafe":3,"review":4,"automated_count":12,"automated_acceptable":7,"automated_unsafe":3,"unnecessary_reviews":4,"accuracy":0.4375,"unsafe_rate":0.1875,"review_rate":0.25,"automated_coverage":0.75,"conditional_correctness":0.5833333333333334,"conditional_unsafe":0.25,"unnecessary_review_rate":0.25,"accuracy_wilson_95":{"lower":0.23098652405492354,"upper":0.6682144360118811,"confidence_level":0.95}}},"mask":{"n":2,"raw":{"n":2,"acceptable":2,"unsafe":0,"review":0,"automated_count":2,"automated_acceptable":2,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":1.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.34238022750665303,"upper":1.0,"confidence_level":0.95}},"effective":{"n":2,"acceptable":1,"unsafe":0,"review":1,"automated_count":1,"automated_acceptable":1,"automated_unsafe":0,"unnecessary_reviews":1,"accuracy":0.5,"unsafe_rate":0.0,"review_rate":0.5,"automated_coverage":0.5,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.5,"accuracy_wilson_95":{"lower":0.09453120573423074,"upper":0.9054687942657693,"confidence_level":0.95}}},"pseudonymize":{"n":2,"raw":{"n":2,"acceptable":2,"unsafe":0,"review":0,"automated_count":2,"automated_acceptable":2,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":1.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.34238022750665303,"upper":1.0,"confidence_level":0.95}},"effective":{"n":2,"acceptable":1,"unsafe":0,"review":1,"automated_count":1,"automated_acceptable":1,"automated_unsafe":0,"unnecessary_reviews":1,"accuracy":0.5,"unsafe_rate":0.0,"review_rate":0.5,"automated_coverage":0.5,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.5,"accuracy_wilson_95":{"lower":0.09453120573423074,"upper":0.9054687942657693,"confidence_level":0.95}}},"generalize":{"n":6,"raw":{"n":6,"acceptable":0,"unsafe":0,"review":0,"automated_count":6,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":2.7755575615628914e-17,"upper":0.3903342879021653,"confidence_level":0.95}},"effective":{"n":6,"acceptable":0,"unsafe":0,"review":3,"automated_count":3,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":3,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":0.5,"automated_coverage":0.5,"conditional_correctness":0.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.5,"accuracy_wilson_95":{"lower":2.7755575615628914e-17,"upper":0.3903342879021653,"confidence_level":0.95}}},"review":{"n":0,"raw":{"n":0,"acceptable":0,"unsafe":0,"review":0,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":null,"unsafe_rate":null,"review_rate":null,"automated_coverage":null,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":null},"effective":{"n":0,"acceptable":0,"unsafe":0,"review":0,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":null,"unsafe_rate":null,"review_rate":null,"automated_coverage":null,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":null}}}},"diagnostic43-compact_no_samples":{"n":43,"raw":{"n":43,"acceptable":23,"unsafe":2,"review":0,"automated_count":43,"automated_acceptable":23,"automated_unsafe":2,"unnecessary_reviews":0,"accuracy":0.5348837209302325,"unsafe_rate":0.046511627906976744,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.5348837209302325,"conditional_unsafe":0.046511627906976744,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.38915640472313967,"upper":0.6748894229587816,"confidence_level":0.95}},"effective":{"n":43,"acceptable":19,"unsafe":0,"review":15,"automated_count":28,"automated_acceptable":18,"automated_unsafe":0,"unnecessary_reviews":14,"accuracy":0.4418604651162791,"unsafe_rate":0.0,"review_rate":0.3488372093023256,"automated_coverage":0.6511627906976745,"conditional_correctness":0.6428571428571429,"conditional_unsafe":0.0,"unnecessary_review_rate":0.32558139534883723,"accuracy_wilson_95":{"lower":0.30433309839170514,"upper":0.5889238554717591,"confidence_level":0.95}},"confusion":{"raw":{"keep":{"keep":1,"drop":3,"mask":6,"pseudonymize":2,"generalize":0,"review":0},"drop":{"keep":0,"drop":13,"mask":1,"pseudonymize":2,"generalize":0,"review":0},"mask":{"keep":0,"drop":0,"mask":2,"pseudonymize":0,"generalize":0,"review":0},"pseudonymize":{"keep":0,"drop":0,"mask":0,"pseudonymize":2,"generalize":0,"review":0},"generalize":{"keep":0,"drop":3,"mask":1,"pseudonymize":2,"generalize":0,"review":0},"review":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0}},"effective":{"keep":{"keep":0,"drop":3,"mask":4,"pseudonymize":0,"generalize":0,"review":5},"drop":{"keep":0,"drop":12,"mask":1,"pseudonymize":0,"generalize":0,"review":3},"mask":{"keep":0,"drop":0,"mask":1,"pseudonymize":0,"generalize":0,"review":1},"pseudonymize":{"keep":0,"drop":0,"mask":0,"pseudonymize":1,"generalize":0,"review":1},"generalize":{"keep":0,"drop":2,"mask":0,"pseudonymize":0,"generalize":0,"review":4},"review":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0}}},"probability_metrics":{"singleton_n":38,"multiacceptable_n":5,"multiacceptable_coverage":0.11627906976744186,"nll":1.6308317478177536,"nll_zero_mass_n":0,"brier":0.737856320723167,"acceptable_set_n":43,"acceptable_set_nll":1.4601858457512549,"acceptable_set_zero_mass_n":0},"calibration_all_labels":{"equal_width":{"n":43,"bin_count":10,"ece":0.10310764919942911,"signed_mean_gap":-0.01803012214151753,"absolute_mean_gap":0.01803012214151753,"bins":[{"index":0,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.0,"upper":0.1,"upper_inclusive":false},{"index":1,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.1,"upper":0.2,"upper_inclusive":false},{"index":2,"count":2,"confidence":0.2705735903706583,"empirical_correctness":0.5,"signed_gap":-0.2294264096293417,"absolute_gap":0.2294264096293417,"lower":0.2,"upper":0.3,"upper_inclusive":false},{"index":3,"count":8,"confidence":0.35970902007124506,"empirical_correctness":0.375,"signed_gap":-0.015290979928754944,"absolute_gap":0.015290979928754944,"lower":0.3,"upper":0.4,"upper_inclusive":false},{"index":4,"count":9,"confidence":0.44503574209036056,"empirical_correctness":0.3333333333333333,"signed_gap":0.11170240875702725,"absolute_gap":0.11170240875702725,"lower":0.4,"upper":0.5,"upper_inclusive":false},{"index":5,"count":11,"confidence":0.5480132331934038,"empirical_correctness":0.6363636363636364,"signed_gap":-0.08835040317023257,"absolute_gap":0.08835040317023257,"lower":0.5,"upper":0.6,"upper_inclusive":false},{"index":6,"count":9,"confidence":0.6470939058813171,"empirical_correctness":0.5555555555555556,"signed_gap":0.09153835032576152,"absolute_gap":0.09153835032576152,"lower":0.6,"upper":0.7,"upper_inclusive":false},{"index":7,"count":3,"confidence":0.7116227906800702,"empirical_correctness":1.0,"signed_gap":-0.28837720931992983,"absolute_gap":0.28837720931992983,"lower":0.7,"upper":0.8,"upper_inclusive":false},{"index":8,"count":1,"confidence":0.813704637690718,"empirical_correctness":1.0,"signed_gap":-0.186295362309282,"absolute_gap":0.186295362309282,"lower":0.8,"upper":0.9,"upper_inclusive":false},{"index":9,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.9,"upper":1.0,"upper_inclusive":true}]},"equal_mass":{"n":43,"bin_count":10,"ece":0.18787367154275533,"signed_mean_gap":-0.018030122141517543,"absolute_mean_gap":0.018030122141517543,"bins":[{"index":0,"count":4,"confidence":0.2962012140279219,"empirical_correctness":0.5,"signed_gap":-0.20379878597207812,"absolute_gap":0.20379878597207812,"min_confidence":0.25835684030277944,"max_confidence":0.34350992183177204},{"index":1,"count":4,"confidence":0.3692913479840168,"empirical_correctness":0.25,"signed_gap":0.1192913479840168,"absolute_gap":0.1192913479840168,"min_confidence":0.3561049661322398,"max_confidence":0.3744927827537324},{"index":2,"count":4,"confidence":0.39438776609369675,"empirical_correctness":0.25,"signed_gap":0.14438776609369675,"absolute_gap":0.14438776609369675,"min_confidence":0.3766862622583553,"max_confidence":0.41257565219667114},{"index":3,"count":5,"confidence":0.44261130703209445,"empirical_correctness":0.4,"signed_gap":0.042611307032094425,"absolute_gap":0.042611307032094425,"min_confidence":0.4262070030923431,"max_confidence":0.474393806330433},{"index":4,"count":4,"confidence":0.4962159241329666,"empirical_correctness":0.5,"signed_gap":-0.0037840758670333874,"absolute_gap":0.0037840758670333874,"min_confidence":0.4798976160520341,"max_confidence":0.507968959245507},{"index":5,"count":4,"confidence":0.531817788698017,"empirical_correctness":1.0,"signed_gap":-0.468182211301983,"absolute_gap":0.468182211301983,"min_confidence":0.5141552659032764,"max_confidence":0.5606426531208145},{"index":6,"count":5,"confidence":0.577514777269003,"empirical_correctness":0.4,"signed_gap":0.177514777269003,"absolute_gap":0.177514777269003,"min_confidence":0.5657035450664544,"max_confidence":0.5898323858704645},{"index":7,"count":4,"confidence":0.6240723585775678,"empirical_correctness":0.25,"signed_gap":0.3740723585775678,"absolute_gap":0.3740723585775678,"min_confidence":0.6037280139379285,"max_confidence":0.6374413605988389},{"index":8,"count":4,"confidence":0.6591670903719977,"empirical_correctness":0.75,"signed_gap":-0.09083290962800228,"absolute_gap":0.09083290962800228,"min_confidence":0.6425282273936572,"max_confidence":0.6664330782868024},{"index":9,"count":5,"confidence":0.7278920733729041,"empirical_correctness":1.0,"signed_gap":-0.27210792662709593,"absolute_gap":0.27210792662709593,"min_confidence":0.6908873571335916,"max_confidence":0.813704637690718}]}},"calibration_singleton_labels":{"equal_width":{"n":38,"bin_count":10,"ece":0.10482290606626574,"signed_mean_gap":0.04088564791424592,"absolute_mean_gap":0.04088564791424592,"bins":[{"index":0,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.0,"upper":0.1,"upper_inclusive":false},{"index":1,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.1,"upper":0.2,"upper_inclusive":false},{"index":2,"count":2,"confidence":0.2705735903706583,"empirical_correctness":0.5,"signed_gap":-0.2294264096293417,"absolute_gap":0.2294264096293417,"lower":0.2,"upper":0.3,"upper_inclusive":false},{"index":3,"count":7,"confidence":0.3620231769625984,"empirical_correctness":0.2857142857142857,"signed_gap":0.0763088912483127,"absolute_gap":0.0763088912483127,"lower":0.3,"upper":0.4,"upper_inclusive":false},{"index":4,"count":9,"confidence":0.44503574209036056,"empirical_correctness":0.3333333333333333,"signed_gap":0.11170240875702725,"absolute_gap":0.11170240875702725,"lower":0.4,"upper":0.5,"upper_inclusive":false},{"index":5,"count":8,"confidence":0.5506416818933042,"empirical_correctness":0.5,"signed_gap":0.0506416818933042,"absolute_gap":0.0506416818933042,"lower":0.5,"upper":0.6,"upper_inclusive":false},{"index":6,"count":9,"confidence":0.6470939058813171,"empirical_correctness":0.5555555555555556,"signed_gap":0.09153835032576152,"absolute_gap":0.09153835032576152,"lower":0.6,"upper":0.7,"upper_inclusive":false},{"index":7,"count":2,"confidence":0.7151701383397944,"empirical_correctness":1.0,"signed_gap":-0.28482986166020563,"absolute_gap":0.28482986166020563,"lower":0.7,"upper":0.8,"upper_inclusive":false},{"index":8,"count":1,"confidence":0.813704637690718,"empirical_correctness":1.0,"signed_gap":-0.186295362309282,"absolute_gap":0.186295362309282,"lower":0.8,"upper":0.9,"upper_inclusive":false},{"index":9,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.9,"upper":1.0,"upper_inclusive":true}]},"equal_mass":{"n":38,"bin_count":10,"ece":0.200630847454243,"signed_mean_gap":0.04088564791424592,"absolute_mean_gap":0.04088564791424592,"bins":[{"index":0,"count":3,"confidence":0.2804316447599719,"empirical_correctness":0.3333333333333333,"signed_gap":-0.052901688573361416,"absolute_gap":0.052901688573361416,"min_confidence":0.25835684030277944,"max_confidence":0.3001477535385989},{"index":1,"count":4,"confidence":0.3692913479840168,"empirical_correctness":0.25,"signed_gap":0.1192913479840168,"absolute_gap":0.1192913479840168,"min_confidence":0.3561049661322398,"max_confidence":0.3744927827537324},{"index":2,"count":4,"confidence":0.39438776609369675,"empirical_correctness":0.25,"signed_gap":0.14438776609369675,"absolute_gap":0.14438776609369675,"min_confidence":0.3766862622583553,"max_confidence":0.41257565219667114},{"index":3,"count":4,"confidence":0.43466568220750984,"empirical_correctness":0.25,"signed_gap":0.18466568220750984,"absolute_gap":0.18466568220750984,"min_confidence":0.4262070030923431,"max_confidence":0.4540625789208064},{"index":4,"count":4,"confidence":0.48848148452936213,"empirical_correctness":0.5,"signed_gap":-0.011518515470637869,"absolute_gap":0.011518515470637869,"min_confidence":0.474393806330433,"max_confidence":0.507968959245507},{"index":5,"count":3,"confidence":0.5326732344502672,"empirical_correctness":1.0,"signed_gap":-0.46732676554973285,"absolute_gap":0.46732676554973285,"min_confidence":0.5141552659032764,"max_confidence":0.5606426531208145},{"index":6,"count":4,"confidence":0.5747861981375313,"empirical_correctness":0.25,"signed_gap":0.3247861981375313,"absolute_gap":0.3247861981375313,"min_confidence":0.5657035450664544,"max_confidence":0.5898323858704645},{"index":7,"count":4,"confidence":0.6240723585775678,"empirical_correctness":0.25,"signed_gap":0.3740723585775678,"absolute_gap":0.3740723585775678,"min_confidence":0.6037280139379285,"max_confidence":0.6374413605988389},{"index":8,"count":4,"confidence":0.6591670903719977,"empirical_correctness":0.75,"signed_gap":-0.09083290962800228,"absolute_gap":0.09083290962800228,"min_confidence":0.6425282273936572,"max_confidence":0.6664330782868024},{"index":9,"count":4,"confidence":0.7337330678759746,"empirical_correctness":1.0,"signed_gap":-0.2662669321240254,"absolute_gap":0.2662669321240254,"min_confidence":0.6908873571335916,"max_confidence":0.813704637690718}]}},"latency_ms":{"n":43,"median":88.39090005494654,"p95":93.87106990907341},"per_action":{"keep":{"n":12,"raw":{"n":12,"acceptable":1,"unsafe":0,"review":0,"automated_count":12,"automated_acceptable":1,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":0.08333333333333333,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.08333333333333333,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.014865094404917095,"upper":0.35387991114111694,"confidence_level":0.95}},"effective":{"n":12,"acceptable":0,"unsafe":0,"review":5,"automated_count":7,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":5,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":0.4166666666666667,"automated_coverage":0.5833333333333334,"conditional_correctness":0.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.4166666666666667,"accuracy_wilson_95":{"lower":2.7755575615628914e-17,"upper":0.24249400665524085,"confidence_level":0.95}}},"drop":{"n":16,"raw":{"n":16,"acceptable":13,"unsafe":2,"review":0,"automated_count":16,"automated_acceptable":13,"automated_unsafe":2,"unnecessary_reviews":0,"accuracy":0.8125,"unsafe_rate":0.125,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.8125,"conditional_unsafe":0.125,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.5699111903802586,"upper":0.9340840092857187,"confidence_level":0.95}},"effective":{"n":16,"acceptable":12,"unsafe":0,"review":3,"automated_count":13,"automated_acceptable":12,"automated_unsafe":0,"unnecessary_reviews":3,"accuracy":0.75,"unsafe_rate":0.0,"review_rate":0.1875,"automated_coverage":0.8125,"conditional_correctness":0.9230769230769231,"conditional_unsafe":0.0,"unnecessary_review_rate":0.1875,"accuracy_wilson_95":{"lower":0.5050168346449122,"upper":0.8981793250878696,"confidence_level":0.95}}},"mask":{"n":2,"raw":{"n":2,"acceptable":2,"unsafe":0,"review":0,"automated_count":2,"automated_acceptable":2,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":1.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.34238022750665303,"upper":1.0,"confidence_level":0.95}},"effective":{"n":2,"acceptable":1,"unsafe":0,"review":1,"automated_count":1,"automated_acceptable":1,"automated_unsafe":0,"unnecessary_reviews":1,"accuracy":0.5,"unsafe_rate":0.0,"review_rate":0.5,"automated_coverage":0.5,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.5,"accuracy_wilson_95":{"lower":0.09453120573423074,"upper":0.9054687942657693,"confidence_level":0.95}}},"pseudonymize":{"n":2,"raw":{"n":2,"acceptable":2,"unsafe":0,"review":0,"automated_count":2,"automated_acceptable":2,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":1.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.34238022750665303,"upper":1.0,"confidence_level":0.95}},"effective":{"n":2,"acceptable":1,"unsafe":0,"review":1,"automated_count":1,"automated_acceptable":1,"automated_unsafe":0,"unnecessary_reviews":1,"accuracy":0.5,"unsafe_rate":0.0,"review_rate":0.5,"automated_coverage":0.5,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.5,"accuracy_wilson_95":{"lower":0.09453120573423074,"upper":0.9054687942657693,"confidence_level":0.95}}},"generalize":{"n":6,"raw":{"n":6,"acceptable":0,"unsafe":0,"review":0,"automated_count":6,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":2.7755575615628914e-17,"upper":0.3903342879021653,"confidence_level":0.95}},"effective":{"n":6,"acceptable":0,"unsafe":0,"review":4,"automated_count":2,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":4,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":0.6666666666666666,"automated_coverage":0.3333333333333333,"conditional_correctness":0.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.6666666666666666,"accuracy_wilson_95":{"lower":2.7755575615628914e-17,"upper":0.3903342879021653,"confidence_level":0.95}}},"review":{"n":0,"raw":{"n":0,"acceptable":0,"unsafe":0,"review":0,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":null,"unsafe_rate":null,"review_rate":null,"automated_coverage":null,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":null},"effective":{"n":0,"acceptable":0,"unsafe":0,"review":0,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":null,"unsafe_rate":null,"review_rate":null,"automated_coverage":null,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":null}}}},"challenge56-no_samples":{"n":56,"raw":{"n":56,"acceptable":37,"unsafe":18,"review":1,"automated_count":55,"automated_acceptable":37,"automated_unsafe":18,"unnecessary_reviews":1,"accuracy":0.6607142857142857,"unsafe_rate":0.32142857142857145,"review_rate":0.017857142857142856,"automated_coverage":0.9821428571428571,"conditional_correctness":0.6727272727272727,"conditional_unsafe":0.32727272727272727,"unnecessary_review_rate":0.017857142857142856,"accuracy_wilson_95":{"lower":0.5299945300706023,"upper":0.770800275984081,"confidence_level":0.95}},"effective":{"n":56,"acceptable":29,"unsafe":0,"review":29,"automated_count":27,"automated_acceptable":27,"automated_unsafe":0,"unnecessary_reviews":27,"accuracy":0.5178571428571429,"unsafe_rate":0.0,"review_rate":0.5178571428571429,"automated_coverage":0.48214285714285715,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.48214285714285715,"accuracy_wilson_95":{"lower":0.39010383924176983,"upper":0.6433178058754173,"confidence_level":0.95}},"confusion":{"raw":{"keep":{"keep":8,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0},"drop":{"keep":7,"drop":14,"mask":0,"pseudonymize":0,"generalize":0,"review":1},"mask":{"keep":0,"drop":0,"mask":7,"pseudonymize":1,"generalize":0,"review":0},"pseudonymize":{"keep":8,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0},"generalize":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":8,"review":0},"review":{"keep":2,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0}},"effective":{"keep":{"keep":5,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":3},"drop":{"keep":0,"drop":14,"mask":0,"pseudonymize":0,"generalize":0,"review":8},"mask":{"keep":0,"drop":0,"mask":3,"pseudonymize":0,"generalize":0,"review":5},"pseudonymize":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":8},"generalize":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":5,"review":3},"review":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":2}}},"probability_metrics":{"singleton_n":56,"multiacceptable_n":0,"multiacceptable_coverage":0.0,"nll":1.0302259159307376,"nll_zero_mass_n":0,"brier":0.5001349276122107,"acceptable_set_n":56,"acceptable_set_nll":1.0302259159307376,"acceptable_set_zero_mass_n":0},"calibration_all_labels":{"equal_width":{"n":56,"bin_count":10,"ece":0.20305978540450934,"signed_mean_gap":-0.12079416254779606,"absolute_mean_gap":0.12079416254779606,"bins":[{"index":0,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.0,"upper":0.1,"upper_inclusive":false},{"index":1,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.1,"upper":0.2,"upper_inclusive":false},{"index":2,"count":2,"confidence":0.2876505845732523,"empirical_correctness":0.0,"signed_gap":0.2876505845732523,"absolute_gap":0.2876505845732523,"lower":0.2,"upper":0.3,"upper_inclusive":false},{"index":3,"count":10,"confidence":0.3458028874934226,"empirical_correctness":0.2,"signed_gap":0.14580288749342257,"absolute_gap":0.14580288749342257,"lower":0.3,"upper":0.4,"upper_inclusive":false},{"index":4,"count":19,"confidence":0.45654149762501023,"empirical_correctness":0.7368421052631579,"signed_gap":-0.2803006076381476,"absolute_gap":0.2803006076381476,"lower":0.4,"upper":0.5,"upper_inclusive":false},{"index":5,"count":6,"confidence":0.5450178993178736,"empirical_correctness":0.5,"signed_gap":0.04501789931787359,"absolute_gap":0.04501789931787359,"lower":0.5,"upper":0.6,"upper_inclusive":false},{"index":6,"count":6,"confidence":0.6375419488098365,"empirical_correctness":0.8333333333333334,"signed_gap":-0.19579138452349687,"absolute_gap":0.19579138452349687,"lower":0.6,"upper":0.7,"upper_inclusive":false},{"index":7,"count":6,"confidence":0.7540461294224109,"empirical_correctness":1.0,"signed_gap":-0.2459538705775891,"absolute_gap":0.2459538705775891,"lower":0.7,"upper":0.8,"upper_inclusive":false},{"index":8,"count":7,"confidence":0.8440389332952528,"empirical_correctness":1.0,"signed_gap":-0.1559610667047472,"absolute_gap":0.1559610667047472,"lower":0.8,"upper":0.9,"upper_inclusive":false},{"index":9,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.9,"upper":1.0,"upper_inclusive":true}]},"equal_mass":{"n":56,"bin_count":10,"ece":0.23160797443517586,"signed_mean_gap":-0.12079416254779608,"absolute_mean_gap":0.12079416254779608,"bins":[{"index":0,"count":5,"confidence":0.3023092141782525,"empirical_correctness":0.2,"signed_gap":0.1023092141782525,"absolute_gap":0.1023092141782525,"min_confidence":0.28248323918165164,"max_confidence":0.31993033298149764},{"index":1,"count":6,"confidence":0.3546312167357987,"empirical_correctness":0.16666666666666666,"signed_gap":0.18796455006913207,"absolute_gap":0.18796455006913207,"min_confidence":0.3213759744926257,"max_confidence":0.38282287632624257},{"index":2,"count":5,"confidence":0.409698823472551,"empirical_correctness":0.8,"signed_gap":-0.39030117652744906,"absolute_gap":0.39030117652744906,"min_confidence":0.39399667277467615,"max_confidence":0.4191713436179356},{"index":3,"count":6,"confidence":0.45271541876179544,"empirical_correctness":0.8333333333333334,"signed_gap":-0.38061791457153793,"absolute_gap":0.38061791457153793,"min_confidence":0.43539888061791043,"max_confidence":0.4634383683271715},{"index":4,"count":6,"confidence":0.47204068448018993,"empirical_correctness":0.5,"signed_gap":-0.02795931551981007,"absolute_gap":0.02795931551981007,"min_confidence":0.46394805226544256,"max_confidence":0.4797451422172496},{"index":5,"count":5,"confidence":0.5038754207632377,"empirical_correctness":0.8,"signed_gap":-0.2961245792367624,"absolute_gap":0.2961245792367624,"min_confidence":0.4872618860796241,"max_confidence":0.527982592567772},{"index":6,"count":6,"confidence":0.5772422269234297,"empirical_correctness":0.3333333333333333,"signed_gap":0.24390889359009643,"absolute_gap":0.24390889359009643,"min_confidence":0.5328793194455528,"max_confidence":0.6304898219944015},{"index":7,"count":5,"confidence":0.6619110889532316,"empirical_correctness":1.0,"signed_gap":-0.3380889110467684,"absolute_gap":0.3380889110467684,"min_confidence":0.6325401618489972,"max_confidence":0.7257724305214627},{"index":8,"count":6,"confidence":0.7712630144630447,"empirical_correctness":1.0,"signed_gap":-0.22873698553695532,"absolute_gap":0.22873698553695532,"min_confidence":0.7376341224030051,"max_confidence":0.8290737407652654},{"index":9,"count":6,"confidence":0.8465331320502507,"empirical_correctness":1.0,"signed_gap":-0.15346686794974929,"absolute_gap":0.15346686794974929,"min_confidence":0.8339013921428919,"max_confidence":0.8578684608335652}]}},"calibration_singleton_labels":{"equal_width":{"n":56,"bin_count":10,"ece":0.20305978540450934,"signed_mean_gap":-0.12079416254779606,"absolute_mean_gap":0.12079416254779606,"bins":[{"index":0,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.0,"upper":0.1,"upper_inclusive":false},{"index":1,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.1,"upper":0.2,"upper_inclusive":false},{"index":2,"count":2,"confidence":0.2876505845732523,"empirical_correctness":0.0,"signed_gap":0.2876505845732523,"absolute_gap":0.2876505845732523,"lower":0.2,"upper":0.3,"upper_inclusive":false},{"index":3,"count":10,"confidence":0.3458028874934226,"empirical_correctness":0.2,"signed_gap":0.14580288749342257,"absolute_gap":0.14580288749342257,"lower":0.3,"upper":0.4,"upper_inclusive":false},{"index":4,"count":19,"confidence":0.45654149762501023,"empirical_correctness":0.7368421052631579,"signed_gap":-0.2803006076381476,"absolute_gap":0.2803006076381476,"lower":0.4,"upper":0.5,"upper_inclusive":false},{"index":5,"count":6,"confidence":0.5450178993178736,"empirical_correctness":0.5,"signed_gap":0.04501789931787359,"absolute_gap":0.04501789931787359,"lower":0.5,"upper":0.6,"upper_inclusive":false},{"index":6,"count":6,"confidence":0.6375419488098365,"empirical_correctness":0.8333333333333334,"signed_gap":-0.19579138452349687,"absolute_gap":0.19579138452349687,"lower":0.6,"upper":0.7,"upper_inclusive":false},{"index":7,"count":6,"confidence":0.7540461294224109,"empirical_correctness":1.0,"signed_gap":-0.2459538705775891,"absolute_gap":0.2459538705775891,"lower":0.7,"upper":0.8,"upper_inclusive":false},{"index":8,"count":7,"confidence":0.8440389332952528,"empirical_correctness":1.0,"signed_gap":-0.1559610667047472,"absolute_gap":0.1559610667047472,"lower":0.8,"upper":0.9,"upper_inclusive":false},{"index":9,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.9,"upper":1.0,"upper_inclusive":true}]},"equal_mass":{"n":56,"bin_count":10,"ece":0.23160797443517586,"signed_mean_gap":-0.12079416254779608,"absolute_mean_gap":0.12079416254779608,"bins":[{"index":0,"count":5,"confidence":0.3023092141782525,"empirical_correctness":0.2,"signed_gap":0.1023092141782525,"absolute_gap":0.1023092141782525,"min_confidence":0.28248323918165164,"max_confidence":0.31993033298149764},{"index":1,"count":6,"confidence":0.3546312167357987,"empirical_correctness":0.16666666666666666,"signed_gap":0.18796455006913207,"absolute_gap":0.18796455006913207,"min_confidence":0.3213759744926257,"max_confidence":0.38282287632624257},{"index":2,"count":5,"confidence":0.409698823472551,"empirical_correctness":0.8,"signed_gap":-0.39030117652744906,"absolute_gap":0.39030117652744906,"min_confidence":0.39399667277467615,"max_confidence":0.4191713436179356},{"index":3,"count":6,"confidence":0.45271541876179544,"empirical_correctness":0.8333333333333334,"signed_gap":-0.38061791457153793,"absolute_gap":0.38061791457153793,"min_confidence":0.43539888061791043,"max_confidence":0.4634383683271715},{"index":4,"count":6,"confidence":0.47204068448018993,"empirical_correctness":0.5,"signed_gap":-0.02795931551981007,"absolute_gap":0.02795931551981007,"min_confidence":0.46394805226544256,"max_confidence":0.4797451422172496},{"index":5,"count":5,"confidence":0.5038754207632377,"empirical_correctness":0.8,"signed_gap":-0.2961245792367624,"absolute_gap":0.2961245792367624,"min_confidence":0.4872618860796241,"max_confidence":0.527982592567772},{"index":6,"count":6,"confidence":0.5772422269234297,"empirical_correctness":0.3333333333333333,"signed_gap":0.24390889359009643,"absolute_gap":0.24390889359009643,"min_confidence":0.5328793194455528,"max_confidence":0.6304898219944015},{"index":7,"count":5,"confidence":0.6619110889532316,"empirical_correctness":1.0,"signed_gap":-0.3380889110467684,"absolute_gap":0.3380889110467684,"min_confidence":0.6325401618489972,"max_confidence":0.7257724305214627},{"index":8,"count":6,"confidence":0.7712630144630447,"empirical_correctness":1.0,"signed_gap":-0.22873698553695532,"absolute_gap":0.22873698553695532,"min_confidence":0.7376341224030051,"max_confidence":0.8290737407652654},{"index":9,"count":6,"confidence":0.8465331320502507,"empirical_correctness":1.0,"signed_gap":-0.15346686794974929,"absolute_gap":0.15346686794974929,"min_confidence":0.8339013921428919,"max_confidence":0.8578684608335652}]}},"latency_ms":{"n":56,"median":68.94064997322857,"p95":90.12602502480149},"per_action":{"keep":{"n":8,"raw":{"n":8,"acceptable":8,"unsafe":0,"review":0,"automated_count":8,"automated_acceptable":8,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":1.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.6755924351161198,"upper":1.0,"confidence_level":0.95}},"effective":{"n":8,"acceptable":5,"unsafe":0,"review":3,"automated_count":5,"automated_acceptable":5,"automated_unsafe":0,"unnecessary_reviews":3,"accuracy":0.625,"unsafe_rate":0.0,"review_rate":0.375,"automated_coverage":0.625,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.375,"accuracy_wilson_95":{"lower":0.30574239460262737,"upper":0.8631557141764026,"confidence_level":0.95}}},"drop":{"n":22,"raw":{"n":22,"acceptable":14,"unsafe":7,"review":1,"automated_count":21,"automated_acceptable":14,"automated_unsafe":7,"unnecessary_reviews":1,"accuracy":0.6363636363636364,"unsafe_rate":0.3181818181818182,"review_rate":0.045454545454545456,"automated_coverage":0.9545454545454546,"conditional_correctness":0.6666666666666666,"conditional_unsafe":0.3333333333333333,"unnecessary_review_rate":0.045454545454545456,"accuracy_wilson_95":{"lower":0.42951708090131013,"upper":0.8026679532963387,"confidence_level":0.95}},"effective":{"n":22,"acceptable":14,"unsafe":0,"review":8,"automated_count":14,"automated_acceptable":14,"automated_unsafe":0,"unnecessary_reviews":8,"accuracy":0.6363636363636364,"unsafe_rate":0.0,"review_rate":0.36363636363636365,"automated_coverage":0.6363636363636364,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.36363636363636365,"accuracy_wilson_95":{"lower":0.42951708090131013,"upper":0.8026679532963387,"confidence_level":0.95}}},"mask":{"n":8,"raw":{"n":8,"acceptable":7,"unsafe":1,"review":0,"automated_count":8,"automated_acceptable":7,"automated_unsafe":1,"unnecessary_reviews":0,"accuracy":0.875,"unsafe_rate":0.125,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.875,"conditional_unsafe":0.125,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.5291118177871464,"upper":0.9775825085499433,"confidence_level":0.95}},"effective":{"n":8,"acceptable":3,"unsafe":0,"review":5,"automated_count":3,"automated_acceptable":3,"automated_unsafe":0,"unnecessary_reviews":5,"accuracy":0.375,"unsafe_rate":0.0,"review_rate":0.625,"automated_coverage":0.375,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.625,"accuracy_wilson_95":{"lower":0.13684428582359742,"upper":0.6942576053973727,"confidence_level":0.95}}},"pseudonymize":{"n":8,"raw":{"n":8,"acceptable":0,"unsafe":8,"review":0,"automated_count":8,"automated_acceptable":0,"automated_unsafe":8,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":1.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":1.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.32440756488388023,"confidence_level":0.95}},"effective":{"n":8,"acceptable":0,"unsafe":0,"review":8,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":8,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.32440756488388023,"confidence_level":0.95}}},"generalize":{"n":8,"raw":{"n":8,"acceptable":8,"unsafe":0,"review":0,"automated_count":8,"automated_acceptable":8,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":1.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.6755924351161198,"upper":1.0,"confidence_level":0.95}},"effective":{"n":8,"acceptable":5,"unsafe":0,"review":3,"automated_count":5,"automated_acceptable":5,"automated_unsafe":0,"unnecessary_reviews":3,"accuracy":0.625,"unsafe_rate":0.0,"review_rate":0.375,"automated_coverage":0.625,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.375,"accuracy_wilson_95":{"lower":0.30574239460262737,"upper":0.8631557141764026,"confidence_level":0.95}}},"review":{"n":2,"raw":{"n":2,"acceptable":0,"unsafe":2,"review":0,"automated_count":2,"automated_acceptable":0,"automated_unsafe":2,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":1.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":1.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.6576197724933469,"confidence_level":0.95}},"effective":{"n":2,"acceptable":2,"unsafe":0,"review":2,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":1.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.34238022750665303,"upper":1.0,"confidence_level":0.95}}}}},"challenge56-compact_no_samples":{"n":56,"raw":{"n":56,"acceptable":31,"unsafe":24,"review":0,"automated_count":56,"automated_acceptable":31,"automated_unsafe":24,"unnecessary_reviews":0,"accuracy":0.5535714285714286,"unsafe_rate":0.42857142857142855,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.5535714285714286,"conditional_unsafe":0.42857142857142855,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.4241321478993947,"upper":0.6761327874521665,"confidence_level":0.95}},"effective":{"n":56,"acceptable":27,"unsafe":0,"review":31,"automated_count":25,"automated_acceptable":25,"automated_unsafe":0,"unnecessary_reviews":29,"accuracy":0.48214285714285715,"unsafe_rate":0.0,"review_rate":0.5535714285714286,"automated_coverage":0.44642857142857145,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.5178571428571429,"accuracy_wilson_95":{"lower":0.3566821941245828,"upper":0.6098961607582303,"confidence_level":0.95}},"confusion":{"raw":{"keep":{"keep":7,"drop":0,"mask":1,"pseudonymize":0,"generalize":0,"review":0},"drop":{"keep":8,"drop":14,"mask":0,"pseudonymize":0,"generalize":0,"review":0},"mask":{"keep":0,"drop":0,"mask":8,"pseudonymize":0,"generalize":0,"review":0},"pseudonymize":{"keep":8,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0},"generalize":{"keep":6,"drop":0,"mask":0,"pseudonymize":0,"generalize":2,"review":0},"review":{"keep":2,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0}},"effective":{"keep":{"keep":3,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":5},"drop":{"keep":0,"drop":14,"mask":0,"pseudonymize":0,"generalize":0,"review":8},"mask":{"keep":0,"drop":0,"mask":8,"pseudonymize":0,"generalize":0,"review":0},"pseudonymize":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":8},"generalize":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":8},"review":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":2}}},"probability_metrics":{"singleton_n":56,"multiacceptable_n":0,"multiacceptable_coverage":0.0,"nll":0.9656053493547737,"nll_zero_mass_n":0,"brier":0.4821154927582826,"acceptable_set_n":56,"acceptable_set_nll":0.9656053493547737,"acceptable_set_zero_mass_n":0},"calibration_all_labels":{"equal_width":{"n":56,"bin_count":10,"ece":0.17744278052423187,"signed_mean_gap":-0.0024622003100154355,"absolute_mean_gap":0.0024622003100154355,"bins":[{"index":0,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.0,"upper":0.1,"upper_inclusive":false},{"index":1,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.1,"upper":0.2,"upper_inclusive":false},{"index":2,"count":2,"confidence":0.2867530854048181,"empirical_correctness":0.0,"signed_gap":0.2867530854048181,"absolute_gap":0.2867530854048181,"lower":0.2,"upper":0.3,"upper_inclusive":false},{"index":3,"count":10,"confidence":0.36305317622292566,"empirical_correctness":0.3,"signed_gap":0.06305317622292567,"absolute_gap":0.06305317622292567,"lower":0.3,"upper":0.4,"upper_inclusive":false},{"index":4,"count":14,"confidence":0.44730134473372424,"empirical_correctness":0.35714285714285715,"signed_gap":0.09015848759086709,"absolute_gap":0.09015848759086709,"lower":0.4,"upper":0.5,"upper_inclusive":false},{"index":5,"count":10,"confidence":0.5433199486687028,"empirical_correctness":0.3,"signed_gap":0.24331994866870282,"absolute_gap":0.24331994866870282,"lower":0.5,"upper":0.6,"upper_inclusive":false},{"index":6,"count":7,"confidence":0.6621683393159998,"empirical_correctness":1.0,"signed_gap":-0.33783166068400017,"absolute_gap":0.33783166068400017,"lower":0.6,"upper":0.7,"upper_inclusive":false},{"index":7,"count":6,"confidence":0.7540694734752758,"empirical_correctness":1.0,"signed_gap":-0.24593052652472425,"absolute_gap":0.24593052652472425,"lower":0.7,"upper":0.8,"upper_inclusive":false},{"index":8,"count":7,"confidence":0.8290093315110603,"empirical_correctness":1.0,"signed_gap":-0.17099066848893973,"absolute_gap":0.17099066848893973,"lower":0.8,"upper":0.9,"upper_inclusive":false},{"index":9,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.9,"upper":1.0,"upper_inclusive":true}]},"equal_mass":{"n":56,"bin_count":10,"ece":0.16222075536125097,"signed_mean_gap":-0.00246220031001543,"absolute_mean_gap":0.00246220031001543,"bins":[{"index":0,"count":5,"confidence":0.31366919240942287,"empirical_correctness":0.0,"signed_gap":0.31366919240942287,"absolute_gap":0.31366919240942287,"min_confidence":0.28469970194260963,"max_confidence":0.3474578777534521},{"index":1,"count":6,"confidence":0.373204282556324,"empirical_correctness":0.3333333333333333,"signed_gap":0.03987094922299067,"absolute_gap":0.03987094922299067,"min_confidence":0.35112583097148525,"max_confidence":0.38901967622647454},{"index":2,"count":5,"confidence":0.4134387741315986,"empirical_correctness":0.4,"signed_gap":0.013438774131598552,"absolute_gap":0.013438774131598552,"min_confidence":0.39646627565383463,"max_confidence":0.4373962326797296},{"index":3,"count":6,"confidence":0.44667370888578994,"empirical_correctness":0.5,"signed_gap":-0.053326291114210056,"absolute_gap":0.053326291114210056,"min_confidence":0.43951840964691236,"max_confidence":0.45582304271130003},{"index":4,"count":6,"confidence":0.4898816126977535,"empirical_correctness":0.16666666666666666,"signed_gap":0.3232149460310868,"absolute_gap":0.3232149460310868,"min_confidence":0.47198458989583686,"max_confidence":0.5145367506659534},{"index":5,"count":5,"confidence":0.5318368674410048,"empirical_correctness":0.4,"signed_gap":0.13183686744100476,"absolute_gap":0.13183686744100476,"min_confidence":0.5146533488792336,"max_confidence":0.5726753762872357},{"index":6,"count":6,"confidence":0.6104195304371126,"empirical_correctness":0.6666666666666666,"signed_gap":-0.05624713622955402,"absolute_gap":0.05624713622955402,"min_confidence":0.5774891341251159,"max_confidence":0.6439634518626737},{"index":7,"count":5,"confidence":0.6842699296917291,"empirical_correctness":1.0,"signed_gap":-0.31573007030827094,"absolute_gap":0.31573007030827094,"min_confidence":0.6630547683340363,"max_confidence":0.7025140046205957},{"index":8,"count":6,"confidence":0.7714286718603592,"empirical_correctness":1.0,"signed_gap":-0.22857132813964076,"absolute_gap":0.22857132813964076,"min_confidence":0.7127481686492737,"max_confidence":0.8066691949310969},{"index":9,"count":6,"confidence":0.8327326876077207,"empirical_correctness":1.0,"signed_gap":-0.16726731239227932,"absolute_gap":0.16726731239227932,"min_confidence":0.8123720836789712,"max_confidence":0.8462708937002756}]}},"calibration_singleton_labels":{"equal_width":{"n":56,"bin_count":10,"ece":0.17744278052423187,"signed_mean_gap":-0.0024622003100154355,"absolute_mean_gap":0.0024622003100154355,"bins":[{"index":0,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.0,"upper":0.1,"upper_inclusive":false},{"index":1,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.1,"upper":0.2,"upper_inclusive":false},{"index":2,"count":2,"confidence":0.2867530854048181,"empirical_correctness":0.0,"signed_gap":0.2867530854048181,"absolute_gap":0.2867530854048181,"lower":0.2,"upper":0.3,"upper_inclusive":false},{"index":3,"count":10,"confidence":0.36305317622292566,"empirical_correctness":0.3,"signed_gap":0.06305317622292567,"absolute_gap":0.06305317622292567,"lower":0.3,"upper":0.4,"upper_inclusive":false},{"index":4,"count":14,"confidence":0.44730134473372424,"empirical_correctness":0.35714285714285715,"signed_gap":0.09015848759086709,"absolute_gap":0.09015848759086709,"lower":0.4,"upper":0.5,"upper_inclusive":false},{"index":5,"count":10,"confidence":0.5433199486687028,"empirical_correctness":0.3,"signed_gap":0.24331994866870282,"absolute_gap":0.24331994866870282,"lower":0.5,"upper":0.6,"upper_inclusive":false},{"index":6,"count":7,"confidence":0.6621683393159998,"empirical_correctness":1.0,"signed_gap":-0.33783166068400017,"absolute_gap":0.33783166068400017,"lower":0.6,"upper":0.7,"upper_inclusive":false},{"index":7,"count":6,"confidence":0.7540694734752758,"empirical_correctness":1.0,"signed_gap":-0.24593052652472425,"absolute_gap":0.24593052652472425,"lower":0.7,"upper":0.8,"upper_inclusive":false},{"index":8,"count":7,"confidence":0.8290093315110603,"empirical_correctness":1.0,"signed_gap":-0.17099066848893973,"absolute_gap":0.17099066848893973,"lower":0.8,"upper":0.9,"upper_inclusive":false},{"index":9,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.9,"upper":1.0,"upper_inclusive":true}]},"equal_mass":{"n":56,"bin_count":10,"ece":0.16222075536125097,"signed_mean_gap":-0.00246220031001543,"absolute_mean_gap":0.00246220031001543,"bins":[{"index":0,"count":5,"confidence":0.31366919240942287,"empirical_correctness":0.0,"signed_gap":0.31366919240942287,"absolute_gap":0.31366919240942287,"min_confidence":0.28469970194260963,"max_confidence":0.3474578777534521},{"index":1,"count":6,"confidence":0.373204282556324,"empirical_correctness":0.3333333333333333,"signed_gap":0.03987094922299067,"absolute_gap":0.03987094922299067,"min_confidence":0.35112583097148525,"max_confidence":0.38901967622647454},{"index":2,"count":5,"confidence":0.4134387741315986,"empirical_correctness":0.4,"signed_gap":0.013438774131598552,"absolute_gap":0.013438774131598552,"min_confidence":0.39646627565383463,"max_confidence":0.4373962326797296},{"index":3,"count":6,"confidence":0.44667370888578994,"empirical_correctness":0.5,"signed_gap":-0.053326291114210056,"absolute_gap":0.053326291114210056,"min_confidence":0.43951840964691236,"max_confidence":0.45582304271130003},{"index":4,"count":6,"confidence":0.4898816126977535,"empirical_correctness":0.16666666666666666,"signed_gap":0.3232149460310868,"absolute_gap":0.3232149460310868,"min_confidence":0.47198458989583686,"max_confidence":0.5145367506659534},{"index":5,"count":5,"confidence":0.5318368674410048,"empirical_correctness":0.4,"signed_gap":0.13183686744100476,"absolute_gap":0.13183686744100476,"min_confidence":0.5146533488792336,"max_confidence":0.5726753762872357},{"index":6,"count":6,"confidence":0.6104195304371126,"empirical_correctness":0.6666666666666666,"signed_gap":-0.05624713622955402,"absolute_gap":0.05624713622955402,"min_confidence":0.5774891341251159,"max_confidence":0.6439634518626737},{"index":7,"count":5,"confidence":0.6842699296917291,"empirical_correctness":1.0,"signed_gap":-0.31573007030827094,"absolute_gap":0.31573007030827094,"min_confidence":0.6630547683340363,"max_confidence":0.7025140046205957},{"index":8,"count":6,"confidence":0.7714286718603592,"empirical_correctness":1.0,"signed_gap":-0.22857132813964076,"absolute_gap":0.22857132813964076,"min_confidence":0.7127481686492737,"max_confidence":0.8066691949310969},{"index":9,"count":6,"confidence":0.8327326876077207,"empirical_correctness":1.0,"signed_gap":-0.16726731239227932,"absolute_gap":0.16726731239227932,"min_confidence":0.8123720836789712,"max_confidence":0.8462708937002756}]}},"latency_ms":{"n":56,"median":78.08010000735521,"p95":101.22192505514249},"per_action":{"keep":{"n":8,"raw":{"n":8,"acceptable":7,"unsafe":0,"review":0,"automated_count":8,"automated_acceptable":7,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":0.875,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.875,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.5291118177871464,"upper":0.9775825085499433,"confidence_level":0.95}},"effective":{"n":8,"acceptable":3,"unsafe":0,"review":5,"automated_count":3,"automated_acceptable":3,"automated_unsafe":0,"unnecessary_reviews":5,"accuracy":0.375,"unsafe_rate":0.0,"review_rate":0.625,"automated_coverage":0.375,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.625,"accuracy_wilson_95":{"lower":0.13684428582359742,"upper":0.6942576053973727,"confidence_level":0.95}}},"drop":{"n":22,"raw":{"n":22,"acceptable":14,"unsafe":8,"review":0,"automated_count":22,"automated_acceptable":14,"automated_unsafe":8,"unnecessary_reviews":0,"accuracy":0.6363636363636364,"unsafe_rate":0.36363636363636365,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.6363636363636364,"conditional_unsafe":0.36363636363636365,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.42951708090131013,"upper":0.8026679532963387,"confidence_level":0.95}},"effective":{"n":22,"acceptable":14,"unsafe":0,"review":8,"automated_count":14,"automated_acceptable":14,"automated_unsafe":0,"unnecessary_reviews":8,"accuracy":0.6363636363636364,"unsafe_rate":0.0,"review_rate":0.36363636363636365,"automated_coverage":0.6363636363636364,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.36363636363636365,"accuracy_wilson_95":{"lower":0.42951708090131013,"upper":0.8026679532963387,"confidence_level":0.95}}},"mask":{"n":8,"raw":{"n":8,"acceptable":8,"unsafe":0,"review":0,"automated_count":8,"automated_acceptable":8,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":1.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.6755924351161198,"upper":1.0,"confidence_level":0.95}},"effective":{"n":8,"acceptable":8,"unsafe":0,"review":0,"automated_count":8,"automated_acceptable":8,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":1.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.6755924351161198,"upper":1.0,"confidence_level":0.95}}},"pseudonymize":{"n":8,"raw":{"n":8,"acceptable":0,"unsafe":8,"review":0,"automated_count":8,"automated_acceptable":0,"automated_unsafe":8,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":1.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":1.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.32440756488388023,"confidence_level":0.95}},"effective":{"n":8,"acceptable":0,"unsafe":0,"review":8,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":8,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.32440756488388023,"confidence_level":0.95}}},"generalize":{"n":8,"raw":{"n":8,"acceptable":2,"unsafe":6,"review":0,"automated_count":8,"automated_acceptable":2,"automated_unsafe":6,"unnecessary_reviews":0,"accuracy":0.25,"unsafe_rate":0.75,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.25,"conditional_unsafe":0.75,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.071479212752109,"upper":0.5907245696898311,"confidence_level":0.95}},"effective":{"n":8,"acceptable":0,"unsafe":0,"review":8,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":8,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.32440756488388023,"confidence_level":0.95}}},"review":{"n":2,"raw":{"n":2,"acceptable":0,"unsafe":2,"review":0,"automated_count":2,"automated_acceptable":0,"automated_unsafe":2,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":1.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":1.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.6576197724933469,"confidence_level":0.95}},"effective":{"n":2,"acceptable":2,"unsafe":0,"review":2,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":1.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.34238022750665303,"upper":1.0,"confidence_level":0.95}}}}}},"composite":{"n":198,"scope":"descriptive sum across four correlated panels","raw":{"n":198,"acceptable":107,"unsafe":50,"review":1,"automated_count":197,"automated_acceptable":107,"automated_unsafe":50,"unnecessary_reviews":1,"accuracy":0.5404040404040404,"unsafe_rate":0.25252525252525254,"review_rate":0.005050505050505051,"automated_coverage":0.9949494949494949,"conditional_correctness":0.5431472081218274,"conditional_unsafe":0.25380710659898476,"unnecessary_review_rate":0.005050505050505051},"effective":{"n":198,"acceptable":89,"unsafe":3,"review":89,"automated_count":109,"automated_acceptable":83,"automated_unsafe":3,"unnecessary_reviews":83,"accuracy":0.4494949494949495,"unsafe_rate":0.015151515151515152,"review_rate":0.4494949494949495,"automated_coverage":0.5505050505050505,"conditional_correctness":0.7614678899082569,"conditional_unsafe":0.027522935779816515,"unnecessary_review_rate":0.41919191919191917}}},{"id":"Horizon-Labs--multilingual-zeroshot-small","platform":"Local","model_id":"Horizon-Labs/multilingual-zeroshot-small","revision":"3fe1f0c48b589cf6220fd21a628a1dd56d3d41f8","loading":"float32 / none","scoring_kind":"Adapted NLI","settings":{"dtype":"float32","quantization":null,"max_tokens":4096,"effective_max_tokens":4096,"temperature":1.0,"scoring_batch_size":1,"trust_remote_code":false,"local_files_only":true,"use_safetensors":true,"guard_threshold":0.45,"seed":0,"tf32":false},"scoring_metadata":{"backend":"nli","protocol":"hf-local-sweep-backends-v1","probability_source":"adapted_entailment_softmax_not_calibrated_sixway","latency_scope":"synchronous_forward_scoring_including_transfer_excluding_tokenization","precision":"float32","equal_mass_convention":"rank bins with floor boundaries at floor(i*n/k), k=min(10,n//2); minimum two rows"},"load_ms":7889.282600022852,"peak_allocated_bytes":606415360,"peak_reserved_bytes":637534208,"panels":{"diagnostic43-no_samples":{"n":43,"raw":{"n":43,"acceptable":16,"unsafe":18,"review":0,"automated_count":43,"automated_acceptable":16,"automated_unsafe":18,"unnecessary_reviews":0,"accuracy":0.37209302325581395,"unsafe_rate":0.4186046511627907,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.37209302325581395,"conditional_unsafe":0.4186046511627907,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.24376339009650572,"upper":0.5214019084031153,"confidence_level":0.95}},"effective":{"n":43,"acceptable":5,"unsafe":0,"review":41,"automated_count":2,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":36,"accuracy":0.11627906976744186,"unsafe_rate":0.0,"review_rate":0.9534883720930233,"automated_coverage":0.046511627906976744,"conditional_correctness":0.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.8372093023255814,"accuracy_wilson_95":{"lower":0.05070427739804166,"upper":0.24479161810082128,"confidence_level":0.95}},"confusion":{"raw":{"keep":{"keep":6,"drop":6,"mask":0,"pseudonymize":0,"generalize":0,"review":0},"drop":{"keep":9,"drop":7,"mask":0,"pseudonymize":0,"generalize":0,"review":0},"mask":{"keep":0,"drop":2,"mask":0,"pseudonymize":0,"generalize":0,"review":0},"pseudonymize":{"keep":2,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0},"generalize":{"keep":5,"drop":1,"mask":0,"pseudonymize":0,"generalize":0,"review":0},"review":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0}},"effective":{"keep":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":12},"drop":{"keep":0,"drop":0,"mask":2,"pseudonymize":0,"generalize":0,"review":14},"mask":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":2},"pseudonymize":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":2},"generalize":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":6},"review":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0}}},"probability_metrics":{"singleton_n":38,"multiacceptable_n":5,"multiacceptable_coverage":0.11627906976744186,"nll":1.729216933484415,"nll_zero_mass_n":0,"brier":0.8124095888498535,"acceptable_set_n":43,"acceptable_set_nll":1.6051145101910955,"acceptable_set_zero_mass_n":0},"calibration_all_labels":{"equal_width":{"n":43,"bin_count":10,"ece":0.18416678870833197,"signed_mean_gap":-0.18416678870833197,"absolute_mean_gap":0.18416678870833197,"bins":[{"index":0,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.0,"upper":0.1,"upper_inclusive":false},{"index":1,"count":43,"confidence":0.18792623454748197,"empirical_correctness":0.37209302325581395,"signed_gap":-0.18416678870833197,"absolute_gap":0.18416678870833197,"lower":0.1,"upper":0.2,"upper_inclusive":false},{"index":2,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.2,"upper":0.3,"upper_inclusive":false},{"index":3,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.3,"upper":0.4,"upper_inclusive":false},{"index":4,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.4,"upper":0.5,"upper_inclusive":false},{"index":5,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.5,"upper":0.6,"upper_inclusive":false},{"index":6,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.6,"upper":0.7,"upper_inclusive":false},{"index":7,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.7,"upper":0.8,"upper_inclusive":false},{"index":8,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.8,"upper":0.9,"upper_inclusive":false},{"index":9,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.9,"upper":1.0,"upper_inclusive":true}]},"equal_mass":{"n":43,"bin_count":10,"ece":0.2540501837813577,"signed_mean_gap":-0.18416678870833197,"absolute_mean_gap":0.18416678870833197,"bins":[{"index":0,"count":4,"confidence":0.1825495958328247,"empirical_correctness":0.25,"signed_gap":-0.06745040416717529,"absolute_gap":0.06745040416717529,"min_confidence":0.18212656676769257,"max_confidence":0.18328911066055298},{"index":1,"count":4,"confidence":0.1842605285346508,"empirical_correctness":0.5,"signed_gap":-0.3157394714653492,"absolute_gap":0.3157394714653492,"min_confidence":0.18375158309936523,"max_confidence":0.1846325695514679},{"index":2,"count":4,"confidence":0.18531592935323715,"empirical_correctness":0.0,"signed_gap":0.18531592935323715,"absolute_gap":0.18531592935323715,"min_confidence":0.18471147119998932,"max_confidence":0.18598328530788422},{"index":3,"count":5,"confidence":0.186377814412117,"empirical_correctness":0.6,"signed_gap":-0.413622185587883,"absolute_gap":0.413622185587883,"min_confidence":0.18604958057403564,"max_confidence":0.18690939247608185},{"index":4,"count":4,"confidence":0.18727165088057518,"empirical_correctness":0.5,"signed_gap":-0.3127283491194248,"absolute_gap":0.3127283491194248,"min_confidence":0.18691971898078918,"max_confidence":0.1879684031009674},{"index":5,"count":4,"confidence":0.18886326253414154,"empirical_correctness":0.25,"signed_gap":-0.06113673746585846,"absolute_gap":0.06113673746585846,"min_confidence":0.18844963610172272,"max_confidence":0.18970753252506256},{"index":6,"count":5,"confidence":0.1899643212556839,"empirical_correctness":0.4,"signed_gap":-0.2100356787443161,"absolute_gap":0.2100356787443161,"min_confidence":0.1897585093975067,"max_confidence":0.19008156657218933},{"index":7,"count":4,"confidence":0.19030731916427612,"empirical_correctness":0.0,"signed_gap":0.19030731916427612,"absolute_gap":0.19030731916427612,"min_confidence":0.19014769792556763,"max_confidence":0.19049608707427979},{"index":8,"count":4,"confidence":0.19085120037198067,"empirical_correctness":0.5,"signed_gap":-0.30914879962801933,"absolute_gap":0.30914879962801933,"min_confidence":0.19056165218353271,"max_confidence":0.1911592185497284},{"index":9,"count":5,"confidence":0.1922878921031952,"empirical_correctness":0.6,"signed_gap":-0.4077121078968048,"absolute_gap":0.4077121078968048,"min_confidence":0.1913422793149948,"max_confidence":0.19300004839897156}]}},"calibration_singleton_labels":{"equal_width":{"n":38,"bin_count":10,"ece":0.15439143894534363,"signed_mean_gap":-0.15439143894534363,"absolute_mean_gap":0.15439143894534363,"bins":[{"index":0,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.0,"upper":0.1,"upper_inclusive":false},{"index":1,"count":38,"confidence":0.18771382421255112,"empirical_correctness":0.34210526315789475,"signed_gap":-0.15439143894534363,"absolute_gap":0.15439143894534363,"lower":0.1,"upper":0.2,"upper_inclusive":false},{"index":2,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.2,"upper":0.3,"upper_inclusive":false},{"index":3,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.3,"upper":0.4,"upper_inclusive":false},{"index":4,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.4,"upper":0.5,"upper_inclusive":false},{"index":5,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.5,"upper":0.6,"upper_inclusive":false},{"index":6,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.6,"upper":0.7,"upper_inclusive":false},{"index":7,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.7,"upper":0.8,"upper_inclusive":false},{"index":8,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.8,"upper":0.9,"upper_inclusive":false},{"index":9,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.9,"upper":1.0,"upper_inclusive":true}]},"equal_mass":{"n":38,"bin_count":10,"ece":0.19340531880918302,"signed_mean_gap":-0.15439143894534363,"absolute_mean_gap":0.15439143894534363,"bins":[{"index":0,"count":3,"confidence":0.18230309089024863,"empirical_correctness":0.3333333333333333,"signed_gap":-0.1510302424430847,"absolute_gap":0.1510302424430847,"min_confidence":0.18212656676769257,"max_confidence":0.18245555460453033},{"index":1,"count":4,"confidence":0.18414491042494774,"empirical_correctness":0.5,"signed_gap":-0.31585508957505226,"absolute_gap":0.31585508957505226,"min_confidence":0.18328911066055298,"max_confidence":0.1846325695514679},{"index":2,"count":4,"confidence":0.18531592935323715,"empirical_correctness":0.0,"signed_gap":0.18531592935323715,"absolute_gap":0.18531592935323715,"min_confidence":0.18471147119998932,"max_confidence":0.18598328530788422},{"index":3,"count":4,"confidence":0.1862449198961258,"empirical_correctness":0.75,"signed_gap":-0.5637550801038742,"absolute_gap":0.5637550801038742,"min_confidence":0.18604958057403564,"max_confidence":0.18639419972896576},{"index":4,"count":4,"confidence":0.1870068982243538,"empirical_correctness":0.25,"signed_gap":-0.06299310177564621,"absolute_gap":0.06299310177564621,"min_confidence":0.18690939247608185,"max_confidence":0.1871626079082489},{"index":5,"count":3,"confidence":0.18842142820358276,"empirical_correctness":0.3333333333333333,"signed_gap":-0.14491190512975055,"absolute_gap":0.14491190512975055,"min_confidence":0.1879684031009674,"max_confidence":0.18872970342636108},{"index":6,"count":4,"confidence":0.18984594196081161,"empirical_correctness":0.25,"signed_gap":-0.060154058039188385,"absolute_gap":0.060154058039188385,"min_confidence":0.18970753252506256,"max_confidence":0.1899988353252411},{"index":7,"count":4,"confidence":0.1901313029229641,"empirical_correctness":0.25,"signed_gap":-0.059868697077035904,"absolute_gap":0.059868697077035904,"min_confidence":0.19006380438804626,"max_confidence":0.19023214280605316},{"index":8,"count":4,"confidence":0.1905541568994522,"empirical_correctness":0.5,"signed_gap":-0.3094458431005478,"absolute_gap":0.3094458431005478,"min_confidence":0.19035334885120392,"max_confidence":0.19080553948879242},{"index":9,"count":4,"confidence":0.19199388101696968,"empirical_correctness":0.25,"signed_gap":-0.05800611898303032,"absolute_gap":0.05800611898303032,"min_confidence":0.19087839126586914,"max_confidence":0.19266609847545624}]}},"latency_ms":{"n":43,"median":105.21880001761019,"p95":112.2569799888879},"per_action":{"keep":{"n":12,"raw":{"n":12,"acceptable":6,"unsafe":0,"review":0,"automated_count":12,"automated_acceptable":6,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":0.5,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.5,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.2537815976337061,"upper":0.7462184023662939,"confidence_level":0.95}},"effective":{"n":12,"acceptable":0,"unsafe":0,"review":12,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":12,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":2.7755575615628914e-17,"upper":0.24249400665524085,"confidence_level":0.95}}},"drop":{"n":16,"raw":{"n":16,"acceptable":7,"unsafe":9,"review":0,"automated_count":16,"automated_acceptable":7,"automated_unsafe":9,"unnecessary_reviews":0,"accuracy":0.4375,"unsafe_rate":0.5625,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.4375,"conditional_unsafe":0.5625,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.23098652405492354,"upper":0.6682144360118811,"confidence_level":0.95}},"effective":{"n":16,"acceptable":0,"unsafe":0,"review":14,"automated_count":2,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":14,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":0.875,"automated_coverage":0.125,"conditional_correctness":0.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.875,"accuracy_wilson_95":{"lower":0.0,"upper":0.1936076805344365,"confidence_level":0.95}}},"mask":{"n":2,"raw":{"n":2,"acceptable":0,"unsafe":0,"review":0,"automated_count":2,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.6576197724933469,"confidence_level":0.95}},"effective":{"n":2,"acceptable":0,"unsafe":0,"review":2,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":2,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.6576197724933469,"confidence_level":0.95}}},"pseudonymize":{"n":2,"raw":{"n":2,"acceptable":0,"unsafe":2,"review":0,"automated_count":2,"automated_acceptable":0,"automated_unsafe":2,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":1.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":1.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.6576197724933469,"confidence_level":0.95}},"effective":{"n":2,"acceptable":0,"unsafe":0,"review":2,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":2,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.6576197724933469,"confidence_level":0.95}}},"generalize":{"n":6,"raw":{"n":6,"acceptable":0,"unsafe":5,"review":0,"automated_count":6,"automated_acceptable":0,"automated_unsafe":5,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":0.8333333333333334,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":0.8333333333333334,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":2.7755575615628914e-17,"upper":0.3903342879021653,"confidence_level":0.95}},"effective":{"n":6,"acceptable":0,"unsafe":0,"review":6,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":6,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":2.7755575615628914e-17,"upper":0.3903342879021653,"confidence_level":0.95}}},"review":{"n":0,"raw":{"n":0,"acceptable":0,"unsafe":0,"review":0,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":null,"unsafe_rate":null,"review_rate":null,"automated_coverage":null,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":null},"effective":{"n":0,"acceptable":0,"unsafe":0,"review":0,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":null,"unsafe_rate":null,"review_rate":null,"automated_coverage":null,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":null}}}},"diagnostic43-compact_no_samples":{"n":43,"raw":{"n":43,"acceptable":19,"unsafe":6,"review":0,"automated_count":43,"automated_acceptable":19,"automated_unsafe":6,"unnecessary_reviews":0,"accuracy":0.4418604651162791,"unsafe_rate":0.13953488372093023,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.4418604651162791,"conditional_unsafe":0.13953488372093023,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.30433309839170514,"upper":0.5889238554717591,"confidence_level":0.95}},"effective":{"n":43,"acceptable":5,"unsafe":0,"review":43,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":38,"accuracy":0.11627906976744186,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":0.8837209302325582,"accuracy_wilson_95":{"lower":0.05070427739804166,"upper":0.24479161810082128,"confidence_level":0.95}},"confusion":{"raw":{"keep":{"keep":1,"drop":11,"mask":0,"pseudonymize":0,"generalize":0,"review":0},"drop":{"keep":2,"drop":14,"mask":0,"pseudonymize":0,"generalize":0,"review":0},"mask":{"keep":0,"drop":2,"mask":0,"pseudonymize":0,"generalize":0,"review":0},"pseudonymize":{"keep":1,"drop":1,"mask":0,"pseudonymize":0,"generalize":0,"review":0},"generalize":{"keep":2,"drop":4,"mask":0,"pseudonymize":0,"generalize":0,"review":0},"review":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0}},"effective":{"keep":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":12},"drop":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":16},"mask":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":2},"pseudonymize":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":2},"generalize":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":6},"review":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0}}},"probability_metrics":{"singleton_n":38,"multiacceptable_n":5,"multiacceptable_coverage":0.11627906976744186,"nll":1.7065746536444495,"nll_zero_mass_n":0,"brier":0.8046130731880107,"acceptable_set_n":43,"acceptable_set_nll":1.5859760011036734,"acceptable_set_zero_mass_n":0},"calibration_all_labels":{"equal_width":{"n":43,"bin_count":10,"ece":0.24838257217129991,"signed_mean_gap":-0.24838257217129991,"absolute_mean_gap":0.24838257217129991,"bins":[{"index":0,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.0,"upper":0.1,"upper_inclusive":false},{"index":1,"count":39,"confidence":0.19273347808764532,"empirical_correctness":0.41025641025641024,"signed_gap":-0.21752293216876492,"absolute_gap":0.21752293216876492,"lower":0.1,"upper":0.2,"upper_inclusive":false},{"index":2,"count":4,"confidence":0.2007359378039837,"empirical_correctness":0.75,"signed_gap":-0.5492640621960163,"absolute_gap":0.5492640621960163,"lower":0.2,"upper":0.3,"upper_inclusive":false},{"index":3,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.3,"upper":0.4,"upper_inclusive":false},{"index":4,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.4,"upper":0.5,"upper_inclusive":false},{"index":5,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.5,"upper":0.6,"upper_inclusive":false},{"index":6,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.6,"upper":0.7,"upper_inclusive":false},{"index":7,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.7,"upper":0.8,"upper_inclusive":false},{"index":8,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.8,"upper":0.9,"upper_inclusive":false},{"index":9,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.9,"upper":1.0,"upper_inclusive":true}]},"equal_mass":{"n":43,"bin_count":10,"ece":0.24838257217129997,"signed_mean_gap":-0.24838257217129997,"absolute_mean_gap":0.24838257217129997,"bins":[{"index":0,"count":4,"confidence":0.1888195164501667,"empirical_correctness":0.25,"signed_gap":-0.0611804835498333,"absolute_gap":0.0611804835498333,"min_confidence":0.18839484453201294,"max_confidence":0.18925771117210388},{"index":1,"count":4,"confidence":0.19016795977950096,"empirical_correctness":0.5,"signed_gap":-0.30983204022049904,"absolute_gap":0.30983204022049904,"min_confidence":0.18986298143863678,"max_confidence":0.19037216901779175},{"index":2,"count":4,"confidence":0.1904766708612442,"empirical_correctness":0.5,"signed_gap":-0.3095233291387558,"absolute_gap":0.3095233291387558,"min_confidence":0.19040019810199738,"max_confidence":0.1906622052192688},{"index":3,"count":5,"confidence":0.19089268445968627,"empirical_correctness":0.2,"signed_gap":-0.009107315540313743,"absolute_gap":0.009107315540313743,"min_confidence":0.19067901372909546,"max_confidence":0.19106066226959229},{"index":4,"count":4,"confidence":0.19174019247293472,"empirical_correctness":0.75,"signed_gap":-0.5582598075270653,"absolute_gap":0.5582598075270653,"min_confidence":0.19128334522247314,"max_confidence":0.19222433865070343},{"index":5,"count":4,"confidence":0.19306685030460358,"empirical_correctness":0.25,"signed_gap":-0.05693314969539642,"absolute_gap":0.05693314969539642,"min_confidence":0.1925159990787506,"max_confidence":0.19343771040439606},{"index":6,"count":5,"confidence":0.19429971873760224,"empirical_correctness":0.4,"signed_gap":-0.20570028126239778,"absolute_gap":0.20570028126239778,"min_confidence":0.19368413090705872,"max_confidence":0.19491226971149445},{"index":7,"count":4,"confidence":0.19540366530418396,"empirical_correctness":0.75,"signed_gap":-0.554596334695816,"absolute_gap":0.554596334695816,"min_confidence":0.1950664520263672,"max_confidence":0.1956719309091568},{"index":8,"count":4,"confidence":0.1980830281972885,"empirical_correctness":0.25,"signed_gap":-0.05191697180271149,"absolute_gap":0.05191697180271149,"min_confidence":0.19680967926979065,"max_confidence":0.19925548136234283},{"index":9,"count":5,"confidence":0.20051116943359376,"empirical_correctness":0.6,"signed_gap":-0.3994888305664062,"absolute_gap":0.3994888305664062,"min_confidence":0.199612095952034,"max_confidence":0.20133748650550842}]}},"calibration_singleton_labels":{"equal_width":{"n":38,"bin_count":10,"ece":0.20142482417194466,"signed_mean_gap":-0.20142482417194466,"absolute_mean_gap":0.20142482417194466,"bins":[{"index":0,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.0,"upper":0.1,"upper_inclusive":false},{"index":1,"count":35,"confidence":0.19267121085098812,"empirical_correctness":0.37142857142857144,"signed_gap":-0.17875736057758332,"absolute_gap":0.17875736057758332,"lower":0.1,"upper":0.2,"upper_inclusive":false},{"index":2,"count":3,"confidence":0.2007881005605062,"empirical_correctness":0.6666666666666666,"signed_gap":-0.4658785661061604,"absolute_gap":0.4658785661061604,"lower":0.2,"upper":0.3,"upper_inclusive":false},{"index":3,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.3,"upper":0.4,"upper_inclusive":false},{"index":4,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.4,"upper":0.5,"upper_inclusive":false},{"index":5,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.5,"upper":0.6,"upper_inclusive":false},{"index":6,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.6,"upper":0.7,"upper_inclusive":false},{"index":7,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.7,"upper":0.8,"upper_inclusive":false},{"index":8,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.8,"upper":0.9,"upper_inclusive":false},{"index":9,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.9,"upper":1.0,"upper_inclusive":true}]},"equal_mass":{"n":38,"bin_count":10,"ece":0.242253929768738,"signed_mean_gap":-0.20142482417194466,"absolute_mean_gap":0.20142482417194466,"bins":[{"index":0,"count":3,"confidence":0.1886734515428543,"empirical_correctness":0.3333333333333333,"signed_gap":-0.144659881790479,"absolute_gap":0.144659881790479,"min_confidence":0.18839484453201294,"max_confidence":0.18902035057544708},{"index":1,"count":4,"confidence":0.189889345318079,"empirical_correctness":0.25,"signed_gap":-0.060110654681921005,"absolute_gap":0.060110654681921005,"min_confidence":0.18925771117210388,"max_confidence":0.1902993768453598},{"index":2,"count":4,"confidence":0.19040416181087494,"empirical_correctness":0.75,"signed_gap":-0.5595958381891251,"absolute_gap":0.5595958381891251,"min_confidence":0.19037216901779175,"max_confidence":0.19043318927288055},{"index":3,"count":4,"confidence":0.19077974930405617,"empirical_correctness":0.25,"signed_gap":-0.05922025069594383,"absolute_gap":0.05922025069594383,"min_confidence":0.1906622052192688,"max_confidence":0.19097290933132172},{"index":4,"count":4,"confidence":0.19144927337765694,"empirical_correctness":0.5,"signed_gap":-0.30855072662234306,"absolute_gap":0.30855072662234306,"min_confidence":0.19106066226959229,"max_confidence":0.19185492396354675},{"index":5,"count":3,"confidence":0.19294323027133942,"empirical_correctness":0.3333333333333333,"signed_gap":-0.1403901030619939,"absolute_gap":0.1403901030619939,"min_confidence":0.1925159990787506,"max_confidence":0.19324377179145813},{"index":6,"count":4,"confidence":0.1939382515847683,"empirical_correctness":0.0,"signed_gap":0.1939382515847683,"absolute_gap":0.1939382515847683,"min_confidence":0.19343771040439606,"max_confidence":0.19439077377319336},{"index":7,"count":4,"confidence":0.19521375000476837,"empirical_correctness":0.75,"signed_gap":-0.5547862499952316,"absolute_gap":0.5547862499952316,"min_confidence":0.19491226971149445,"max_confidence":0.19555050134658813},{"index":8,"count":4,"confidence":0.1980830281972885,"empirical_correctness":0.25,"signed_gap":-0.05191697180271149,"absolute_gap":0.05191697180271149,"min_confidence":0.19680967926979065,"max_confidence":0.19925548136234283},{"index":9,"count":4,"confidence":0.20049409940838814,"empirical_correctness":0.5,"signed_gap":-0.29950590059161186,"absolute_gap":0.29950590059161186,"min_confidence":0.199612095952034,"max_confidence":0.20133748650550842}]}},"latency_ms":{"n":43,"median":111.88800004310906,"p95":117.61335986666381},"per_action":{"keep":{"n":12,"raw":{"n":12,"acceptable":1,"unsafe":0,"review":0,"automated_count":12,"automated_acceptable":1,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":0.08333333333333333,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.08333333333333333,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.014865094404917095,"upper":0.35387991114111694,"confidence_level":0.95}},"effective":{"n":12,"acceptable":0,"unsafe":0,"review":12,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":12,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":2.7755575615628914e-17,"upper":0.24249400665524085,"confidence_level":0.95}}},"drop":{"n":16,"raw":{"n":16,"acceptable":14,"unsafe":2,"review":0,"automated_count":16,"automated_acceptable":14,"automated_unsafe":2,"unnecessary_reviews":0,"accuracy":0.875,"unsafe_rate":0.125,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.875,"conditional_unsafe":0.125,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.639771727342413,"upper":0.9650225122567595,"confidence_level":0.95}},"effective":{"n":16,"acceptable":0,"unsafe":0,"review":16,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":16,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.1936076805344365,"confidence_level":0.95}}},"mask":{"n":2,"raw":{"n":2,"acceptable":0,"unsafe":0,"review":0,"automated_count":2,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.6576197724933469,"confidence_level":0.95}},"effective":{"n":2,"acceptable":0,"unsafe":0,"review":2,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":2,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.6576197724933469,"confidence_level":0.95}}},"pseudonymize":{"n":2,"raw":{"n":2,"acceptable":0,"unsafe":1,"review":0,"automated_count":2,"automated_acceptable":0,"automated_unsafe":1,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":0.5,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":0.5,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.6576197724933469,"confidence_level":0.95}},"effective":{"n":2,"acceptable":0,"unsafe":0,"review":2,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":2,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.6576197724933469,"confidence_level":0.95}}},"generalize":{"n":6,"raw":{"n":6,"acceptable":0,"unsafe":2,"review":0,"automated_count":6,"automated_acceptable":0,"automated_unsafe":2,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":0.3333333333333333,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":0.3333333333333333,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":2.7755575615628914e-17,"upper":0.3903342879021653,"confidence_level":0.95}},"effective":{"n":6,"acceptable":0,"unsafe":0,"review":6,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":6,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":2.7755575615628914e-17,"upper":0.3903342879021653,"confidence_level":0.95}}},"review":{"n":0,"raw":{"n":0,"acceptable":0,"unsafe":0,"review":0,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":null,"unsafe_rate":null,"review_rate":null,"automated_coverage":null,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":null},"effective":{"n":0,"acceptable":0,"unsafe":0,"review":0,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":null,"unsafe_rate":null,"review_rate":null,"automated_coverage":null,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":null}}}},"challenge56-no_samples":{"n":56,"raw":{"n":56,"acceptable":10,"unsafe":41,"review":0,"automated_count":56,"automated_acceptable":10,"automated_unsafe":41,"unnecessary_reviews":0,"accuracy":0.17857142857142858,"unsafe_rate":0.7321428571428571,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.17857142857142858,"conditional_unsafe":0.7321428571428571,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.09999856752412305,"upper":0.2984118203665106,"confidence_level":0.95}},"effective":{"n":56,"acceptable":2,"unsafe":0,"review":56,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":54,"accuracy":0.03571428571428571,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":0.9642857142857143,"accuracy_wilson_95":{"lower":0.00984942514823639,"upper":0.12118780180490107,"confidence_level":0.95}},"confusion":{"raw":{"keep":{"keep":8,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0},"drop":{"keep":20,"drop":2,"mask":0,"pseudonymize":0,"generalize":0,"review":0},"mask":{"keep":7,"drop":1,"mask":0,"pseudonymize":0,"generalize":0,"review":0},"pseudonymize":{"keep":6,"drop":2,"mask":0,"pseudonymize":0,"generalize":0,"review":0},"generalize":{"keep":7,"drop":1,"mask":0,"pseudonymize":0,"generalize":0,"review":0},"review":{"keep":1,"drop":1,"mask":0,"pseudonymize":0,"generalize":0,"review":0}},"effective":{"keep":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":8},"drop":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":22},"mask":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":8},"pseudonymize":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":8},"generalize":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":8},"review":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":2}}},"probability_metrics":{"singleton_n":56,"multiacceptable_n":0,"multiacceptable_coverage":0.0,"nll":1.7766542896266517,"nll_zero_mass_n":0,"brier":0.8284161052853486,"acceptable_set_n":56,"acceptable_set_nll":1.7766542896266517,"acceptable_set_zero_mass_n":0},"calibration_all_labels":{"equal_width":{"n":56,"bin_count":10,"ece":0.01068996904151781,"signed_mean_gap":0.01068996904151781,"absolute_mean_gap":0.01068996904151781,"bins":[{"index":0,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.0,"upper":0.1,"upper_inclusive":false},{"index":1,"count":56,"confidence":0.18926139761294639,"empirical_correctness":0.17857142857142858,"signed_gap":0.01068996904151781,"absolute_gap":0.01068996904151781,"lower":0.1,"upper":0.2,"upper_inclusive":false},{"index":2,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.2,"upper":0.3,"upper_inclusive":false},{"index":3,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.3,"upper":0.4,"upper_inclusive":false},{"index":4,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.4,"upper":0.5,"upper_inclusive":false},{"index":5,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.5,"upper":0.6,"upper_inclusive":false},{"index":6,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.6,"upper":0.7,"upper_inclusive":false},{"index":7,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.7,"upper":0.8,"upper_inclusive":false},{"index":8,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.8,"upper":0.9,"upper_inclusive":false},{"index":9,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.9,"upper":1.0,"upper_inclusive":true}]},"equal_mass":{"n":56,"bin_count":10,"ece":0.08088749966451102,"signed_mean_gap":0.010689969041517805,"absolute_mean_gap":0.010689969041517805,"bins":[{"index":0,"count":5,"confidence":0.18294871151447295,"empirical_correctness":0.2,"signed_gap":-0.01705128848552706,"absolute_gap":0.01705128848552706,"min_confidence":0.1812708079814911,"max_confidence":0.18411694467067719},{"index":1,"count":6,"confidence":0.1854532485206922,"empirical_correctness":0.16666666666666666,"signed_gap":0.01878658185402554,"absolute_gap":0.01878658185402554,"min_confidence":0.1847907304763794,"max_confidence":0.18606196343898773},{"index":2,"count":5,"confidence":0.1870100736618042,"empirical_correctness":0.0,"signed_gap":0.1870100736618042,"absolute_gap":0.1870100736618042,"min_confidence":0.1861424595117569,"max_confidence":0.18751510977745056},{"index":3,"count":6,"confidence":0.1879103680451711,"empirical_correctness":0.16666666666666666,"signed_gap":0.021243701378504454,"absolute_gap":0.021243701378504454,"min_confidence":0.18753482401371002,"max_confidence":0.18815281987190247},{"index":4,"count":6,"confidence":0.1888530751069387,"empirical_correctness":0.16666666666666666,"signed_gap":0.022186408440272032,"absolute_gap":0.022186408440272032,"min_confidence":0.18827104568481445,"max_confidence":0.18939237296581268},{"index":5,"count":5,"confidence":0.18975868225097656,"empirical_correctness":0.2,"signed_gap":-0.010241317749023454,"absolute_gap":0.010241317749023454,"min_confidence":0.18948085606098175,"max_confidence":0.19011802971363068},{"index":6,"count":6,"confidence":0.19040424128373465,"empirical_correctness":0.16666666666666666,"signed_gap":0.02373757461706799,"absolute_gap":0.02373757461706799,"min_confidence":0.19023188948631287,"max_confidence":0.19081750512123108},{"index":7,"count":5,"confidence":0.19145603179931642,"empirical_correctness":0.0,"signed_gap":0.19145603179931642,"absolute_gap":0.19145603179931642,"min_confidence":0.1911427527666092,"max_confidence":0.1920682042837143},{"index":8,"count":6,"confidence":0.19268549978733063,"empirical_correctness":0.16666666666666666,"signed_gap":0.02601883312066397,"absolute_gap":0.02601883312066397,"min_confidence":0.1921851485967636,"max_confidence":0.1930297166109085},{"index":9,"count":6,"confidence":0.19515536228815714,"empirical_correctness":0.5,"signed_gap":-0.3048446377118429,"absolute_gap":0.3048446377118429,"min_confidence":0.19349321722984314,"max_confidence":0.19660845398902893}]}},"calibration_singleton_labels":{"equal_width":{"n":56,"bin_count":10,"ece":0.01068996904151781,"signed_mean_gap":0.01068996904151781,"absolute_mean_gap":0.01068996904151781,"bins":[{"index":0,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.0,"upper":0.1,"upper_inclusive":false},{"index":1,"count":56,"confidence":0.18926139761294639,"empirical_correctness":0.17857142857142858,"signed_gap":0.01068996904151781,"absolute_gap":0.01068996904151781,"lower":0.1,"upper":0.2,"upper_inclusive":false},{"index":2,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.2,"upper":0.3,"upper_inclusive":false},{"index":3,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.3,"upper":0.4,"upper_inclusive":false},{"index":4,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.4,"upper":0.5,"upper_inclusive":false},{"index":5,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.5,"upper":0.6,"upper_inclusive":false},{"index":6,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.6,"upper":0.7,"upper_inclusive":false},{"index":7,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.7,"upper":0.8,"upper_inclusive":false},{"index":8,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.8,"upper":0.9,"upper_inclusive":false},{"index":9,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.9,"upper":1.0,"upper_inclusive":true}]},"equal_mass":{"n":56,"bin_count":10,"ece":0.08088749966451102,"signed_mean_gap":0.010689969041517805,"absolute_mean_gap":0.010689969041517805,"bins":[{"index":0,"count":5,"confidence":0.18294871151447295,"empirical_correctness":0.2,"signed_gap":-0.01705128848552706,"absolute_gap":0.01705128848552706,"min_confidence":0.1812708079814911,"max_confidence":0.18411694467067719},{"index":1,"count":6,"confidence":0.1854532485206922,"empirical_correctness":0.16666666666666666,"signed_gap":0.01878658185402554,"absolute_gap":0.01878658185402554,"min_confidence":0.1847907304763794,"max_confidence":0.18606196343898773},{"index":2,"count":5,"confidence":0.1870100736618042,"empirical_correctness":0.0,"signed_gap":0.1870100736618042,"absolute_gap":0.1870100736618042,"min_confidence":0.1861424595117569,"max_confidence":0.18751510977745056},{"index":3,"count":6,"confidence":0.1879103680451711,"empirical_correctness":0.16666666666666666,"signed_gap":0.021243701378504454,"absolute_gap":0.021243701378504454,"min_confidence":0.18753482401371002,"max_confidence":0.18815281987190247},{"index":4,"count":6,"confidence":0.1888530751069387,"empirical_correctness":0.16666666666666666,"signed_gap":0.022186408440272032,"absolute_gap":0.022186408440272032,"min_confidence":0.18827104568481445,"max_confidence":0.18939237296581268},{"index":5,"count":5,"confidence":0.18975868225097656,"empirical_correctness":0.2,"signed_gap":-0.010241317749023454,"absolute_gap":0.010241317749023454,"min_confidence":0.18948085606098175,"max_confidence":0.19011802971363068},{"index":6,"count":6,"confidence":0.19040424128373465,"empirical_correctness":0.16666666666666666,"signed_gap":0.02373757461706799,"absolute_gap":0.02373757461706799,"min_confidence":0.19023188948631287,"max_confidence":0.19081750512123108},{"index":7,"count":5,"confidence":0.19145603179931642,"empirical_correctness":0.0,"signed_gap":0.19145603179931642,"absolute_gap":0.19145603179931642,"min_confidence":0.1911427527666092,"max_confidence":0.1920682042837143},{"index":8,"count":6,"confidence":0.19268549978733063,"empirical_correctness":0.16666666666666666,"signed_gap":0.02601883312066397,"absolute_gap":0.02601883312066397,"min_confidence":0.1921851485967636,"max_confidence":0.1930297166109085},{"index":9,"count":6,"confidence":0.19515536228815714,"empirical_correctness":0.5,"signed_gap":-0.3048446377118429,"absolute_gap":0.3048446377118429,"min_confidence":0.19349321722984314,"max_confidence":0.19660845398902893}]}},"latency_ms":{"n":56,"median":95.01645003911108,"p95":111.83902493212372},"per_action":{"keep":{"n":8,"raw":{"n":8,"acceptable":8,"unsafe":0,"review":0,"automated_count":8,"automated_acceptable":8,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":1.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.6755924351161198,"upper":1.0,"confidence_level":0.95}},"effective":{"n":8,"acceptable":0,"unsafe":0,"review":8,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":8,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.32440756488388023,"confidence_level":0.95}}},"drop":{"n":22,"raw":{"n":22,"acceptable":2,"unsafe":20,"review":0,"automated_count":22,"automated_acceptable":2,"automated_unsafe":20,"unnecessary_reviews":0,"accuracy":0.09090909090909091,"unsafe_rate":0.9090909090909091,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.09090909090909091,"conditional_unsafe":0.9090909090909091,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.025295428990995267,"upper":0.27814946841605814,"confidence_level":0.95}},"effective":{"n":22,"acceptable":0,"unsafe":0,"review":22,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":22,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.14865487460862087,"confidence_level":0.95}}},"mask":{"n":8,"raw":{"n":8,"acceptable":0,"unsafe":7,"review":0,"automated_count":8,"automated_acceptable":0,"automated_unsafe":7,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":0.875,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":0.875,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.32440756488388023,"confidence_level":0.95}},"effective":{"n":8,"acceptable":0,"unsafe":0,"review":8,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":8,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.32440756488388023,"confidence_level":0.95}}},"pseudonymize":{"n":8,"raw":{"n":8,"acceptable":0,"unsafe":6,"review":0,"automated_count":8,"automated_acceptable":0,"automated_unsafe":6,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":0.75,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":0.75,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.32440756488388023,"confidence_level":0.95}},"effective":{"n":8,"acceptable":0,"unsafe":0,"review":8,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":8,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.32440756488388023,"confidence_level":0.95}}},"generalize":{"n":8,"raw":{"n":8,"acceptable":0,"unsafe":7,"review":0,"automated_count":8,"automated_acceptable":0,"automated_unsafe":7,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":0.875,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":0.875,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.32440756488388023,"confidence_level":0.95}},"effective":{"n":8,"acceptable":0,"unsafe":0,"review":8,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":8,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.32440756488388023,"confidence_level":0.95}}},"review":{"n":2,"raw":{"n":2,"acceptable":0,"unsafe":1,"review":0,"automated_count":2,"automated_acceptable":0,"automated_unsafe":1,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":0.5,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":0.5,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.6576197724933469,"confidence_level":0.95}},"effective":{"n":2,"acceptable":2,"unsafe":0,"review":2,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":1.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.34238022750665303,"upper":1.0,"confidence_level":0.95}}}}},"challenge56-compact_no_samples":{"n":56,"raw":{"n":56,"acceptable":8,"unsafe":34,"review":0,"automated_count":56,"automated_acceptable":8,"automated_unsafe":34,"unnecessary_reviews":0,"accuracy":0.14285714285714285,"unsafe_rate":0.6071428571428571,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.14285714285714285,"conditional_unsafe":0.6071428571428571,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.07420788536920624,"upper":0.25735921228705333,"confidence_level":0.95}},"effective":{"n":56,"acceptable":2,"unsafe":0,"review":56,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":54,"accuracy":0.03571428571428571,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":0.9642857142857143,"accuracy_wilson_95":{"lower":0.00984942514823639,"upper":0.12118780180490107,"confidence_level":0.95}},"confusion":{"raw":{"keep":{"keep":6,"drop":2,"mask":0,"pseudonymize":0,"generalize":0,"review":0},"drop":{"keep":20,"drop":2,"mask":0,"pseudonymize":0,"generalize":0,"review":0},"mask":{"keep":3,"drop":5,"mask":0,"pseudonymize":0,"generalize":0,"review":0},"pseudonymize":{"keep":5,"drop":3,"mask":0,"pseudonymize":0,"generalize":0,"review":0},"generalize":{"keep":5,"drop":3,"mask":0,"pseudonymize":0,"generalize":0,"review":0},"review":{"keep":1,"drop":1,"mask":0,"pseudonymize":0,"generalize":0,"review":0}},"effective":{"keep":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":8},"drop":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":22},"mask":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":8},"pseudonymize":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":8},"generalize":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":8},"review":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":2}}},"probability_metrics":{"singleton_n":56,"multiacceptable_n":0,"multiacceptable_coverage":0.0,"nll":1.77409859097147,"nll_zero_mass_n":0,"brier":0.8274457288357013,"acceptable_set_n":56,"acceptable_set_nll":1.77409859097147,"acceptable_set_zero_mass_n":0},"calibration_all_labels":{"equal_width":{"n":56,"bin_count":10,"ece":0.04241227252142771,"signed_mean_gap":0.04241227252142771,"absolute_mean_gap":0.04241227252142771,"bins":[{"index":0,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.0,"upper":0.1,"upper_inclusive":false},{"index":1,"count":56,"confidence":0.18526941537857056,"empirical_correctness":0.14285714285714285,"signed_gap":0.04241227252142771,"absolute_gap":0.04241227252142771,"lower":0.1,"upper":0.2,"upper_inclusive":false},{"index":2,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.2,"upper":0.3,"upper_inclusive":false},{"index":3,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.3,"upper":0.4,"upper_inclusive":false},{"index":4,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.4,"upper":0.5,"upper_inclusive":false},{"index":5,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.5,"upper":0.6,"upper_inclusive":false},{"index":6,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.6,"upper":0.7,"upper_inclusive":false},{"index":7,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.7,"upper":0.8,"upper_inclusive":false},{"index":8,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.8,"upper":0.9,"upper_inclusive":false},{"index":9,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.9,"upper":1.0,"upper_inclusive":true}]},"equal_mass":{"n":56,"bin_count":10,"ece":0.10818623432091304,"signed_mean_gap":0.0424122725214277,"absolute_mean_gap":0.0424122725214277,"bins":[{"index":0,"count":5,"confidence":0.17778631448745727,"empirical_correctness":0.0,"signed_gap":0.17778631448745727,"absolute_gap":0.17778631448745727,"min_confidence":0.17646755278110504,"max_confidence":0.17877651751041412},{"index":1,"count":6,"confidence":0.18109470357497534,"empirical_correctness":0.0,"signed_gap":0.18109470357497534,"absolute_gap":0.18109470357497534,"min_confidence":0.17934009432792664,"max_confidence":0.18219800293445587},{"index":2,"count":5,"confidence":0.18263038098812104,"empirical_correctness":0.0,"signed_gap":0.18263038098812104,"absolute_gap":0.18263038098812104,"min_confidence":0.182239830493927,"max_confidence":0.18313615024089813},{"index":3,"count":6,"confidence":0.18367589513460794,"empirical_correctness":0.3333333333333333,"signed_gap":-0.14965743819872537,"absolute_gap":0.14965743819872537,"min_confidence":0.1831454634666443,"max_confidence":0.18416938185691833},{"index":4,"count":6,"confidence":0.18458600590626398,"empirical_correctness":0.16666666666666666,"signed_gap":0.01791933923959732,"absolute_gap":0.01791933923959732,"min_confidence":0.18421103060245514,"max_confidence":0.18493293225765228},{"index":5,"count":5,"confidence":0.1857507884502411,"empirical_correctness":0.0,"signed_gap":0.1857507884502411,"absolute_gap":0.1857507884502411,"min_confidence":0.18542958796024323,"max_confidence":0.18599063158035278},{"index":6,"count":6,"confidence":0.18640616039435068,"empirical_correctness":0.3333333333333333,"signed_gap":-0.14692717293898264,"absolute_gap":0.14692717293898264,"min_confidence":0.18603326380252838,"max_confidence":0.18682920932769775},{"index":7,"count":5,"confidence":0.1875673472881317,"empirical_correctness":0.2,"signed_gap":-0.012432652711868303,"absolute_gap":0.012432652711868303,"min_confidence":0.18688198924064636,"max_confidence":0.1879919022321701},{"index":8,"count":6,"confidence":0.1891660119096438,"empirical_correctness":0.16666666666666666,"signed_gap":0.022499345242977142,"absolute_gap":0.022499345242977142,"min_confidence":0.18844002485275269,"max_confidence":0.1897200495004654},{"index":9,"count":6,"confidence":0.1928067406018575,"empirical_correctness":0.16666666666666666,"signed_gap":0.026140073935190855,"absolute_gap":0.026140073935190855,"min_confidence":0.19011597335338593,"max_confidence":0.19605156779289246}]}},"calibration_singleton_labels":{"equal_width":{"n":56,"bin_count":10,"ece":0.04241227252142771,"signed_mean_gap":0.04241227252142771,"absolute_mean_gap":0.04241227252142771,"bins":[{"index":0,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.0,"upper":0.1,"upper_inclusive":false},{"index":1,"count":56,"confidence":0.18526941537857056,"empirical_correctness":0.14285714285714285,"signed_gap":0.04241227252142771,"absolute_gap":0.04241227252142771,"lower":0.1,"upper":0.2,"upper_inclusive":false},{"index":2,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.2,"upper":0.3,"upper_inclusive":false},{"index":3,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.3,"upper":0.4,"upper_inclusive":false},{"index":4,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.4,"upper":0.5,"upper_inclusive":false},{"index":5,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.5,"upper":0.6,"upper_inclusive":false},{"index":6,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.6,"upper":0.7,"upper_inclusive":false},{"index":7,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.7,"upper":0.8,"upper_inclusive":false},{"index":8,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.8,"upper":0.9,"upper_inclusive":false},{"index":9,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.9,"upper":1.0,"upper_inclusive":true}]},"equal_mass":{"n":56,"bin_count":10,"ece":0.10818623432091304,"signed_mean_gap":0.0424122725214277,"absolute_mean_gap":0.0424122725214277,"bins":[{"index":0,"count":5,"confidence":0.17778631448745727,"empirical_correctness":0.0,"signed_gap":0.17778631448745727,"absolute_gap":0.17778631448745727,"min_confidence":0.17646755278110504,"max_confidence":0.17877651751041412},{"index":1,"count":6,"confidence":0.18109470357497534,"empirical_correctness":0.0,"signed_gap":0.18109470357497534,"absolute_gap":0.18109470357497534,"min_confidence":0.17934009432792664,"max_confidence":0.18219800293445587},{"index":2,"count":5,"confidence":0.18263038098812104,"empirical_correctness":0.0,"signed_gap":0.18263038098812104,"absolute_gap":0.18263038098812104,"min_confidence":0.182239830493927,"max_confidence":0.18313615024089813},{"index":3,"count":6,"confidence":0.18367589513460794,"empirical_correctness":0.3333333333333333,"signed_gap":-0.14965743819872537,"absolute_gap":0.14965743819872537,"min_confidence":0.1831454634666443,"max_confidence":0.18416938185691833},{"index":4,"count":6,"confidence":0.18458600590626398,"empirical_correctness":0.16666666666666666,"signed_gap":0.01791933923959732,"absolute_gap":0.01791933923959732,"min_confidence":0.18421103060245514,"max_confidence":0.18493293225765228},{"index":5,"count":5,"confidence":0.1857507884502411,"empirical_correctness":0.0,"signed_gap":0.1857507884502411,"absolute_gap":0.1857507884502411,"min_confidence":0.18542958796024323,"max_confidence":0.18599063158035278},{"index":6,"count":6,"confidence":0.18640616039435068,"empirical_correctness":0.3333333333333333,"signed_gap":-0.14692717293898264,"absolute_gap":0.14692717293898264,"min_confidence":0.18603326380252838,"max_confidence":0.18682920932769775},{"index":7,"count":5,"confidence":0.1875673472881317,"empirical_correctness":0.2,"signed_gap":-0.012432652711868303,"absolute_gap":0.012432652711868303,"min_confidence":0.18688198924064636,"max_confidence":0.1879919022321701},{"index":8,"count":6,"confidence":0.1891660119096438,"empirical_correctness":0.16666666666666666,"signed_gap":0.022499345242977142,"absolute_gap":0.022499345242977142,"min_confidence":0.18844002485275269,"max_confidence":0.1897200495004654},{"index":9,"count":6,"confidence":0.1928067406018575,"empirical_correctness":0.16666666666666666,"signed_gap":0.026140073935190855,"absolute_gap":0.026140073935190855,"min_confidence":0.19011597335338593,"max_confidence":0.19605156779289246}]}},"latency_ms":{"n":56,"median":94.43924995139241,"p95":111.0179249662906},"per_action":{"keep":{"n":8,"raw":{"n":8,"acceptable":6,"unsafe":0,"review":0,"automated_count":8,"automated_acceptable":6,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":0.75,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.75,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.40927543031016883,"upper":0.9285207872478909,"confidence_level":0.95}},"effective":{"n":8,"acceptable":0,"unsafe":0,"review":8,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":8,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.32440756488388023,"confidence_level":0.95}}},"drop":{"n":22,"raw":{"n":22,"acceptable":2,"unsafe":20,"review":0,"automated_count":22,"automated_acceptable":2,"automated_unsafe":20,"unnecessary_reviews":0,"accuracy":0.09090909090909091,"unsafe_rate":0.9090909090909091,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.09090909090909091,"conditional_unsafe":0.9090909090909091,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.025295428990995267,"upper":0.27814946841605814,"confidence_level":0.95}},"effective":{"n":22,"acceptable":0,"unsafe":0,"review":22,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":22,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.14865487460862087,"confidence_level":0.95}}},"mask":{"n":8,"raw":{"n":8,"acceptable":0,"unsafe":3,"review":0,"automated_count":8,"automated_acceptable":0,"automated_unsafe":3,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":0.375,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":0.375,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.32440756488388023,"confidence_level":0.95}},"effective":{"n":8,"acceptable":0,"unsafe":0,"review":8,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":8,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.32440756488388023,"confidence_level":0.95}}},"pseudonymize":{"n":8,"raw":{"n":8,"acceptable":0,"unsafe":5,"review":0,"automated_count":8,"automated_acceptable":0,"automated_unsafe":5,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":0.625,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":0.625,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.32440756488388023,"confidence_level":0.95}},"effective":{"n":8,"acceptable":0,"unsafe":0,"review":8,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":8,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.32440756488388023,"confidence_level":0.95}}},"generalize":{"n":8,"raw":{"n":8,"acceptable":0,"unsafe":5,"review":0,"automated_count":8,"automated_acceptable":0,"automated_unsafe":5,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":0.625,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":0.625,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.32440756488388023,"confidence_level":0.95}},"effective":{"n":8,"acceptable":0,"unsafe":0,"review":8,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":8,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.32440756488388023,"confidence_level":0.95}}},"review":{"n":2,"raw":{"n":2,"acceptable":0,"unsafe":1,"review":0,"automated_count":2,"automated_acceptable":0,"automated_unsafe":1,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":0.5,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":0.5,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.6576197724933469,"confidence_level":0.95}},"effective":{"n":2,"acceptable":2,"unsafe":0,"review":2,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":1.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.34238022750665303,"upper":1.0,"confidence_level":0.95}}}}}},"composite":{"n":198,"scope":"descriptive sum across four correlated panels","raw":{"n":198,"acceptable":53,"unsafe":99,"review":0,"automated_count":198,"automated_acceptable":53,"automated_unsafe":99,"unnecessary_reviews":0,"accuracy":0.2676767676767677,"unsafe_rate":0.5,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.2676767676767677,"conditional_unsafe":0.5,"unnecessary_review_rate":0.0},"effective":{"n":198,"acceptable":14,"unsafe":0,"review":196,"automated_count":2,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":182,"accuracy":0.0707070707070707,"unsafe_rate":0.0,"review_rate":0.98989898989899,"automated_coverage":0.010101010101010102,"conditional_correctness":0.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.9191919191919192}}},{"id":"MoritzLaurer--ModernBERT-base-zeroshot-v2.0","platform":"Local","model_id":"MoritzLaurer/ModernBERT-base-zeroshot-v2.0","revision":"d421c4545a438fd006fb43f8b981c5d908faa1e1","loading":"float32 / none","scoring_kind":"Adapted NLI","settings":{"dtype":"float32","quantization":null,"max_tokens":4096,"effective_max_tokens":512,"temperature":1.0,"scoring_batch_size":1,"trust_remote_code":false,"local_files_only":true,"use_safetensors":true,"guard_threshold":0.45,"seed":0,"tf32":false},"scoring_metadata":{"backend":"nli","protocol":"hf-local-sweep-backends-v1","probability_source":"adapted_entailment_softmax_not_calibrated_sixway","latency_scope":"synchronous_forward_scoring_including_transfer_excluding_tokenization","precision":"float32","equal_mass_convention":"rank bins with floor boundaries at floor(i*n/k), k=min(10,n//2); minimum two rows"},"load_ms":5699.521200032905,"peak_allocated_bytes":665742336,"peak_reserved_bytes":681574400,"panels":{"diagnostic43-no_samples":{"n":43,"raw":{"n":43,"acceptable":6,"unsafe":12,"review":0,"automated_count":43,"automated_acceptable":6,"automated_unsafe":12,"unnecessary_reviews":0,"accuracy":0.13953488372093023,"unsafe_rate":0.27906976744186046,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.13953488372093023,"conditional_unsafe":0.27906976744186046,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.06555723427964878,"upper":0.2726358796738286,"confidence_level":0.95}},"effective":{"n":43,"acceptable":5,"unsafe":0,"review":43,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":38,"accuracy":0.11627906976744186,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":0.8837209302325582,"accuracy_wilson_95":{"lower":0.05070427739804166,"upper":0.24479161810082128,"confidence_level":0.95}},"confusion":{"raw":{"keep":{"keep":0,"drop":0,"mask":3,"pseudonymize":9,"generalize":0,"review":0},"drop":{"keep":0,"drop":1,"mask":4,"pseudonymize":8,"generalize":3,"review":0},"mask":{"keep":0,"drop":0,"mask":0,"pseudonymize":1,"generalize":1,"review":0},"pseudonymize":{"keep":0,"drop":0,"mask":1,"pseudonymize":1,"generalize":0,"review":0},"generalize":{"keep":0,"drop":0,"mask":0,"pseudonymize":5,"generalize":1,"review":0},"review":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0}},"effective":{"keep":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":12},"drop":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":16},"mask":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":2},"pseudonymize":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":2},"generalize":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":6},"review":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0}}},"probability_metrics":{"singleton_n":38,"multiacceptable_n":5,"multiacceptable_coverage":0.11627906976744186,"nll":1.805167922751127,"nll_zero_mass_n":0,"brier":0.8445981023739241,"acceptable_set_n":43,"acceptable_set_nll":1.6887316601181965,"acceptable_set_zero_mass_n":0},"calibration_all_labels":{"equal_width":{"n":43,"bin_count":10,"ece":0.0959777468165686,"signed_mean_gap":0.07652852043162944,"absolute_mean_gap":0.07652852043162944,"bins":[{"index":0,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.0,"upper":0.1,"upper_inclusive":false},{"index":1,"count":8,"confidence":0.19773020409047604,"empirical_correctness":0.25,"signed_gap":-0.052269795909523964,"absolute_gap":0.052269795909523964,"lower":0.1,"upper":0.2,"upper_inclusive":false},{"index":2,"count":35,"confidence":0.22025384988103594,"empirical_correctness":0.11428571428571428,"signed_gap":0.10596813559532166,"absolute_gap":0.10596813559532166,"lower":0.2,"upper":0.3,"upper_inclusive":false},{"index":3,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.3,"upper":0.4,"upper_inclusive":false},{"index":4,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.4,"upper":0.5,"upper_inclusive":false},{"index":5,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.5,"upper":0.6,"upper_inclusive":false},{"index":6,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.6,"upper":0.7,"upper_inclusive":false},{"index":7,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.7,"upper":0.8,"upper_inclusive":false},{"index":8,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.8,"upper":0.9,"upper_inclusive":false},{"index":9,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.9,"upper":1.0,"upper_inclusive":true}]},"equal_mass":{"n":43,"bin_count":10,"ece":0.1997610715239547,"signed_mean_gap":0.07652852043162944,"absolute_mean_gap":0.07652852043162944,"bins":[{"index":0,"count":4,"confidence":0.19683023914694786,"empirical_correctness":0.5,"signed_gap":-0.30316976085305214,"absolute_gap":0.30316976085305214,"min_confidence":0.19495637714862823,"max_confidence":0.19838818907737732},{"index":1,"count":4,"confidence":0.1986301690340042,"empirical_correctness":0.0,"signed_gap":0.1986301690340042,"absolute_gap":0.1986301690340042,"min_confidence":0.19848336279392242,"max_confidence":0.19892708957195282},{"index":2,"count":4,"confidence":0.20146925747394562,"empirical_correctness":0.5,"signed_gap":-0.2985307425260544,"absolute_gap":0.2985307425260544,"min_confidence":0.20025862753391266,"max_confidence":0.20318111777305603},{"index":3,"count":5,"confidence":0.2066132456064224,"empirical_correctness":0.0,"signed_gap":0.2066132456064224,"absolute_gap":0.2066132456064224,"min_confidence":0.20359762012958527,"max_confidence":0.20807920396327972},{"index":4,"count":4,"confidence":0.2091706246137619,"empirical_correctness":0.25,"signed_gap":-0.0408293753862381,"absolute_gap":0.0408293753862381,"min_confidence":0.2084445059299469,"max_confidence":0.2101234644651413},{"index":5,"count":4,"confidence":0.21257229894399643,"empirical_correctness":0.0,"signed_gap":0.21257229894399643,"absolute_gap":0.21257229894399643,"min_confidence":0.2103177011013031,"max_confidence":0.2139868438243866},{"index":6,"count":5,"confidence":0.21851450204849243,"empirical_correctness":0.0,"signed_gap":0.21851450204849243,"absolute_gap":0.21851450204849243,"min_confidence":0.21417316794395447,"max_confidence":0.22440463304519653},{"index":7,"count":4,"confidence":0.23015491664409637,"empirical_correctness":0.25,"signed_gap":-0.019845083355903625,"absolute_gap":0.019845083355903625,"min_confidence":0.22685696184635162,"max_confidence":0.23404958844184875},{"index":8,"count":4,"confidence":0.23594613373279572,"empirical_correctness":0.0,"signed_gap":0.23594613373279572,"absolute_gap":0.23594613373279572,"min_confidence":0.2348550707101822,"max_confidence":0.23690596222877502},{"index":9,"count":5,"confidence":0.2451986163854599,"empirical_correctness":0.0,"signed_gap":0.2451986163854599,"absolute_gap":0.2451986163854599,"min_confidence":0.2422655075788498,"max_confidence":0.24905577301979065}]}},"calibration_singleton_labels":{"equal_width":{"n":38,"bin_count":10,"ece":0.13824142319591423,"signed_mean_gap":0.13824142319591423,"absolute_mean_gap":0.13824142319591423,"bins":[{"index":0,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.0,"upper":0.1,"upper_inclusive":false},{"index":1,"count":7,"confidence":0.19788586454732077,"empirical_correctness":0.14285714285714285,"signed_gap":0.05502872169017792,"absolute_gap":0.05502872169017792,"lower":0.1,"upper":0.2,"upper_inclusive":false},{"index":2,"count":31,"confidence":0.22154751708430628,"empirical_correctness":0.06451612903225806,"signed_gap":0.15703138805204822,"absolute_gap":0.15703138805204822,"lower":0.2,"upper":0.3,"upper_inclusive":false},{"index":3,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.3,"upper":0.4,"upper_inclusive":false},{"index":4,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.4,"upper":0.5,"upper_inclusive":false},{"index":5,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.5,"upper":0.6,"upper_inclusive":false},{"index":6,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.6,"upper":0.7,"upper_inclusive":false},{"index":7,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.7,"upper":0.8,"upper_inclusive":false},{"index":8,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.8,"upper":0.9,"upper_inclusive":false},{"index":9,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.9,"upper":1.0,"upper_inclusive":true}]},"equal_mass":{"n":38,"bin_count":10,"ece":0.17610060658894086,"signed_mean_gap":0.13824142319591423,"absolute_mean_gap":0.13824142319591423,"bins":[{"index":0,"count":3,"confidence":0.1968934585650762,"empirical_correctness":0.3333333333333333,"signed_gap":-0.1364398747682571,"absolute_gap":0.1364398747682571,"min_confidence":0.19495637714862823,"max_confidence":0.19838818907737732},{"index":1,"count":4,"confidence":0.1986301690340042,"empirical_correctness":0.0,"signed_gap":0.1986301690340042,"absolute_gap":0.1986301690340042,"min_confidence":0.19848336279392242,"max_confidence":0.19892708957195282},{"index":2,"count":4,"confidence":0.20228586718440056,"empirical_correctness":0.25,"signed_gap":-0.04771413281559944,"absolute_gap":0.04771413281559944,"min_confidence":0.20025862753391266,"max_confidence":0.20359762012958527},{"index":3,"count":4,"confidence":0.20777180418372154,"empirical_correctness":0.0,"signed_gap":0.20777180418372154,"absolute_gap":0.20777180418372154,"min_confidence":0.2072601169347763,"max_confidence":0.2084445059299469},{"index":4,"count":4,"confidence":0.2104702852666378,"empirical_correctness":0.0,"signed_gap":0.2104702852666378,"absolute_gap":0.2104702852666378,"min_confidence":0.20876452326774597,"max_confidence":0.21267545223236084},{"index":5,"count":3,"confidence":0.21382307012875876,"empirical_correctness":0.0,"signed_gap":0.21382307012875876,"absolute_gap":0.21382307012875876,"min_confidence":0.21330919861793518,"max_confidence":0.21417316794395447},{"index":6,"count":4,"confidence":0.2202129177749157,"empirical_correctness":0.25,"signed_gap":-0.029787082225084305,"absolute_gap":0.029787082225084305,"min_confidence":0.21506628394126892,"max_confidence":0.22685696184635162},{"index":7,"count":4,"confidence":0.23215444386005402,"empirical_correctness":0.0,"signed_gap":0.23215444386005402,"absolute_gap":0.23215444386005402,"min_confidence":0.22889213263988495,"max_confidence":0.2348550707101822},{"index":8,"count":4,"confidence":0.23779874294996262,"empirical_correctness":0.0,"signed_gap":0.23779874294996262,"absolute_gap":0.23779874294996262,"min_confidence":0.23544558882713318,"max_confidence":0.2422655075788498},{"index":9,"count":4,"confidence":0.24593189358711243,"empirical_correctness":0.0,"signed_gap":0.24593189358711243,"absolute_gap":0.24593189358711243,"min_confidence":0.2427581250667572,"max_confidence":0.24905577301979065}]}},"latency_ms":{"n":43,"median":138.92469997517765,"p95":142.07300995476544},"per_action":{"keep":{"n":12,"raw":{"n":12,"acceptable":0,"unsafe":0,"review":0,"automated_count":12,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":2.7755575615628914e-17,"upper":0.24249400665524085,"confidence_level":0.95}},"effective":{"n":12,"acceptable":0,"unsafe":0,"review":12,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":12,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":2.7755575615628914e-17,"upper":0.24249400665524085,"confidence_level":0.95}}},"drop":{"n":16,"raw":{"n":16,"acceptable":1,"unsafe":9,"review":0,"automated_count":16,"automated_acceptable":1,"automated_unsafe":9,"unnecessary_reviews":0,"accuracy":0.0625,"unsafe_rate":0.5625,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0625,"conditional_unsafe":0.5625,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.01111934476464252,"upper":0.28328737570298945,"confidence_level":0.95}},"effective":{"n":16,"acceptable":0,"unsafe":0,"review":16,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":16,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.1936076805344365,"confidence_level":0.95}}},"mask":{"n":2,"raw":{"n":2,"acceptable":0,"unsafe":1,"review":0,"automated_count":2,"automated_acceptable":0,"automated_unsafe":1,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":0.5,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":0.5,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.6576197724933469,"confidence_level":0.95}},"effective":{"n":2,"acceptable":0,"unsafe":0,"review":2,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":2,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.6576197724933469,"confidence_level":0.95}}},"pseudonymize":{"n":2,"raw":{"n":2,"acceptable":1,"unsafe":0,"review":0,"automated_count":2,"automated_acceptable":1,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":0.5,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.5,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.09453120573423074,"upper":0.9054687942657693,"confidence_level":0.95}},"effective":{"n":2,"acceptable":0,"unsafe":0,"review":2,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":2,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.6576197724933469,"confidence_level":0.95}}},"generalize":{"n":6,"raw":{"n":6,"acceptable":1,"unsafe":0,"review":0,"automated_count":6,"automated_acceptable":1,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":0.16666666666666666,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.16666666666666666,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.030053369748306635,"upper":0.5635028221864702,"confidence_level":0.95}},"effective":{"n":6,"acceptable":0,"unsafe":0,"review":6,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":6,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":2.7755575615628914e-17,"upper":0.3903342879021653,"confidence_level":0.95}}},"review":{"n":0,"raw":{"n":0,"acceptable":0,"unsafe":0,"review":0,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":null,"unsafe_rate":null,"review_rate":null,"automated_coverage":null,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":null},"effective":{"n":0,"acceptable":0,"unsafe":0,"review":0,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":null,"unsafe_rate":null,"review_rate":null,"automated_coverage":null,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":null}}}},"diagnostic43-compact_no_samples":{"n":43,"raw":{"n":43,"acceptable":3,"unsafe":24,"review":0,"automated_count":43,"automated_acceptable":3,"automated_unsafe":24,"unnecessary_reviews":0,"accuracy":0.06976744186046512,"unsafe_rate":0.5581395348837209,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.06976744186046512,"conditional_unsafe":0.5581395348837209,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.024011589577214484,"upper":0.18608986901241975,"confidence_level":0.95}},"effective":{"n":43,"acceptable":5,"unsafe":0,"review":43,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":38,"accuracy":0.11627906976744186,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":0.8837209302325582,"accuracy_wilson_95":{"lower":0.05070427739804166,"upper":0.24479161810082128,"confidence_level":0.95}},"confusion":{"raw":{"keep":{"keep":0,"drop":0,"mask":0,"pseudonymize":11,"generalize":1,"review":0},"drop":{"keep":0,"drop":0,"mask":0,"pseudonymize":15,"generalize":1,"review":0},"mask":{"keep":0,"drop":0,"mask":0,"pseudonymize":2,"generalize":0,"review":0},"pseudonymize":{"keep":0,"drop":0,"mask":0,"pseudonymize":2,"generalize":0,"review":0},"generalize":{"keep":1,"drop":0,"mask":0,"pseudonymize":4,"generalize":1,"review":0},"review":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0}},"effective":{"keep":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":12},"drop":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":16},"mask":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":2},"pseudonymize":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":2},"generalize":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":6},"review":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0}}},"probability_metrics":{"singleton_n":38,"multiacceptable_n":5,"multiacceptable_coverage":0.11627906976744186,"nll":1.754355098718298,"nll_zero_mass_n":0,"brier":0.8280814091518071,"acceptable_set_n":43,"acceptable_set_nll":1.6505715384076893,"acceptable_set_zero_mass_n":0},"calibration_all_labels":{"equal_width":{"n":43,"bin_count":10,"ece":0.1516075744185337,"signed_mean_gap":0.1516075744185337,"absolute_mean_gap":0.1516075744185337,"bins":[{"index":0,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.0,"upper":0.1,"upper_inclusive":false},{"index":1,"count":7,"confidence":0.19615633147103445,"empirical_correctness":0.0,"signed_gap":0.19615633147103445,"absolute_gap":0.19615633147103445,"lower":0.1,"upper":0.2,"upper_inclusive":false},{"index":2,"count":36,"confidence":0.22627864943610299,"empirical_correctness":0.08333333333333333,"signed_gap":0.14294531610276967,"absolute_gap":0.14294531610276967,"lower":0.2,"upper":0.3,"upper_inclusive":false},{"index":3,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.3,"upper":0.4,"upper_inclusive":false},{"index":4,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.4,"upper":0.5,"upper_inclusive":false},{"index":5,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.5,"upper":0.6,"upper_inclusive":false},{"index":6,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.6,"upper":0.7,"upper_inclusive":false},{"index":7,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.7,"upper":0.8,"upper_inclusive":false},{"index":8,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.8,"upper":0.9,"upper_inclusive":false},{"index":9,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.9,"upper":1.0,"upper_inclusive":true}]},"equal_mass":{"n":43,"bin_count":10,"ece":0.16092284820800604,"signed_mean_gap":0.15160757441853367,"absolute_mean_gap":0.15160757441853367,"bins":[{"index":0,"count":4,"confidence":0.1952342838048935,"empirical_correctness":0.0,"signed_gap":0.1952342838048935,"absolute_gap":0.1952342838048935,"min_confidence":0.19399645924568176,"max_confidence":0.19594325125217438},{"index":1,"count":4,"confidence":0.19813628122210503,"empirical_correctness":0.0,"signed_gap":0.19813628122210503,"absolute_gap":0.19813628122210503,"min_confidence":0.1960688829421997,"max_confidence":0.20038793981075287},{"index":2,"count":4,"confidence":0.2044394686818123,"empirical_correctness":0.25,"signed_gap":-0.045560531318187714,"absolute_gap":0.045560531318187714,"min_confidence":0.20155876874923706,"max_confidence":0.20709337294101715},{"index":3,"count":5,"confidence":0.21028343737125396,"empirical_correctness":0.2,"signed_gap":0.01028343737125395,"absolute_gap":0.01028343737125395,"min_confidence":0.20736271142959595,"max_confidence":0.21341536939144135},{"index":4,"count":4,"confidence":0.21545538678765297,"empirical_correctness":0.0,"signed_gap":0.21545538678765297,"absolute_gap":0.21545538678765297,"min_confidence":0.2149108201265335,"max_confidence":0.2160106599330902},{"index":5,"count":4,"confidence":0.21889333054423332,"empirical_correctness":0.0,"signed_gap":0.21889333054423332,"absolute_gap":0.21889333054423332,"min_confidence":0.21774733066558838,"max_confidence":0.21979551017284393},{"index":6,"count":5,"confidence":0.22459026873111726,"empirical_correctness":0.0,"signed_gap":0.22459026873111726,"absolute_gap":0.22459026873111726,"min_confidence":0.22000570595264435,"max_confidence":0.2313588410615921},{"index":7,"count":4,"confidence":0.23415404558181763,"empirical_correctness":0.0,"signed_gap":0.23415404558181763,"absolute_gap":0.23415404558181763,"min_confidence":0.23158220946788788,"max_confidence":0.239604189991951},{"index":8,"count":4,"confidence":0.2454909346997738,"empirical_correctness":0.25,"signed_gap":-0.0045090653002262115,"absolute_gap":0.0045090653002262115,"min_confidence":0.2411540150642395,"max_confidence":0.2520636320114136},{"index":9,"count":5,"confidence":0.2595084488391876,"empirical_correctness":0.0,"signed_gap":0.2595084488391876,"absolute_gap":0.2595084488391876,"min_confidence":0.2547793388366699,"max_confidence":0.26497676968574524}]}},"calibration_singleton_labels":{"equal_width":{"n":38,"bin_count":10,"ece":0.14231223809091667,"signed_mean_gap":0.14231223809091667,"absolute_mean_gap":0.14231223809091667,"bins":[{"index":0,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.0,"upper":0.1,"upper_inclusive":false},{"index":1,"count":7,"confidence":0.19615633147103445,"empirical_correctness":0.0,"signed_gap":0.19615633147103445,"absolute_gap":0.19615633147103445,"lower":0.1,"upper":0.2,"upper_inclusive":false},{"index":2,"count":31,"confidence":0.22692808797282557,"empirical_correctness":0.0967741935483871,"signed_gap":0.13015389442443848,"absolute_gap":0.13015389442443848,"lower":0.2,"upper":0.3,"upper_inclusive":false},{"index":3,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.3,"upper":0.4,"upper_inclusive":false},{"index":4,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.4,"upper":0.5,"upper_inclusive":false},{"index":5,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.5,"upper":0.6,"upper_inclusive":false},{"index":6,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.6,"upper":0.7,"upper_inclusive":false},{"index":7,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.7,"upper":0.8,"upper_inclusive":false},{"index":8,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.8,"upper":0.9,"upper_inclusive":false},{"index":9,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.9,"upper":1.0,"upper_inclusive":true}]},"equal_mass":{"n":38,"bin_count":10,"ece":0.1614708053438287,"signed_mean_gap":0.14231223809091667,"absolute_mean_gap":0.14231223809091667,"bins":[{"index":0,"count":3,"confidence":0.19499796132246652,"empirical_correctness":0.0,"signed_gap":0.19499796132246652,"absolute_gap":0.19499796132246652,"min_confidence":0.19399645924568176,"max_confidence":0.19569261372089386},{"index":1,"count":4,"confidence":0.1970251090824604,"empirical_correctness":0.0,"signed_gap":0.1970251090824604,"absolute_gap":0.1970251090824604,"min_confidence":0.19594325125217438,"max_confidence":0.19880810379981995},{"index":2,"count":4,"confidence":0.20291876792907715,"empirical_correctness":0.25,"signed_gap":-0.04708123207092285,"absolute_gap":0.04708123207092285,"min_confidence":0.20038793981075287,"max_confidence":0.20709337294101715},{"index":3,"count":4,"confidence":0.21058710291981697,"empirical_correctness":0.25,"signed_gap":-0.03941289708018303,"absolute_gap":0.03941289708018303,"min_confidence":0.20736271142959595,"max_confidence":0.21341536939144135},{"index":4,"count":4,"confidence":0.21545538678765297,"empirical_correctness":0.0,"signed_gap":0.21545538678765297,"absolute_gap":0.21545538678765297,"min_confidence":0.2149108201265335,"max_confidence":0.2160106599330902},{"index":5,"count":3,"confidence":0.218610813220342,"empirical_correctness":0.0,"signed_gap":0.218610813220342,"absolute_gap":0.218610813220342,"min_confidence":0.21774733066558838,"max_confidence":0.21979551017284393},{"index":6,"count":4,"confidence":0.2256636880338192,"empirical_correctness":0.0,"signed_gap":0.2256636880338192,"absolute_gap":0.2256636880338192,"min_confidence":0.22000570595264435,"max_confidence":0.2313588410615921},{"index":7,"count":4,"confidence":0.23415404558181763,"empirical_correctness":0.0,"signed_gap":0.23415404558181763,"absolute_gap":0.23415404558181763,"min_confidence":0.23158220946788788,"max_confidence":0.239604189991951},{"index":8,"count":4,"confidence":0.2454909346997738,"empirical_correctness":0.25,"signed_gap":-0.0045090653002262115,"absolute_gap":0.0045090653002262115,"min_confidence":0.2411540150642395,"max_confidence":0.2520636320114136},{"index":9,"count":4,"confidence":0.260464645922184,"empirical_correctness":0.0,"signed_gap":0.260464645922184,"absolute_gap":0.260464645922184,"min_confidence":0.2547793388366699,"max_confidence":0.26497676968574524}]}},"latency_ms":{"n":43,"median":165.89529998600483,"p95":168.23220991063863},"per_action":{"keep":{"n":12,"raw":{"n":12,"acceptable":0,"unsafe":0,"review":0,"automated_count":12,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":2.7755575615628914e-17,"upper":0.24249400665524085,"confidence_level":0.95}},"effective":{"n":12,"acceptable":0,"unsafe":0,"review":12,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":12,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":2.7755575615628914e-17,"upper":0.24249400665524085,"confidence_level":0.95}}},"drop":{"n":16,"raw":{"n":16,"acceptable":0,"unsafe":16,"review":0,"automated_count":16,"automated_acceptable":0,"automated_unsafe":16,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":1.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":1.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.1936076805344365,"confidence_level":0.95}},"effective":{"n":16,"acceptable":0,"unsafe":0,"review":16,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":16,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.1936076805344365,"confidence_level":0.95}}},"mask":{"n":2,"raw":{"n":2,"acceptable":0,"unsafe":2,"review":0,"automated_count":2,"automated_acceptable":0,"automated_unsafe":2,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":1.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":1.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.6576197724933469,"confidence_level":0.95}},"effective":{"n":2,"acceptable":0,"unsafe":0,"review":2,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":2,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.6576197724933469,"confidence_level":0.95}}},"pseudonymize":{"n":2,"raw":{"n":2,"acceptable":2,"unsafe":0,"review":0,"automated_count":2,"automated_acceptable":2,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":1.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.34238022750665303,"upper":1.0,"confidence_level":0.95}},"effective":{"n":2,"acceptable":0,"unsafe":0,"review":2,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":2,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.6576197724933469,"confidence_level":0.95}}},"generalize":{"n":6,"raw":{"n":6,"acceptable":1,"unsafe":1,"review":0,"automated_count":6,"automated_acceptable":1,"automated_unsafe":1,"unnecessary_reviews":0,"accuracy":0.16666666666666666,"unsafe_rate":0.16666666666666666,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.16666666666666666,"conditional_unsafe":0.16666666666666666,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.030053369748306635,"upper":0.5635028221864702,"confidence_level":0.95}},"effective":{"n":6,"acceptable":0,"unsafe":0,"review":6,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":6,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":2.7755575615628914e-17,"upper":0.3903342879021653,"confidence_level":0.95}}},"review":{"n":0,"raw":{"n":0,"acceptable":0,"unsafe":0,"review":0,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":null,"unsafe_rate":null,"review_rate":null,"automated_coverage":null,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":null},"effective":{"n":0,"acceptable":0,"unsafe":0,"review":0,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":null,"unsafe_rate":null,"review_rate":null,"automated_coverage":null,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":null}}}},"challenge56-no_samples":{"n":56,"raw":{"n":56,"acceptable":21,"unsafe":13,"review":0,"automated_count":56,"automated_acceptable":21,"automated_unsafe":13,"unnecessary_reviews":0,"accuracy":0.375,"unsafe_rate":0.23214285714285715,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.375,"conditional_unsafe":0.23214285714285715,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.260102143264709,"upper":0.5059463409149819,"confidence_level":0.95}},"effective":{"n":56,"acceptable":2,"unsafe":0,"review":56,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":54,"accuracy":0.03571428571428571,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":0.9642857142857143,"accuracy_wilson_95":{"lower":0.00984942514823639,"upper":0.12118780180490107,"confidence_level":0.95}},"confusion":{"raw":{"keep":{"keep":3,"drop":0,"mask":2,"pseudonymize":0,"generalize":3,"review":0},"drop":{"keep":0,"drop":0,"mask":5,"pseudonymize":5,"generalize":12,"review":0},"mask":{"keep":0,"drop":0,"mask":7,"pseudonymize":0,"generalize":1,"review":0},"pseudonymize":{"keep":0,"drop":0,"mask":0,"pseudonymize":4,"generalize":4,"review":0},"generalize":{"keep":1,"drop":0,"mask":0,"pseudonymize":0,"generalize":7,"review":0},"review":{"keep":0,"drop":0,"mask":0,"pseudonymize":2,"generalize":0,"review":0}},"effective":{"keep":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":8},"drop":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":22},"mask":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":8},"pseudonymize":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":8},"generalize":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":8},"review":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":2}}},"probability_metrics":{"singleton_n":56,"multiacceptable_n":0,"multiacceptable_coverage":0.0,"nll":1.733456576097915,"nll_zero_mass_n":0,"brier":0.814622952478963,"acceptable_set_n":56,"acceptable_set_nll":1.733456576097915,"acceptable_set_zero_mass_n":0},"calibration_all_labels":{"equal_width":{"n":56,"bin_count":10,"ece":0.18186935276857444,"signed_mean_gap":-0.17276036606303283,"absolute_mean_gap":0.17276036606303283,"bins":[{"index":0,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.0,"upper":0.1,"upper_inclusive":false},{"index":1,"count":27,"confidence":0.1946315417687098,"empirical_correctness":0.18518518518518517,"signed_gap":0.009446356583524634,"absolute_gap":0.009446356583524634,"lower":0.1,"upper":0.2,"upper_inclusive":false},{"index":2,"count":29,"confidence":0.20932303009362058,"empirical_correctness":0.5517241379310345,"signed_gap":-0.3424011078374139,"absolute_gap":0.3424011078374139,"lower":0.2,"upper":0.3,"upper_inclusive":false},{"index":3,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.3,"upper":0.4,"upper_inclusive":false},{"index":4,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.4,"upper":0.5,"upper_inclusive":false},{"index":5,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.5,"upper":0.6,"upper_inclusive":false},{"index":6,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.6,"upper":0.7,"upper_inclusive":false},{"index":7,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.7,"upper":0.8,"upper_inclusive":false},{"index":8,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.8,"upper":0.9,"upper_inclusive":false},{"index":9,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.9,"upper":1.0,"upper_inclusive":true}]},"equal_mass":{"n":56,"bin_count":10,"ece":0.22203302197158337,"signed_mean_gap":-0.17276036606303283,"absolute_mean_gap":0.17276036606303283,"bins":[{"index":0,"count":5,"confidence":0.1891349583864212,"empirical_correctness":0.4,"signed_gap":-0.21086504161357883,"absolute_gap":0.21086504161357883,"min_confidence":0.1874818056821823,"max_confidence":0.19118504226207733},{"index":1,"count":6,"confidence":0.19218701869249344,"empirical_correctness":0.3333333333333333,"signed_gap":-0.14114631464083988,"absolute_gap":0.14114631464083988,"min_confidence":0.19147901237010956,"max_confidence":0.1931537687778473},{"index":2,"count":5,"confidence":0.19533725082874298,"empirical_correctness":0.2,"signed_gap":-0.00466274917125703,"absolute_gap":0.00466274917125703,"min_confidence":0.19481226801872253,"max_confidence":0.19589656591415405},{"index":3,"count":6,"confidence":0.19698316852251688,"empirical_correctness":0.0,"signed_gap":0.19698316852251688,"absolute_gap":0.19698316852251688,"min_confidence":0.19601604342460632,"max_confidence":0.19863368570804596},{"index":4,"count":6,"confidence":0.19962255905071893,"empirical_correctness":0.16666666666666666,"signed_gap":0.03295589238405228,"absolute_gap":0.03295589238405228,"min_confidence":0.19912631809711456,"max_confidence":0.20006589591503143},{"index":5,"count":5,"confidence":0.20147487819194793,"empirical_correctness":0.4,"signed_gap":-0.1985251218080521,"absolute_gap":0.1985251218080521,"min_confidence":0.2002653330564499,"max_confidence":0.2025148719549179},{"index":6,"count":6,"confidence":0.2038707286119461,"empirical_correctness":0.3333333333333333,"signed_gap":-0.1294626047213872,"absolute_gap":0.1294626047213872,"min_confidence":0.20280583202838898,"max_confidence":0.20592601597309113},{"index":7,"count":5,"confidence":0.20852948427200318,"empirical_correctness":0.8,"signed_gap":-0.5914705157279969,"absolute_gap":0.5914705157279969,"min_confidence":0.206091046333313,"max_confidence":0.2096806913614273},{"index":8,"count":6,"confidence":0.21090872337420782,"empirical_correctness":0.5,"signed_gap":-0.28909127662579215,"absolute_gap":0.28909127662579215,"min_confidence":0.21012894809246063,"max_confidence":0.212220698595047},{"index":9,"count":6,"confidence":0.22193390876054764,"empirical_correctness":0.6666666666666666,"signed_gap":-0.444732757906119,"absolute_gap":0.444732757906119,"min_confidence":0.21259361505508423,"max_confidence":0.23607413470745087}]}},"calibration_singleton_labels":{"equal_width":{"n":56,"bin_count":10,"ece":0.18186935276857444,"signed_mean_gap":-0.17276036606303283,"absolute_mean_gap":0.17276036606303283,"bins":[{"index":0,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.0,"upper":0.1,"upper_inclusive":false},{"index":1,"count":27,"confidence":0.1946315417687098,"empirical_correctness":0.18518518518518517,"signed_gap":0.009446356583524634,"absolute_gap":0.009446356583524634,"lower":0.1,"upper":0.2,"upper_inclusive":false},{"index":2,"count":29,"confidence":0.20932303009362058,"empirical_correctness":0.5517241379310345,"signed_gap":-0.3424011078374139,"absolute_gap":0.3424011078374139,"lower":0.2,"upper":0.3,"upper_inclusive":false},{"index":3,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.3,"upper":0.4,"upper_inclusive":false},{"index":4,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.4,"upper":0.5,"upper_inclusive":false},{"index":5,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.5,"upper":0.6,"upper_inclusive":false},{"index":6,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.6,"upper":0.7,"upper_inclusive":false},{"index":7,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.7,"upper":0.8,"upper_inclusive":false},{"index":8,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.8,"upper":0.9,"upper_inclusive":false},{"index":9,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.9,"upper":1.0,"upper_inclusive":true}]},"equal_mass":{"n":56,"bin_count":10,"ece":0.22203302197158337,"signed_mean_gap":-0.17276036606303283,"absolute_mean_gap":0.17276036606303283,"bins":[{"index":0,"count":5,"confidence":0.1891349583864212,"empirical_correctness":0.4,"signed_gap":-0.21086504161357883,"absolute_gap":0.21086504161357883,"min_confidence":0.1874818056821823,"max_confidence":0.19118504226207733},{"index":1,"count":6,"confidence":0.19218701869249344,"empirical_correctness":0.3333333333333333,"signed_gap":-0.14114631464083988,"absolute_gap":0.14114631464083988,"min_confidence":0.19147901237010956,"max_confidence":0.1931537687778473},{"index":2,"count":5,"confidence":0.19533725082874298,"empirical_correctness":0.2,"signed_gap":-0.00466274917125703,"absolute_gap":0.00466274917125703,"min_confidence":0.19481226801872253,"max_confidence":0.19589656591415405},{"index":3,"count":6,"confidence":0.19698316852251688,"empirical_correctness":0.0,"signed_gap":0.19698316852251688,"absolute_gap":0.19698316852251688,"min_confidence":0.19601604342460632,"max_confidence":0.19863368570804596},{"index":4,"count":6,"confidence":0.19962255905071893,"empirical_correctness":0.16666666666666666,"signed_gap":0.03295589238405228,"absolute_gap":0.03295589238405228,"min_confidence":0.19912631809711456,"max_confidence":0.20006589591503143},{"index":5,"count":5,"confidence":0.20147487819194793,"empirical_correctness":0.4,"signed_gap":-0.1985251218080521,"absolute_gap":0.1985251218080521,"min_confidence":0.2002653330564499,"max_confidence":0.2025148719549179},{"index":6,"count":6,"confidence":0.2038707286119461,"empirical_correctness":0.3333333333333333,"signed_gap":-0.1294626047213872,"absolute_gap":0.1294626047213872,"min_confidence":0.20280583202838898,"max_confidence":0.20592601597309113},{"index":7,"count":5,"confidence":0.20852948427200318,"empirical_correctness":0.8,"signed_gap":-0.5914705157279969,"absolute_gap":0.5914705157279969,"min_confidence":0.206091046333313,"max_confidence":0.2096806913614273},{"index":8,"count":6,"confidence":0.21090872337420782,"empirical_correctness":0.5,"signed_gap":-0.28909127662579215,"absolute_gap":0.28909127662579215,"min_confidence":0.21012894809246063,"max_confidence":0.212220698595047},{"index":9,"count":6,"confidence":0.22193390876054764,"empirical_correctness":0.6666666666666666,"signed_gap":-0.444732757906119,"absolute_gap":0.444732757906119,"min_confidence":0.21259361505508423,"max_confidence":0.23607413470745087}]}},"latency_ms":{"n":56,"median":116.6182499146089,"p95":137.6867999206297},"per_action":{"keep":{"n":8,"raw":{"n":8,"acceptable":3,"unsafe":0,"review":0,"automated_count":8,"automated_acceptable":3,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":0.375,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.375,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.13684428582359742,"upper":0.6942576053973727,"confidence_level":0.95}},"effective":{"n":8,"acceptable":0,"unsafe":0,"review":8,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":8,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.32440756488388023,"confidence_level":0.95}}},"drop":{"n":22,"raw":{"n":22,"acceptable":0,"unsafe":10,"review":0,"automated_count":22,"automated_acceptable":0,"automated_unsafe":10,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":0.45454545454545453,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":0.45454545454545453,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.14865487460862087,"confidence_level":0.95}},"effective":{"n":22,"acceptable":0,"unsafe":0,"review":22,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":22,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.14865487460862087,"confidence_level":0.95}}},"mask":{"n":8,"raw":{"n":8,"acceptable":7,"unsafe":0,"review":0,"automated_count":8,"automated_acceptable":7,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":0.875,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.875,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.5291118177871464,"upper":0.9775825085499433,"confidence_level":0.95}},"effective":{"n":8,"acceptable":0,"unsafe":0,"review":8,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":8,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.32440756488388023,"confidence_level":0.95}}},"pseudonymize":{"n":8,"raw":{"n":8,"acceptable":4,"unsafe":0,"review":0,"automated_count":8,"automated_acceptable":4,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":0.5,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.5,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.21521606221387757,"upper":0.7847839377861224,"confidence_level":0.95}},"effective":{"n":8,"acceptable":0,"unsafe":0,"review":8,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":8,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.32440756488388023,"confidence_level":0.95}}},"generalize":{"n":8,"raw":{"n":8,"acceptable":7,"unsafe":1,"review":0,"automated_count":8,"automated_acceptable":7,"automated_unsafe":1,"unnecessary_reviews":0,"accuracy":0.875,"unsafe_rate":0.125,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.875,"conditional_unsafe":0.125,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.5291118177871464,"upper":0.9775825085499433,"confidence_level":0.95}},"effective":{"n":8,"acceptable":0,"unsafe":0,"review":8,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":8,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.32440756488388023,"confidence_level":0.95}}},"review":{"n":2,"raw":{"n":2,"acceptable":0,"unsafe":2,"review":0,"automated_count":2,"automated_acceptable":0,"automated_unsafe":2,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":1.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":1.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.6576197724933469,"confidence_level":0.95}},"effective":{"n":2,"acceptable":2,"unsafe":0,"review":2,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":1.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.34238022750665303,"upper":1.0,"confidence_level":0.95}}}}},"challenge56-compact_no_samples":{"n":56,"raw":{"n":56,"acceptable":12,"unsafe":22,"review":0,"automated_count":56,"automated_acceptable":12,"automated_unsafe":22,"unnecessary_reviews":0,"accuracy":0.21428571428571427,"unsafe_rate":0.39285714285714285,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.21428571428571427,"conditional_unsafe":0.39285714285714285,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.1270590424909565,"upper":0.33819463563405117,"confidence_level":0.95}},"effective":{"n":56,"acceptable":2,"unsafe":0,"review":56,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":54,"accuracy":0.03571428571428571,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":0.9642857142857143,"accuracy_wilson_95":{"lower":0.00984942514823639,"upper":0.12118780180490107,"confidence_level":0.95}},"confusion":{"raw":{"keep":{"keep":0,"drop":0,"mask":0,"pseudonymize":8,"generalize":0,"review":0},"drop":{"keep":1,"drop":0,"mask":0,"pseudonymize":15,"generalize":6,"review":0},"mask":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":8,"review":0},"pseudonymize":{"keep":0,"drop":0,"mask":0,"pseudonymize":8,"generalize":0,"review":0},"generalize":{"keep":0,"drop":0,"mask":0,"pseudonymize":4,"generalize":4,"review":0},"review":{"keep":1,"drop":0,"mask":0,"pseudonymize":1,"generalize":0,"review":0}},"effective":{"keep":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":8},"drop":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":22},"mask":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":8},"pseudonymize":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":8},"generalize":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":8},"review":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":2}}},"probability_metrics":{"singleton_n":56,"multiacceptable_n":0,"multiacceptable_coverage":0.0,"nll":1.7180720659289344,"nll_zero_mass_n":0,"brier":0.8138052040495316,"acceptable_set_n":56,"acceptable_set_nll":1.7180720659289344,"acceptable_set_zero_mass_n":0},"calibration_all_labels":{"equal_width":{"n":56,"bin_count":10,"ece":0.07807209422545772,"signed_mean_gap":-0.0014547591230698836,"absolute_mean_gap":0.0014547591230698836,"bins":[{"index":0,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.0,"upper":0.1,"upper_inclusive":false},{"index":1,"count":11,"confidence":0.19502594389698721,"empirical_correctness":0.0,"signed_gap":0.19502594389698721,"absolute_gap":0.19502594389698721,"lower":0.1,"upper":0.2,"upper_inclusive":false},{"index":2,"count":45,"confidence":0.21718329124980504,"empirical_correctness":0.26666666666666666,"signed_gap":-0.049483375416861625,"absolute_gap":0.049483375416861625,"lower":0.2,"upper":0.3,"upper_inclusive":false},{"index":3,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.3,"upper":0.4,"upper_inclusive":false},{"index":4,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.4,"upper":0.5,"upper_inclusive":false},{"index":5,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.5,"upper":0.6,"upper_inclusive":false},{"index":6,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.6,"upper":0.7,"upper_inclusive":false},{"index":7,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.7,"upper":0.8,"upper_inclusive":false},{"index":8,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.8,"upper":0.9,"upper_inclusive":false},{"index":9,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.9,"upper":1.0,"upper_inclusive":true}]},"equal_mass":{"n":56,"bin_count":10,"ece":0.13041892621134007,"signed_mean_gap":-0.0014547591230698934,"absolute_mean_gap":0.0014547591230698934,"bins":[{"index":0,"count":5,"confidence":0.19279286861419678,"empirical_correctness":0.0,"signed_gap":0.19279286861419678,"absolute_gap":0.19279286861419678,"min_confidence":0.1899794191122055,"max_confidence":0.19503068923950195},{"index":1,"count":6,"confidence":0.19688683996597925,"empirical_correctness":0.0,"signed_gap":0.19688683996597925,"absolute_gap":0.19688683996597925,"min_confidence":0.19583989679813385,"max_confidence":0.19896142184734344},{"index":2,"count":5,"confidence":0.20156686305999755,"empirical_correctness":0.2,"signed_gap":0.0015668630599975364,"absolute_gap":0.0015668630599975364,"min_confidence":0.2000473141670227,"max_confidence":0.20331425964832306},{"index":3,"count":6,"confidence":0.2045061488946279,"empirical_correctness":0.5,"signed_gap":-0.2954938511053721,"absolute_gap":0.2954938511053721,"min_confidence":0.20363761484622955,"max_confidence":0.2052634060382843},{"index":4,"count":6,"confidence":0.20909958829482397,"empirical_correctness":0.3333333333333333,"signed_gap":-0.12423374503850934,"absolute_gap":0.12423374503850934,"min_confidence":0.20538315176963806,"max_confidence":0.21145811676979065},{"index":5,"count":5,"confidence":0.21294492185115815,"empirical_correctness":0.2,"signed_gap":0.012944921851158142,"absolute_gap":0.012944921851158142,"min_confidence":0.21250057220458984,"max_confidence":0.2134045958518982},{"index":6,"count":6,"confidence":0.21513664722442627,"empirical_correctness":0.0,"signed_gap":0.21513664722442627,"absolute_gap":0.21513664722442627,"min_confidence":0.2138814777135849,"max_confidence":0.21679382026195526},{"index":7,"count":5,"confidence":0.22046649754047393,"empirical_correctness":0.2,"signed_gap":0.020466497540473916,"absolute_gap":0.020466497540473916,"min_confidence":0.21816417574882507,"max_confidence":0.22165797650814056},{"index":8,"count":6,"confidence":0.22891157865524292,"empirical_correctness":0.3333333333333333,"signed_gap":-0.1044217546780904,"absolute_gap":0.1044217546780904,"min_confidence":0.22650942206382751,"max_confidence":0.23103542625904083},{"index":9,"count":6,"confidence":0.24207215259472528,"empirical_correctness":0.3333333333333333,"signed_gap":-0.09126118073860803,"absolute_gap":0.09126118073860803,"min_confidence":0.2322654277086258,"max_confidence":0.26463285088539124}]}},"calibration_singleton_labels":{"equal_width":{"n":56,"bin_count":10,"ece":0.07807209422545772,"signed_mean_gap":-0.0014547591230698836,"absolute_mean_gap":0.0014547591230698836,"bins":[{"index":0,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.0,"upper":0.1,"upper_inclusive":false},{"index":1,"count":11,"confidence":0.19502594389698721,"empirical_correctness":0.0,"signed_gap":0.19502594389698721,"absolute_gap":0.19502594389698721,"lower":0.1,"upper":0.2,"upper_inclusive":false},{"index":2,"count":45,"confidence":0.21718329124980504,"empirical_correctness":0.26666666666666666,"signed_gap":-0.049483375416861625,"absolute_gap":0.049483375416861625,"lower":0.2,"upper":0.3,"upper_inclusive":false},{"index":3,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.3,"upper":0.4,"upper_inclusive":false},{"index":4,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.4,"upper":0.5,"upper_inclusive":false},{"index":5,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.5,"upper":0.6,"upper_inclusive":false},{"index":6,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.6,"upper":0.7,"upper_inclusive":false},{"index":7,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.7,"upper":0.8,"upper_inclusive":false},{"index":8,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.8,"upper":0.9,"upper_inclusive":false},{"index":9,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.9,"upper":1.0,"upper_inclusive":true}]},"equal_mass":{"n":56,"bin_count":10,"ece":0.13041892621134007,"signed_mean_gap":-0.0014547591230698934,"absolute_mean_gap":0.0014547591230698934,"bins":[{"index":0,"count":5,"confidence":0.19279286861419678,"empirical_correctness":0.0,"signed_gap":0.19279286861419678,"absolute_gap":0.19279286861419678,"min_confidence":0.1899794191122055,"max_confidence":0.19503068923950195},{"index":1,"count":6,"confidence":0.19688683996597925,"empirical_correctness":0.0,"signed_gap":0.19688683996597925,"absolute_gap":0.19688683996597925,"min_confidence":0.19583989679813385,"max_confidence":0.19896142184734344},{"index":2,"count":5,"confidence":0.20156686305999755,"empirical_correctness":0.2,"signed_gap":0.0015668630599975364,"absolute_gap":0.0015668630599975364,"min_confidence":0.2000473141670227,"max_confidence":0.20331425964832306},{"index":3,"count":6,"confidence":0.2045061488946279,"empirical_correctness":0.5,"signed_gap":-0.2954938511053721,"absolute_gap":0.2954938511053721,"min_confidence":0.20363761484622955,"max_confidence":0.2052634060382843},{"index":4,"count":6,"confidence":0.20909958829482397,"empirical_correctness":0.3333333333333333,"signed_gap":-0.12423374503850934,"absolute_gap":0.12423374503850934,"min_confidence":0.20538315176963806,"max_confidence":0.21145811676979065},{"index":5,"count":5,"confidence":0.21294492185115815,"empirical_correctness":0.2,"signed_gap":0.012944921851158142,"absolute_gap":0.012944921851158142,"min_confidence":0.21250057220458984,"max_confidence":0.2134045958518982},{"index":6,"count":6,"confidence":0.21513664722442627,"empirical_correctness":0.0,"signed_gap":0.21513664722442627,"absolute_gap":0.21513664722442627,"min_confidence":0.2138814777135849,"max_confidence":0.21679382026195526},{"index":7,"count":5,"confidence":0.22046649754047393,"empirical_correctness":0.2,"signed_gap":0.020466497540473916,"absolute_gap":0.020466497540473916,"min_confidence":0.21816417574882507,"max_confidence":0.22165797650814056},{"index":8,"count":6,"confidence":0.22891157865524292,"empirical_correctness":0.3333333333333333,"signed_gap":-0.1044217546780904,"absolute_gap":0.1044217546780904,"min_confidence":0.22650942206382751,"max_confidence":0.23103542625904083},{"index":9,"count":6,"confidence":0.24207215259472528,"empirical_correctness":0.3333333333333333,"signed_gap":-0.09126118073860803,"absolute_gap":0.09126118073860803,"min_confidence":0.2322654277086258,"max_confidence":0.26463285088539124}]}},"latency_ms":{"n":56,"median":140.44845011085272,"p95":142.91177492123097},"per_action":{"keep":{"n":8,"raw":{"n":8,"acceptable":0,"unsafe":0,"review":0,"automated_count":8,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.32440756488388023,"confidence_level":0.95}},"effective":{"n":8,"acceptable":0,"unsafe":0,"review":8,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":8,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.32440756488388023,"confidence_level":0.95}}},"drop":{"n":22,"raw":{"n":22,"acceptable":0,"unsafe":20,"review":0,"automated_count":22,"automated_acceptable":0,"automated_unsafe":20,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":0.9090909090909091,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":0.9090909090909091,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.14865487460862087,"confidence_level":0.95}},"effective":{"n":22,"acceptable":0,"unsafe":0,"review":22,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":22,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.14865487460862087,"confidence_level":0.95}}},"mask":{"n":8,"raw":{"n":8,"acceptable":0,"unsafe":0,"review":0,"automated_count":8,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.32440756488388023,"confidence_level":0.95}},"effective":{"n":8,"acceptable":0,"unsafe":0,"review":8,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":8,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.32440756488388023,"confidence_level":0.95}}},"pseudonymize":{"n":8,"raw":{"n":8,"acceptable":8,"unsafe":0,"review":0,"automated_count":8,"automated_acceptable":8,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":1.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.6755924351161198,"upper":1.0,"confidence_level":0.95}},"effective":{"n":8,"acceptable":0,"unsafe":0,"review":8,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":8,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.32440756488388023,"confidence_level":0.95}}},"generalize":{"n":8,"raw":{"n":8,"acceptable":4,"unsafe":0,"review":0,"automated_count":8,"automated_acceptable":4,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":0.5,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.5,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.21521606221387757,"upper":0.7847839377861224,"confidence_level":0.95}},"effective":{"n":8,"acceptable":0,"unsafe":0,"review":8,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":8,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.32440756488388023,"confidence_level":0.95}}},"review":{"n":2,"raw":{"n":2,"acceptable":0,"unsafe":2,"review":0,"automated_count":2,"automated_acceptable":0,"automated_unsafe":2,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":1.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":1.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.6576197724933469,"confidence_level":0.95}},"effective":{"n":2,"acceptable":2,"unsafe":0,"review":2,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":1.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.34238022750665303,"upper":1.0,"confidence_level":0.95}}}}}},"composite":{"n":198,"scope":"descriptive sum across four correlated panels","raw":{"n":198,"acceptable":42,"unsafe":71,"review":0,"automated_count":198,"automated_acceptable":42,"automated_unsafe":71,"unnecessary_reviews":0,"accuracy":0.21212121212121213,"unsafe_rate":0.35858585858585856,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.21212121212121213,"conditional_unsafe":0.35858585858585856,"unnecessary_review_rate":0.0},"effective":{"n":198,"acceptable":14,"unsafe":0,"review":198,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":184,"accuracy":0.0707070707070707,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":0.9292929292929293}}},{"id":"MoritzLaurer--deberta-v3-base-zeroshot-v2.0","platform":"Local","model_id":"MoritzLaurer/deberta-v3-base-zeroshot-v2.0","revision":"8e7e5af5983a0ddb1a5b45a38b129ab69e2258e8","loading":"float32 / none","scoring_kind":"Adapted NLI","settings":{"dtype":"float32","quantization":null,"max_tokens":4096,"effective_max_tokens":512,"temperature":1.0,"scoring_batch_size":1,"trust_remote_code":false,"local_files_only":true,"use_safetensors":true,"guard_threshold":0.45,"seed":0,"tf32":false},"scoring_metadata":{"backend":"nli","protocol":"hf-local-sweep-backends-v1","probability_source":"adapted_entailment_softmax_not_calibrated_sixway","latency_scope":"synchronous_forward_scoring_including_transfer_excluding_tokenization","precision":"float32","equal_mass_convention":"rank bins with floor boundaries at floor(i*n/k), k=min(10,n//2); minimum two rows"},"load_ms":6260.787799954414,"peak_allocated_bytes":846678016,"peak_reserved_bytes":870318080,"panels":{"diagnostic43-no_samples":{"n":43,"raw":{"n":43,"acceptable":10,"unsafe":3,"review":22,"automated_count":21,"automated_acceptable":8,"automated_unsafe":3,"unnecessary_reviews":20,"accuracy":0.23255813953488372,"unsafe_rate":0.06976744186046512,"review_rate":0.5116279069767442,"automated_coverage":0.4883720930232558,"conditional_correctness":0.38095238095238093,"conditional_unsafe":0.14285714285714285,"unnecessary_review_rate":0.46511627906976744,"accuracy_wilson_95":{"lower":0.1315367847938448,"upper":0.37744520297809003,"confidence_level":0.95}},"effective":{"n":43,"acceptable":5,"unsafe":0,"review":43,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":38,"accuracy":0.11627906976744186,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":0.8837209302325582,"accuracy_wilson_95":{"lower":0.05070427739804166,"upper":0.24479161810082128,"confidence_level":0.95}},"confusion":{"raw":{"keep":{"keep":0,"drop":4,"mask":0,"pseudonymize":1,"generalize":0,"review":7},"drop":{"keep":0,"drop":5,"mask":0,"pseudonymize":2,"generalize":0,"review":9},"mask":{"keep":0,"drop":1,"mask":0,"pseudonymize":0,"generalize":0,"review":1},"pseudonymize":{"keep":0,"drop":0,"mask":0,"pseudonymize":1,"generalize":0,"review":1},"generalize":{"keep":0,"drop":2,"mask":0,"pseudonymize":2,"generalize":0,"review":2},"review":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0}},"effective":{"keep":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":12},"drop":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":16},"mask":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":2},"pseudonymize":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":2},"generalize":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":6},"review":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0}}},"probability_metrics":{"singleton_n":38,"multiacceptable_n":5,"multiacceptable_coverage":0.11627906976744186,"nll":1.9185165208171542,"nll_zero_mass_n":0,"brier":0.8524087356967255,"acceptable_set_n":43,"acceptable_set_nll":1.7562146230380533,"acceptable_set_zero_mass_n":0},"calibration_all_labels":{"equal_width":{"n":43,"bin_count":10,"ece":0.0028722913459290067,"signed_mean_gap":0.0028722913459290067,"absolute_mean_gap":0.0028722913459290067,"bins":[{"index":0,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.0,"upper":0.1,"upper_inclusive":false},{"index":1,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.1,"upper":0.2,"upper_inclusive":false},{"index":2,"count":43,"confidence":0.23543043088081272,"empirical_correctness":0.23255813953488372,"signed_gap":0.0028722913459290067,"absolute_gap":0.0028722913459290067,"lower":0.2,"upper":0.3,"upper_inclusive":false},{"index":3,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.3,"upper":0.4,"upper_inclusive":false},{"index":4,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.4,"upper":0.5,"upper_inclusive":false},{"index":5,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.5,"upper":0.6,"upper_inclusive":false},{"index":6,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.6,"upper":0.7,"upper_inclusive":false},{"index":7,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.7,"upper":0.8,"upper_inclusive":false},{"index":8,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.8,"upper":0.9,"upper_inclusive":false},{"index":9,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.9,"upper":1.0,"upper_inclusive":true}]},"equal_mass":{"n":43,"bin_count":10,"ece":0.17497534010299418,"signed_mean_gap":0.002872291345928982,"absolute_mean_gap":0.002872291345928982,"bins":[{"index":0,"count":4,"confidence":0.21418722718954086,"empirical_correctness":0.0,"signed_gap":0.21418722718954086,"absolute_gap":0.21418722718954086,"min_confidence":0.21157306432724,"max_confidence":0.21626919507980347},{"index":1,"count":4,"confidence":0.2175627127289772,"empirical_correctness":0.0,"signed_gap":0.2175627127289772,"absolute_gap":0.2175627127289772,"min_confidence":0.21645571291446686,"max_confidence":0.21880455315113068},{"index":2,"count":4,"confidence":0.2204469032585621,"empirical_correctness":0.25,"signed_gap":-0.029553096741437912,"absolute_gap":0.029553096741437912,"min_confidence":0.21949812769889832,"max_confidence":0.2209213227033615},{"index":3,"count":5,"confidence":0.225221848487854,"empirical_correctness":0.4,"signed_gap":-0.174778151512146,"absolute_gap":0.174778151512146,"min_confidence":0.22218650579452515,"max_confidence":0.2277645319700241},{"index":4,"count":4,"confidence":0.2288135550916195,"empirical_correctness":0.75,"signed_gap":-0.5211864449083805,"absolute_gap":0.5211864449083805,"min_confidence":0.22813212871551514,"max_confidence":0.2299911230802536},{"index":5,"count":4,"confidence":0.23185719177126884,"empirical_correctness":0.0,"signed_gap":0.23185719177126884,"absolute_gap":0.23185719177126884,"min_confidence":0.23109057545661926,"max_confidence":0.23282214999198914},{"index":6,"count":5,"confidence":0.23632945716381074,"empirical_correctness":0.2,"signed_gap":0.036329457163810724,"absolute_gap":0.036329457163810724,"min_confidence":0.23334237933158875,"max_confidence":0.23887795209884644},{"index":7,"count":4,"confidence":0.23990028351545334,"empirical_correctness":0.0,"signed_gap":0.23990028351545334,"absolute_gap":0.23990028351545334,"min_confidence":0.23894049227237701,"max_confidence":0.24055635929107666},{"index":8,"count":4,"confidence":0.2570117823779583,"empirical_correctness":0.25,"signed_gap":0.007011782377958298,"absolute_gap":0.007011782377958298,"min_confidence":0.24327436089515686,"max_confidence":0.2715909779071808},{"index":9,"count":5,"confidence":0.27532667517662046,"empirical_correctness":0.4,"signed_gap":-0.12467332482337956,"absolute_gap":0.12467332482337956,"min_confidence":0.2728791832923889,"max_confidence":0.2785387337207794}]}},"calibration_singleton_labels":{"equal_width":{"n":38,"bin_count":10,"ece":0.07778787691342204,"signed_mean_gap":0.07778787691342204,"absolute_mean_gap":0.07778787691342204,"bins":[{"index":0,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.0,"upper":0.1,"upper_inclusive":false},{"index":1,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.1,"upper":0.2,"upper_inclusive":false},{"index":2,"count":38,"confidence":0.2356826137555273,"empirical_correctness":0.15789473684210525,"signed_gap":0.07778787691342204,"absolute_gap":0.07778787691342204,"lower":0.2,"upper":0.3,"upper_inclusive":false},{"index":3,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.3,"upper":0.4,"upper_inclusive":false},{"index":4,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.4,"upper":0.5,"upper_inclusive":false},{"index":5,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.5,"upper":0.6,"upper_inclusive":false},{"index":6,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.6,"upper":0.7,"upper_inclusive":false},{"index":7,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.7,"upper":0.8,"upper_inclusive":false},{"index":8,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.8,"upper":0.9,"upper_inclusive":false},{"index":9,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.9,"upper":1.0,"upper_inclusive":true}]},"equal_mass":{"n":38,"bin_count":10,"ece":0.145949482133514,"signed_mean_gap":0.07778787691342204,"absolute_mean_gap":0.07778787691342204,"bins":[{"index":0,"count":3,"confidence":0.21447507043679556,"empirical_correctness":0.0,"signed_gap":0.21447507043679556,"absolute_gap":0.21447507043679556,"min_confidence":0.21157306432724,"max_confidence":0.21626919507980347},{"index":1,"count":4,"confidence":0.2175627127289772,"empirical_correctness":0.0,"signed_gap":0.2175627127289772,"absolute_gap":0.2175627127289772,"min_confidence":0.21645571291446686,"max_confidence":0.21880455315113068},{"index":2,"count":4,"confidence":0.220763199031353,"empirical_correctness":0.25,"signed_gap":-0.029236800968647003,"absolute_gap":0.029236800968647003,"min_confidence":0.21949812769889832,"max_confidence":0.22218650579452515},{"index":3,"count":4,"confidence":0.22598068416118622,"empirical_correctness":0.25,"signed_gap":-0.024019315838813782,"absolute_gap":0.024019315838813782,"min_confidence":0.22281980514526367,"max_confidence":0.2277645319700241},{"index":4,"count":4,"confidence":0.22948849201202393,"empirical_correctness":0.5,"signed_gap":-0.2705115079879761,"absolute_gap":0.2705115079879761,"min_confidence":0.22813212871551514,"max_confidence":0.23109057545661926},{"index":5,"count":3,"confidence":0.2321127305428187,"empirical_correctness":0.0,"signed_gap":0.2321127305428187,"absolute_gap":0.2321127305428187,"min_confidence":0.23129847645759583,"max_confidence":0.23282214999198914},{"index":6,"count":4,"confidence":0.23707622662186623,"empirical_correctness":0.0,"signed_gap":0.23707622662186623,"absolute_gap":0.23707622662186623,"min_confidence":0.23351816833019257,"max_confidence":0.23887795209884644},{"index":7,"count":4,"confidence":0.23990028351545334,"empirical_correctness":0.0,"signed_gap":0.23990028351545334,"absolute_gap":0.23990028351545334,"min_confidence":0.23894049227237701,"max_confidence":0.24055635929107666},{"index":8,"count":4,"confidence":0.25733383372426033,"empirical_correctness":0.25,"signed_gap":0.00733383372426033,"absolute_gap":0.00733383372426033,"min_confidence":0.24327436089515686,"max_confidence":0.2728791832923889},{"index":9,"count":4,"confidence":0.2759385481476784,"empirical_correctness":0.25,"signed_gap":0.025938548147678375,"absolute_gap":0.025938548147678375,"min_confidence":0.27437180280685425,"max_confidence":0.2785387337207794}]}},"latency_ms":{"n":43,"median":168.55980013497174,"p95":173.81692002527416},"per_action":{"keep":{"n":12,"raw":{"n":12,"acceptable":0,"unsafe":0,"review":7,"automated_count":5,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":7,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":0.5833333333333334,"automated_coverage":0.4166666666666667,"conditional_correctness":0.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.5833333333333334,"accuracy_wilson_95":{"lower":2.7755575615628914e-17,"upper":0.24249400665524085,"confidence_level":0.95}},"effective":{"n":12,"acceptable":0,"unsafe":0,"review":12,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":12,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":2.7755575615628914e-17,"upper":0.24249400665524085,"confidence_level":0.95}}},"drop":{"n":16,"raw":{"n":16,"acceptable":5,"unsafe":2,"review":9,"automated_count":7,"automated_acceptable":5,"automated_unsafe":2,"unnecessary_reviews":9,"accuracy":0.3125,"unsafe_rate":0.125,"review_rate":0.5625,"automated_coverage":0.4375,"conditional_correctness":0.7142857142857143,"conditional_unsafe":0.2857142857142857,"unnecessary_review_rate":0.5625,"accuracy_wilson_95":{"lower":0.14164643854782039,"upper":0.5559564416525933,"confidence_level":0.95}},"effective":{"n":16,"acceptable":0,"unsafe":0,"review":16,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":16,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.1936076805344365,"confidence_level":0.95}}},"mask":{"n":2,"raw":{"n":2,"acceptable":0,"unsafe":0,"review":1,"automated_count":1,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":1,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":0.5,"automated_coverage":0.5,"conditional_correctness":0.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.5,"accuracy_wilson_95":{"lower":0.0,"upper":0.6576197724933469,"confidence_level":0.95}},"effective":{"n":2,"acceptable":0,"unsafe":0,"review":2,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":2,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.6576197724933469,"confidence_level":0.95}}},"pseudonymize":{"n":2,"raw":{"n":2,"acceptable":1,"unsafe":0,"review":1,"automated_count":1,"automated_acceptable":1,"automated_unsafe":0,"unnecessary_reviews":1,"accuracy":0.5,"unsafe_rate":0.0,"review_rate":0.5,"automated_coverage":0.5,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.5,"accuracy_wilson_95":{"lower":0.09453120573423074,"upper":0.9054687942657693,"confidence_level":0.95}},"effective":{"n":2,"acceptable":0,"unsafe":0,"review":2,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":2,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.6576197724933469,"confidence_level":0.95}}},"generalize":{"n":6,"raw":{"n":6,"acceptable":0,"unsafe":0,"review":2,"automated_count":4,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":2,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":0.3333333333333333,"automated_coverage":0.6666666666666666,"conditional_correctness":0.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.3333333333333333,"accuracy_wilson_95":{"lower":2.7755575615628914e-17,"upper":0.3903342879021653,"confidence_level":0.95}},"effective":{"n":6,"acceptable":0,"unsafe":0,"review":6,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":6,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":2.7755575615628914e-17,"upper":0.3903342879021653,"confidence_level":0.95}}},"review":{"n":0,"raw":{"n":0,"acceptable":0,"unsafe":0,"review":0,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":null,"unsafe_rate":null,"review_rate":null,"automated_coverage":null,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":null},"effective":{"n":0,"acceptable":0,"unsafe":0,"review":0,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":null,"unsafe_rate":null,"review_rate":null,"automated_coverage":null,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":null}}}},"diagnostic43-compact_no_samples":{"n":43,"raw":{"n":43,"acceptable":8,"unsafe":9,"review":20,"automated_count":23,"automated_acceptable":5,"automated_unsafe":9,"unnecessary_reviews":17,"accuracy":0.18604651162790697,"unsafe_rate":0.20930232558139536,"review_rate":0.46511627906976744,"automated_coverage":0.5348837209302325,"conditional_correctness":0.21739130434782608,"conditional_unsafe":0.391304347826087,"unnecessary_review_rate":0.3953488372093023,"accuracy_wilson_95":{"lower":0.09741751428996226,"upper":0.32617003657274385,"confidence_level":0.95}},"effective":{"n":43,"acceptable":5,"unsafe":0,"review":43,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":38,"accuracy":0.11627906976744186,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":0.8837209302325582,"accuracy_wilson_95":{"lower":0.05070427739804166,"upper":0.24479161810082128,"confidence_level":0.95}},"confusion":{"raw":{"keep":{"keep":4,"drop":0,"mask":1,"pseudonymize":1,"generalize":0,"review":6},"drop":{"keep":3,"drop":0,"mask":3,"pseudonymize":2,"generalize":0,"review":8},"mask":{"keep":1,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":1},"pseudonymize":{"keep":0,"drop":0,"mask":1,"pseudonymize":1,"generalize":0,"review":0},"generalize":{"keep":1,"drop":0,"mask":1,"pseudonymize":2,"generalize":0,"review":2},"review":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0}},"effective":{"keep":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":12},"drop":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":16},"mask":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":2},"pseudonymize":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":2},"generalize":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":6},"review":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0}}},"probability_metrics":{"singleton_n":38,"multiacceptable_n":5,"multiacceptable_coverage":0.11627906976744186,"nll":1.958764113395178,"nll_zero_mass_n":0,"brier":0.8610290811680986,"acceptable_set_n":43,"acceptable_set_nll":1.798088643176973,"acceptable_set_zero_mass_n":0},"calibration_all_labels":{"equal_width":{"n":43,"bin_count":10,"ece":0.04372522824032362,"signed_mean_gap":0.04372522824032362,"absolute_mean_gap":0.04372522824032362,"bins":[{"index":0,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.0,"upper":0.1,"upper_inclusive":false},{"index":1,"count":7,"confidence":0.1891447752714157,"empirical_correctness":0.14285714285714285,"signed_gap":0.04628763241427286,"absolute_gap":0.04628763241427286,"lower":0.1,"upper":0.2,"upper_inclusive":false},{"index":2,"count":36,"confidence":0.23767142742872238,"empirical_correctness":0.19444444444444445,"signed_gap":0.043226982984277934,"absolute_gap":0.043226982984277934,"lower":0.2,"upper":0.3,"upper_inclusive":false},{"index":3,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.3,"upper":0.4,"upper_inclusive":false},{"index":4,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.4,"upper":0.5,"upper_inclusive":false},{"index":5,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.5,"upper":0.6,"upper_inclusive":false},{"index":6,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.6,"upper":0.7,"upper_inclusive":false},{"index":7,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.7,"upper":0.8,"upper_inclusive":false},{"index":8,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.8,"upper":0.9,"upper_inclusive":false},{"index":9,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.9,"upper":1.0,"upper_inclusive":true}]},"equal_mass":{"n":43,"bin_count":10,"ece":0.14411439070867937,"signed_mean_gap":0.04372522824032362,"absolute_mean_gap":0.04372522824032362,"bins":[{"index":0,"count":4,"confidence":0.18785787001252174,"empirical_correctness":0.25,"signed_gap":-0.062142129987478256,"absolute_gap":0.062142129987478256,"min_confidence":0.18709257245063782,"max_confidence":0.18867862224578857},{"index":1,"count":4,"confidence":0.1945127658545971,"empirical_correctness":0.25,"signed_gap":-0.05548723414540291,"absolute_gap":0.05548723414540291,"min_confidence":0.1899862140417099,"max_confidence":0.20546911656856537},{"index":2,"count":4,"confidence":0.20739006251096725,"empirical_correctness":0.0,"signed_gap":0.20739006251096725,"absolute_gap":0.20739006251096725,"min_confidence":0.20628002285957336,"max_confidence":0.2079814225435257},{"index":3,"count":5,"confidence":0.21443047225475312,"empirical_correctness":0.0,"signed_gap":0.21443047225475312,"absolute_gap":0.21443047225475312,"min_confidence":0.20814920961856842,"max_confidence":0.22104671597480774},{"index":4,"count":4,"confidence":0.22207852825522423,"empirical_correctness":0.25,"signed_gap":-0.027921471744775772,"absolute_gap":0.027921471744775772,"min_confidence":0.2210502177476883,"max_confidence":0.22357316315174103},{"index":5,"count":4,"confidence":0.22574179619550705,"empirical_correctness":0.0,"signed_gap":0.22574179619550705,"absolute_gap":0.22574179619550705,"min_confidence":0.22437021136283875,"max_confidence":0.22762776911258698},{"index":6,"count":5,"confidence":0.23548987805843352,"empirical_correctness":0.2,"signed_gap":0.03548987805843351,"absolute_gap":0.03548987805843351,"min_confidence":0.22777673602104187,"max_confidence":0.2525006830692291},{"index":7,"count":4,"confidence":0.25379565358161926,"empirical_correctness":0.0,"signed_gap":0.25379565358161926,"absolute_gap":0.25379565358161926,"min_confidence":0.2529907524585724,"max_confidence":0.25467735528945923},{"index":8,"count":4,"confidence":0.26031000167131424,"empirical_correctness":0.25,"signed_gap":0.01031000167131424,"absolute_gap":0.01031000167131424,"min_confidence":0.25561949610710144,"max_confidence":0.26813894510269165},{"index":9,"count":5,"confidence":0.2847672700881958,"empirical_correctness":0.6,"signed_gap":-0.3152327299118042,"absolute_gap":0.3152327299118042,"min_confidence":0.2762417793273926,"max_confidence":0.2969203591346741}]}},"calibration_singleton_labels":{"equal_width":{"n":38,"bin_count":10,"ece":0.09763746316495694,"signed_mean_gap":0.09763746316495694,"absolute_mean_gap":0.09763746316495694,"bins":[{"index":0,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.0,"upper":0.1,"upper_inclusive":false},{"index":1,"count":6,"confidence":0.1892926221092542,"empirical_correctness":0.0,"signed_gap":0.1892926221092542,"absolute_gap":0.1892926221092542,"lower":0.1,"upper":0.2,"upper_inclusive":false},{"index":2,"count":32,"confidence":0.2367021208629012,"empirical_correctness":0.15625,"signed_gap":0.08045212086290121,"absolute_gap":0.08045212086290121,"lower":0.2,"upper":0.3,"upper_inclusive":false},{"index":3,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.3,"upper":0.4,"upper_inclusive":false},{"index":4,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.4,"upper":0.5,"upper_inclusive":false},{"index":5,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.5,"upper":0.6,"upper_inclusive":false},{"index":6,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.6,"upper":0.7,"upper_inclusive":false},{"index":7,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.7,"upper":0.8,"upper_inclusive":false},{"index":8,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.8,"upper":0.9,"upper_inclusive":false},{"index":9,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.9,"upper":1.0,"upper_inclusive":true}]},"equal_mass":{"n":38,"bin_count":10,"ece":0.21289783363279544,"signed_mean_gap":0.09763746316495694,"absolute_mean_gap":0.09763746316495694,"bins":[{"index":0,"count":3,"confidence":0.1877245952685674,"empirical_correctness":0.0,"signed_gap":0.1877245952685674,"absolute_gap":0.1877245952685674,"min_confidence":0.18709257245063782,"max_confidence":0.18867862224578857},{"index":1,"count":4,"confidence":0.1945127658545971,"empirical_correctness":0.25,"signed_gap":-0.05548723414540291,"absolute_gap":0.05548723414540291,"min_confidence":0.1899862140417099,"max_confidence":0.20546911656856537},{"index":2,"count":4,"confidence":0.20785735920071602,"empirical_correctness":0.0,"signed_gap":0.20785735920071602,"absolute_gap":0.20785735920071602,"min_confidence":0.20742076635360718,"max_confidence":0.20814920961856842},{"index":3,"count":4,"confidence":0.21600078791379929,"empirical_correctness":0.0,"signed_gap":0.21600078791379929,"absolute_gap":0.21600078791379929,"min_confidence":0.20923389494419098,"max_confidence":0.22104671597480774},{"index":4,"count":4,"confidence":0.22207852825522423,"empirical_correctness":0.25,"signed_gap":-0.027921471744775772,"absolute_gap":0.027921471744775772,"min_confidence":0.2210502177476883,"max_confidence":0.22357316315174103},{"index":5,"count":3,"confidence":0.2251131385564804,"empirical_correctness":0.0,"signed_gap":0.2251131385564804,"absolute_gap":0.2251131385564804,"min_confidence":0.22437021136283875,"max_confidence":0.2258034646511078},{"index":6,"count":4,"confidence":0.22968335449695587,"empirical_correctness":0.0,"signed_gap":0.22968335449695587,"absolute_gap":0.22968335449695587,"min_confidence":0.22762776911258698,"max_confidence":0.23281894624233246},{"index":7,"count":4,"confidence":0.25325148552656174,"empirical_correctness":0.0,"signed_gap":0.25325148552656174,"absolute_gap":0.25325148552656174,"min_confidence":0.2525006830692291,"max_confidence":0.25430017709732056},{"index":8,"count":4,"confidence":0.25862137228250504,"empirical_correctness":0.0,"signed_gap":0.25862137228250504,"absolute_gap":0.25862137228250504,"min_confidence":0.25467735528945923,"max_confidence":0.26813894510269165},{"index":9,"count":4,"confidence":0.28592194616794586,"empirical_correctness":0.75,"signed_gap":-0.46407805383205414,"absolute_gap":0.46407805383205414,"min_confidence":0.2762417793273926,"max_confidence":0.2969203591346741}]}},"latency_ms":{"n":43,"median":168.83939993567765,"p95":171.62297992035747},"per_action":{"keep":{"n":12,"raw":{"n":12,"acceptable":4,"unsafe":0,"review":6,"automated_count":6,"automated_acceptable":4,"automated_unsafe":0,"unnecessary_reviews":6,"accuracy":0.3333333333333333,"unsafe_rate":0.0,"review_rate":0.5,"automated_coverage":0.5,"conditional_correctness":0.6666666666666666,"conditional_unsafe":0.0,"unnecessary_review_rate":0.5,"accuracy_wilson_95":{"lower":0.13812009109121315,"upper":0.6093779111272004,"confidence_level":0.95}},"effective":{"n":12,"acceptable":0,"unsafe":0,"review":12,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":12,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":2.7755575615628914e-17,"upper":0.24249400665524085,"confidence_level":0.95}}},"drop":{"n":16,"raw":{"n":16,"acceptable":0,"unsafe":5,"review":8,"automated_count":8,"automated_acceptable":0,"automated_unsafe":5,"unnecessary_reviews":8,"accuracy":0.0,"unsafe_rate":0.3125,"review_rate":0.5,"automated_coverage":0.5,"conditional_correctness":0.0,"conditional_unsafe":0.625,"unnecessary_review_rate":0.5,"accuracy_wilson_95":{"lower":0.0,"upper":0.1936076805344365,"confidence_level":0.95}},"effective":{"n":16,"acceptable":0,"unsafe":0,"review":16,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":16,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.1936076805344365,"confidence_level":0.95}}},"mask":{"n":2,"raw":{"n":2,"acceptable":0,"unsafe":1,"review":1,"automated_count":1,"automated_acceptable":0,"automated_unsafe":1,"unnecessary_reviews":1,"accuracy":0.0,"unsafe_rate":0.5,"review_rate":0.5,"automated_coverage":0.5,"conditional_correctness":0.0,"conditional_unsafe":1.0,"unnecessary_review_rate":0.5,"accuracy_wilson_95":{"lower":0.0,"upper":0.6576197724933469,"confidence_level":0.95}},"effective":{"n":2,"acceptable":0,"unsafe":0,"review":2,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":2,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.6576197724933469,"confidence_level":0.95}}},"pseudonymize":{"n":2,"raw":{"n":2,"acceptable":1,"unsafe":0,"review":0,"automated_count":2,"automated_acceptable":1,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":0.5,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.5,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.09453120573423074,"upper":0.9054687942657693,"confidence_level":0.95}},"effective":{"n":2,"acceptable":0,"unsafe":0,"review":2,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":2,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.6576197724933469,"confidence_level":0.95}}},"generalize":{"n":6,"raw":{"n":6,"acceptable":0,"unsafe":1,"review":2,"automated_count":4,"automated_acceptable":0,"automated_unsafe":1,"unnecessary_reviews":2,"accuracy":0.0,"unsafe_rate":0.16666666666666666,"review_rate":0.3333333333333333,"automated_coverage":0.6666666666666666,"conditional_correctness":0.0,"conditional_unsafe":0.25,"unnecessary_review_rate":0.3333333333333333,"accuracy_wilson_95":{"lower":2.7755575615628914e-17,"upper":0.3903342879021653,"confidence_level":0.95}},"effective":{"n":6,"acceptable":0,"unsafe":0,"review":6,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":6,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":2.7755575615628914e-17,"upper":0.3903342879021653,"confidence_level":0.95}}},"review":{"n":0,"raw":{"n":0,"acceptable":0,"unsafe":0,"review":0,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":null,"unsafe_rate":null,"review_rate":null,"automated_coverage":null,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":null},"effective":{"n":0,"acceptable":0,"unsafe":0,"review":0,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":null,"unsafe_rate":null,"review_rate":null,"automated_coverage":null,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":null}}}},"challenge56-no_samples":{"n":56,"raw":{"n":56,"acceptable":11,"unsafe":14,"review":29,"automated_count":27,"automated_acceptable":9,"automated_unsafe":14,"unnecessary_reviews":27,"accuracy":0.19642857142857142,"unsafe_rate":0.25,"review_rate":0.5178571428571429,"automated_coverage":0.48214285714285715,"conditional_correctness":0.3333333333333333,"conditional_unsafe":0.5185185185185185,"unnecessary_review_rate":0.48214285714285715,"accuracy_wilson_95":{"lower":0.11338595674323658,"upper":0.31844607626458404,"confidence_level":0.95}},"effective":{"n":56,"acceptable":2,"unsafe":0,"review":56,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":54,"accuracy":0.03571428571428571,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":0.9642857142857143,"accuracy_wilson_95":{"lower":0.00984942514823639,"upper":0.12118780180490107,"confidence_level":0.95}},"confusion":{"raw":{"keep":{"keep":8,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0},"drop":{"keep":9,"drop":0,"mask":1,"pseudonymize":0,"generalize":0,"review":12},"mask":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":8},"pseudonymize":{"keep":2,"drop":0,"mask":1,"pseudonymize":1,"generalize":0,"review":4},"generalize":{"keep":3,"drop":0,"mask":0,"pseudonymize":2,"generalize":0,"review":3},"review":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":2}},"effective":{"keep":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":8},"drop":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":22},"mask":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":8},"pseudonymize":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":8},"generalize":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":8},"review":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":2}}},"probability_metrics":{"singleton_n":56,"multiacceptable_n":0,"multiacceptable_coverage":0.0,"nll":1.8273110247815763,"nll_zero_mass_n":0,"brier":0.8263117937918986,"acceptable_set_n":56,"acceptable_set_nll":1.8273110247815763,"acceptable_set_zero_mass_n":0},"calibration_all_labels":{"equal_width":{"n":56,"bin_count":10,"ece":0.23005756549537182,"signed_mean_gap":0.04196198204798356,"absolute_mean_gap":0.04196198204798356,"bins":[{"index":0,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.0,"upper":0.1,"upper_inclusive":false},{"index":1,"count":7,"confidence":0.19244985708168574,"empirical_correctness":0.14285714285714285,"signed_gap":0.049592714224542894,"absolute_gap":0.049592714224542894,"lower":0.1,"upper":0.2,"upper_inclusive":false},{"index":2,"count":41,"confidence":0.22608288613761343,"empirical_correctness":0.04878048780487805,"signed_gap":0.17730239833273537,"absolute_gap":0.17730239833273537,"lower":0.2,"upper":0.3,"upper_inclusive":false},{"index":3,"count":8,"confidence":0.34166545793414116,"empirical_correctness":1.0,"signed_gap":-0.6583345420658588,"absolute_gap":0.6583345420658588,"lower":0.3,"upper":0.4,"upper_inclusive":false},{"index":4,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.4,"upper":0.5,"upper_inclusive":false},{"index":5,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.5,"upper":0.6,"upper_inclusive":false},{"index":6,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.6,"upper":0.7,"upper_inclusive":false},{"index":7,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.7,"upper":0.8,"upper_inclusive":false},{"index":8,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.8,"upper":0.9,"upper_inclusive":false},{"index":9,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.9,"upper":1.0,"upper_inclusive":true}]},"equal_mass":{"n":56,"bin_count":10,"ece":0.19557775423995086,"signed_mean_gap":0.04196198204798358,"absolute_mean_gap":0.04196198204798358,"bins":[{"index":0,"count":5,"confidence":0.18995851874351502,"empirical_correctness":0.2,"signed_gap":-0.010041481256484991,"absolute_gap":0.010041481256484991,"min_confidence":0.18749237060546875,"max_confidence":0.1946079283952713},{"index":1,"count":6,"confidence":0.2018670936425527,"empirical_correctness":0.0,"signed_gap":0.2018670936425527,"absolute_gap":0.2018670936425527,"min_confidence":0.1982421725988388,"max_confidence":0.20611070096492767},{"index":2,"count":5,"confidence":0.2097841501235962,"empirical_correctness":0.0,"signed_gap":0.2097841501235962,"absolute_gap":0.2097841501235962,"min_confidence":0.20778611302375793,"max_confidence":0.21179406344890594},{"index":3,"count":6,"confidence":0.21755906691153845,"empirical_correctness":0.0,"signed_gap":0.21755906691153845,"absolute_gap":0.21755906691153845,"min_confidence":0.21220877766609192,"max_confidence":0.22060048580169678},{"index":4,"count":6,"confidence":0.22273003061612448,"empirical_correctness":0.16666666666666666,"signed_gap":0.05606336394945782,"absolute_gap":0.05606336394945782,"min_confidence":0.22067826986312866,"max_confidence":0.22462213039398193},{"index":5,"count":5,"confidence":0.22696382999420167,"empirical_correctness":0.2,"signed_gap":0.02696382999420166,"absolute_gap":0.02696382999420166,"min_confidence":0.22486145794391632,"max_confidence":0.22882990539073944},{"index":6,"count":6,"confidence":0.23344977696736655,"empirical_correctness":0.0,"signed_gap":0.23344977696736655,"absolute_gap":0.23344977696736655,"min_confidence":0.23091723024845123,"max_confidence":0.23603563010692596},{"index":7,"count":5,"confidence":0.24274738132953644,"empirical_correctness":0.0,"signed_gap":0.24274738132953644,"absolute_gap":0.24274738132953644,"min_confidence":0.2383388876914978,"max_confidence":0.24691647291183472},{"index":8,"count":6,"confidence":0.27676602452993393,"empirical_correctness":0.3333333333333333,"signed_gap":-0.056567308803399385,"absolute_gap":0.056567308803399385,"min_confidence":0.2495027333498001,"max_confidence":0.3237422704696655},{"index":9,"count":6,"confidence":0.34806160628795624,"empirical_correctness":1.0,"signed_gap":-0.6519383937120438,"absolute_gap":0.6519383937120438,"min_confidence":0.3306894600391388,"max_confidence":0.3734471797943115}]}},"calibration_singleton_labels":{"equal_width":{"n":56,"bin_count":10,"ece":0.23005756549537182,"signed_mean_gap":0.04196198204798356,"absolute_mean_gap":0.04196198204798356,"bins":[{"index":0,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.0,"upper":0.1,"upper_inclusive":false},{"index":1,"count":7,"confidence":0.19244985708168574,"empirical_correctness":0.14285714285714285,"signed_gap":0.049592714224542894,"absolute_gap":0.049592714224542894,"lower":0.1,"upper":0.2,"upper_inclusive":false},{"index":2,"count":41,"confidence":0.22608288613761343,"empirical_correctness":0.04878048780487805,"signed_gap":0.17730239833273537,"absolute_gap":0.17730239833273537,"lower":0.2,"upper":0.3,"upper_inclusive":false},{"index":3,"count":8,"confidence":0.34166545793414116,"empirical_correctness":1.0,"signed_gap":-0.6583345420658588,"absolute_gap":0.6583345420658588,"lower":0.3,"upper":0.4,"upper_inclusive":false},{"index":4,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.4,"upper":0.5,"upper_inclusive":false},{"index":5,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.5,"upper":0.6,"upper_inclusive":false},{"index":6,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.6,"upper":0.7,"upper_inclusive":false},{"index":7,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.7,"upper":0.8,"upper_inclusive":false},{"index":8,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.8,"upper":0.9,"upper_inclusive":false},{"index":9,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.9,"upper":1.0,"upper_inclusive":true}]},"equal_mass":{"n":56,"bin_count":10,"ece":0.19557775423995086,"signed_mean_gap":0.04196198204798358,"absolute_mean_gap":0.04196198204798358,"bins":[{"index":0,"count":5,"confidence":0.18995851874351502,"empirical_correctness":0.2,"signed_gap":-0.010041481256484991,"absolute_gap":0.010041481256484991,"min_confidence":0.18749237060546875,"max_confidence":0.1946079283952713},{"index":1,"count":6,"confidence":0.2018670936425527,"empirical_correctness":0.0,"signed_gap":0.2018670936425527,"absolute_gap":0.2018670936425527,"min_confidence":0.1982421725988388,"max_confidence":0.20611070096492767},{"index":2,"count":5,"confidence":0.2097841501235962,"empirical_correctness":0.0,"signed_gap":0.2097841501235962,"absolute_gap":0.2097841501235962,"min_confidence":0.20778611302375793,"max_confidence":0.21179406344890594},{"index":3,"count":6,"confidence":0.21755906691153845,"empirical_correctness":0.0,"signed_gap":0.21755906691153845,"absolute_gap":0.21755906691153845,"min_confidence":0.21220877766609192,"max_confidence":0.22060048580169678},{"index":4,"count":6,"confidence":0.22273003061612448,"empirical_correctness":0.16666666666666666,"signed_gap":0.05606336394945782,"absolute_gap":0.05606336394945782,"min_confidence":0.22067826986312866,"max_confidence":0.22462213039398193},{"index":5,"count":5,"confidence":0.22696382999420167,"empirical_correctness":0.2,"signed_gap":0.02696382999420166,"absolute_gap":0.02696382999420166,"min_confidence":0.22486145794391632,"max_confidence":0.22882990539073944},{"index":6,"count":6,"confidence":0.23344977696736655,"empirical_correctness":0.0,"signed_gap":0.23344977696736655,"absolute_gap":0.23344977696736655,"min_confidence":0.23091723024845123,"max_confidence":0.23603563010692596},{"index":7,"count":5,"confidence":0.24274738132953644,"empirical_correctness":0.0,"signed_gap":0.24274738132953644,"absolute_gap":0.24274738132953644,"min_confidence":0.2383388876914978,"max_confidence":0.24691647291183472},{"index":8,"count":6,"confidence":0.27676602452993393,"empirical_correctness":0.3333333333333333,"signed_gap":-0.056567308803399385,"absolute_gap":0.056567308803399385,"min_confidence":0.2495027333498001,"max_confidence":0.3237422704696655},{"index":9,"count":6,"confidence":0.34806160628795624,"empirical_correctness":1.0,"signed_gap":-0.6519383937120438,"absolute_gap":0.6519383937120438,"min_confidence":0.3306894600391388,"max_confidence":0.3734471797943115}]}},"latency_ms":{"n":56,"median":146.39395009726286,"p95":151.54245001031086},"per_action":{"keep":{"n":8,"raw":{"n":8,"acceptable":8,"unsafe":0,"review":0,"automated_count":8,"automated_acceptable":8,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":1.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.6755924351161198,"upper":1.0,"confidence_level":0.95}},"effective":{"n":8,"acceptable":0,"unsafe":0,"review":8,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":8,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.32440756488388023,"confidence_level":0.95}}},"drop":{"n":22,"raw":{"n":22,"acceptable":0,"unsafe":9,"review":12,"automated_count":10,"automated_acceptable":0,"automated_unsafe":9,"unnecessary_reviews":12,"accuracy":0.0,"unsafe_rate":0.4090909090909091,"review_rate":0.5454545454545454,"automated_coverage":0.45454545454545453,"conditional_correctness":0.0,"conditional_unsafe":0.9,"unnecessary_review_rate":0.5454545454545454,"accuracy_wilson_95":{"lower":0.0,"upper":0.14865487460862087,"confidence_level":0.95}},"effective":{"n":22,"acceptable":0,"unsafe":0,"review":22,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":22,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.14865487460862087,"confidence_level":0.95}}},"mask":{"n":8,"raw":{"n":8,"acceptable":0,"unsafe":0,"review":8,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":8,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.32440756488388023,"confidence_level":0.95}},"effective":{"n":8,"acceptable":0,"unsafe":0,"review":8,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":8,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.32440756488388023,"confidence_level":0.95}}},"pseudonymize":{"n":8,"raw":{"n":8,"acceptable":1,"unsafe":2,"review":4,"automated_count":4,"automated_acceptable":1,"automated_unsafe":2,"unnecessary_reviews":4,"accuracy":0.125,"unsafe_rate":0.25,"review_rate":0.5,"automated_coverage":0.5,"conditional_correctness":0.25,"conditional_unsafe":0.5,"unnecessary_review_rate":0.5,"accuracy_wilson_95":{"lower":0.02241749145005667,"upper":0.4708881822128535,"confidence_level":0.95}},"effective":{"n":8,"acceptable":0,"unsafe":0,"review":8,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":8,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.32440756488388023,"confidence_level":0.95}}},"generalize":{"n":8,"raw":{"n":8,"acceptable":0,"unsafe":3,"review":3,"automated_count":5,"automated_acceptable":0,"automated_unsafe":3,"unnecessary_reviews":3,"accuracy":0.0,"unsafe_rate":0.375,"review_rate":0.375,"automated_coverage":0.625,"conditional_correctness":0.0,"conditional_unsafe":0.6,"unnecessary_review_rate":0.375,"accuracy_wilson_95":{"lower":0.0,"upper":0.32440756488388023,"confidence_level":0.95}},"effective":{"n":8,"acceptable":0,"unsafe":0,"review":8,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":8,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.32440756488388023,"confidence_level":0.95}}},"review":{"n":2,"raw":{"n":2,"acceptable":2,"unsafe":0,"review":2,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":1.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.34238022750665303,"upper":1.0,"confidence_level":0.95}},"effective":{"n":2,"acceptable":2,"unsafe":0,"review":2,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":1.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.34238022750665303,"upper":1.0,"confidence_level":0.95}}}}},"challenge56-compact_no_samples":{"n":56,"raw":{"n":56,"acceptable":14,"unsafe":9,"review":9,"automated_count":47,"automated_acceptable":14,"automated_unsafe":9,"unnecessary_reviews":9,"accuracy":0.25,"unsafe_rate":0.16071428571428573,"review_rate":0.16071428571428573,"automated_coverage":0.8392857142857143,"conditional_correctness":0.2978723404255319,"conditional_unsafe":0.19148936170212766,"unnecessary_review_rate":0.16071428571428573,"accuracy_wilson_95":{"lower":0.15517054688270684,"upper":0.376926421476675,"confidence_level":0.95}},"effective":{"n":56,"acceptable":2,"unsafe":0,"review":56,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":54,"accuracy":0.03571428571428571,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":0.9642857142857143,"accuracy_wilson_95":{"lower":0.00984942514823639,"upper":0.12118780180490107,"confidence_level":0.95}},"confusion":{"raw":{"keep":{"keep":8,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0},"drop":{"keep":6,"drop":0,"mask":9,"pseudonymize":0,"generalize":0,"review":7},"mask":{"keep":0,"drop":0,"mask":6,"pseudonymize":0,"generalize":0,"review":2},"pseudonymize":{"keep":0,"drop":0,"mask":8,"pseudonymize":0,"generalize":0,"review":0},"generalize":{"keep":3,"drop":0,"mask":1,"pseudonymize":4,"generalize":0,"review":0},"review":{"keep":0,"drop":0,"mask":2,"pseudonymize":0,"generalize":0,"review":0}},"effective":{"keep":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":8},"drop":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":22},"mask":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":8},"pseudonymize":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":8},"generalize":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":8},"review":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":2}}},"probability_metrics":{"singleton_n":56,"multiacceptable_n":0,"multiacceptable_coverage":0.0,"nll":1.7716307810596474,"nll_zero_mass_n":0,"brier":0.8271449950469831,"acceptable_set_n":56,"acceptable_set_nll":1.7716307810596474,"acceptable_set_zero_mass_n":0},"calibration_all_labels":{"equal_width":{"n":56,"bin_count":10,"ece":0.11878730809049946,"signed_mean_gap":-0.03623649823878493,"absolute_mean_gap":0.03623649823878493,"bins":[{"index":0,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.0,"upper":0.1,"upper_inclusive":false},{"index":1,"count":12,"confidence":0.19261855632066727,"empirical_correctness":0.0,"signed_gap":0.19261855632066727,"absolute_gap":0.19261855632066727,"lower":0.1,"upper":0.2,"upper_inclusive":false},{"index":2,"count":44,"confidence":0.21953030506318266,"empirical_correctness":0.3181818181818182,"signed_gap":-0.09865151311863551,"absolute_gap":0.09865151311863551,"lower":0.2,"upper":0.3,"upper_inclusive":false},{"index":3,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.3,"upper":0.4,"upper_inclusive":false},{"index":4,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.4,"upper":0.5,"upper_inclusive":false},{"index":5,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.5,"upper":0.6,"upper_inclusive":false},{"index":6,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.6,"upper":0.7,"upper_inclusive":false},{"index":7,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.7,"upper":0.8,"upper_inclusive":false},{"index":8,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.8,"upper":0.9,"upper_inclusive":false},{"index":9,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.9,"upper":1.0,"upper_inclusive":true}]},"equal_mass":{"n":56,"bin_count":10,"ece":0.2748877935643707,"signed_mean_gap":-0.03623649823878493,"absolute_mean_gap":0.03623649823878493,"bins":[{"index":0,"count":5,"confidence":0.18862515389919282,"empirical_correctness":0.0,"signed_gap":0.18862515389919282,"absolute_gap":0.18862515389919282,"min_confidence":0.18295522034168243,"max_confidence":0.1922454982995987},{"index":1,"count":6,"confidence":0.19476459672053656,"empirical_correctness":0.0,"signed_gap":0.19476459672053656,"absolute_gap":0.19476459672053656,"min_confidence":0.19344766438007355,"max_confidence":0.19815018773078918},{"index":2,"count":5,"confidence":0.20068852603435516,"empirical_correctness":0.0,"signed_gap":0.20068852603435516,"absolute_gap":0.20068852603435516,"min_confidence":0.19970932602882385,"max_confidence":0.20159274339675903},{"index":3,"count":6,"confidence":0.20440756529569626,"empirical_correctness":0.0,"signed_gap":0.20440756529569626,"absolute_gap":0.20440756529569626,"min_confidence":0.2021399289369583,"max_confidence":0.2066480815410614},{"index":4,"count":6,"confidence":0.21023864050706229,"empirical_correctness":0.0,"signed_gap":0.21023864050706229,"absolute_gap":0.21023864050706229,"min_confidence":0.20670492947101593,"max_confidence":0.21257826685905457},{"index":5,"count":5,"confidence":0.21584061086177825,"empirical_correctness":0.0,"signed_gap":0.21584061086177825,"absolute_gap":0.21584061086177825,"min_confidence":0.21277524530887604,"max_confidence":0.2184010148048401},{"index":6,"count":6,"confidence":0.22121720016002655,"empirical_correctness":0.5,"signed_gap":-0.27878279983997345,"absolute_gap":0.27878279983997345,"min_confidence":0.21869191527366638,"max_confidence":0.22365403175354004},{"index":7,"count":5,"confidence":0.22584662437438965,"empirical_correctness":0.6,"signed_gap":-0.37415337562561035,"absolute_gap":0.37415337562561035,"min_confidence":0.22465522587299347,"max_confidence":0.2275152951478958},{"index":8,"count":6,"confidence":0.23155960192282996,"empirical_correctness":0.5,"signed_gap":-0.26844039807717,"absolute_gap":0.26844039807717,"min_confidence":0.22811222076416016,"max_confidence":0.23532482981681824},{"index":9,"count":6,"confidence":0.24043764919042587,"empirical_correctness":0.8333333333333334,"signed_gap":-0.5928956841429075,"absolute_gap":0.5928956841429075,"min_confidence":0.2358616143465042,"max_confidence":0.24657882750034332}]}},"calibration_singleton_labels":{"equal_width":{"n":56,"bin_count":10,"ece":0.11878730809049946,"signed_mean_gap":-0.03623649823878493,"absolute_mean_gap":0.03623649823878493,"bins":[{"index":0,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.0,"upper":0.1,"upper_inclusive":false},{"index":1,"count":12,"confidence":0.19261855632066727,"empirical_correctness":0.0,"signed_gap":0.19261855632066727,"absolute_gap":0.19261855632066727,"lower":0.1,"upper":0.2,"upper_inclusive":false},{"index":2,"count":44,"confidence":0.21953030506318266,"empirical_correctness":0.3181818181818182,"signed_gap":-0.09865151311863551,"absolute_gap":0.09865151311863551,"lower":0.2,"upper":0.3,"upper_inclusive":false},{"index":3,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.3,"upper":0.4,"upper_inclusive":false},{"index":4,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.4,"upper":0.5,"upper_inclusive":false},{"index":5,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.5,"upper":0.6,"upper_inclusive":false},{"index":6,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.6,"upper":0.7,"upper_inclusive":false},{"index":7,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.7,"upper":0.8,"upper_inclusive":false},{"index":8,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.8,"upper":0.9,"upper_inclusive":false},{"index":9,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.9,"upper":1.0,"upper_inclusive":true}]},"equal_mass":{"n":56,"bin_count":10,"ece":0.2748877935643707,"signed_mean_gap":-0.03623649823878493,"absolute_mean_gap":0.03623649823878493,"bins":[{"index":0,"count":5,"confidence":0.18862515389919282,"empirical_correctness":0.0,"signed_gap":0.18862515389919282,"absolute_gap":0.18862515389919282,"min_confidence":0.18295522034168243,"max_confidence":0.1922454982995987},{"index":1,"count":6,"confidence":0.19476459672053656,"empirical_correctness":0.0,"signed_gap":0.19476459672053656,"absolute_gap":0.19476459672053656,"min_confidence":0.19344766438007355,"max_confidence":0.19815018773078918},{"index":2,"count":5,"confidence":0.20068852603435516,"empirical_correctness":0.0,"signed_gap":0.20068852603435516,"absolute_gap":0.20068852603435516,"min_confidence":0.19970932602882385,"max_confidence":0.20159274339675903},{"index":3,"count":6,"confidence":0.20440756529569626,"empirical_correctness":0.0,"signed_gap":0.20440756529569626,"absolute_gap":0.20440756529569626,"min_confidence":0.2021399289369583,"max_confidence":0.2066480815410614},{"index":4,"count":6,"confidence":0.21023864050706229,"empirical_correctness":0.0,"signed_gap":0.21023864050706229,"absolute_gap":0.21023864050706229,"min_confidence":0.20670492947101593,"max_confidence":0.21257826685905457},{"index":5,"count":5,"confidence":0.21584061086177825,"empirical_correctness":0.0,"signed_gap":0.21584061086177825,"absolute_gap":0.21584061086177825,"min_confidence":0.21277524530887604,"max_confidence":0.2184010148048401},{"index":6,"count":6,"confidence":0.22121720016002655,"empirical_correctness":0.5,"signed_gap":-0.27878279983997345,"absolute_gap":0.27878279983997345,"min_confidence":0.21869191527366638,"max_confidence":0.22365403175354004},{"index":7,"count":5,"confidence":0.22584662437438965,"empirical_correctness":0.6,"signed_gap":-0.37415337562561035,"absolute_gap":0.37415337562561035,"min_confidence":0.22465522587299347,"max_confidence":0.2275152951478958},{"index":8,"count":6,"confidence":0.23155960192282996,"empirical_correctness":0.5,"signed_gap":-0.26844039807717,"absolute_gap":0.26844039807717,"min_confidence":0.22811222076416016,"max_confidence":0.23532482981681824},{"index":9,"count":6,"confidence":0.24043764919042587,"empirical_correctness":0.8333333333333334,"signed_gap":-0.5928956841429075,"absolute_gap":0.5928956841429075,"min_confidence":0.2358616143465042,"max_confidence":0.24657882750034332}]}},"latency_ms":{"n":56,"median":146.44340006634593,"p95":150.78507509315386},"per_action":{"keep":{"n":8,"raw":{"n":8,"acceptable":8,"unsafe":0,"review":0,"automated_count":8,"automated_acceptable":8,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":1.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.6755924351161198,"upper":1.0,"confidence_level":0.95}},"effective":{"n":8,"acceptable":0,"unsafe":0,"review":8,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":8,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.32440756488388023,"confidence_level":0.95}}},"drop":{"n":22,"raw":{"n":22,"acceptable":0,"unsafe":6,"review":7,"automated_count":15,"automated_acceptable":0,"automated_unsafe":6,"unnecessary_reviews":7,"accuracy":0.0,"unsafe_rate":0.2727272727272727,"review_rate":0.3181818181818182,"automated_coverage":0.6818181818181818,"conditional_correctness":0.0,"conditional_unsafe":0.4,"unnecessary_review_rate":0.3181818181818182,"accuracy_wilson_95":{"lower":0.0,"upper":0.14865487460862087,"confidence_level":0.95}},"effective":{"n":22,"acceptable":0,"unsafe":0,"review":22,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":22,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.14865487460862087,"confidence_level":0.95}}},"mask":{"n":8,"raw":{"n":8,"acceptable":6,"unsafe":0,"review":2,"automated_count":6,"automated_acceptable":6,"automated_unsafe":0,"unnecessary_reviews":2,"accuracy":0.75,"unsafe_rate":0.0,"review_rate":0.25,"automated_coverage":0.75,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.25,"accuracy_wilson_95":{"lower":0.40927543031016883,"upper":0.9285207872478909,"confidence_level":0.95}},"effective":{"n":8,"acceptable":0,"unsafe":0,"review":8,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":8,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.32440756488388023,"confidence_level":0.95}}},"pseudonymize":{"n":8,"raw":{"n":8,"acceptable":0,"unsafe":0,"review":0,"automated_count":8,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.32440756488388023,"confidence_level":0.95}},"effective":{"n":8,"acceptable":0,"unsafe":0,"review":8,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":8,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.32440756488388023,"confidence_level":0.95}}},"generalize":{"n":8,"raw":{"n":8,"acceptable":0,"unsafe":3,"review":0,"automated_count":8,"automated_acceptable":0,"automated_unsafe":3,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":0.375,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":0.375,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.32440756488388023,"confidence_level":0.95}},"effective":{"n":8,"acceptable":0,"unsafe":0,"review":8,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":8,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.32440756488388023,"confidence_level":0.95}}},"review":{"n":2,"raw":{"n":2,"acceptable":0,"unsafe":0,"review":0,"automated_count":2,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.6576197724933469,"confidence_level":0.95}},"effective":{"n":2,"acceptable":2,"unsafe":0,"review":2,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":1.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.34238022750665303,"upper":1.0,"confidence_level":0.95}}}}}},"composite":{"n":198,"scope":"descriptive sum across four correlated panels","raw":{"n":198,"acceptable":43,"unsafe":35,"review":80,"automated_count":118,"automated_acceptable":36,"automated_unsafe":35,"unnecessary_reviews":73,"accuracy":0.21717171717171718,"unsafe_rate":0.17676767676767677,"review_rate":0.40404040404040403,"automated_coverage":0.5959595959595959,"conditional_correctness":0.3050847457627119,"conditional_unsafe":0.2966101694915254,"unnecessary_review_rate":0.3686868686868687},"effective":{"n":198,"acceptable":14,"unsafe":0,"review":198,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":184,"accuracy":0.0707070707070707,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":0.9292929292929293}}},{"id":"MoritzLaurer--bge-m3-zeroshot-v2.0","platform":"Local","model_id":"MoritzLaurer/bge-m3-zeroshot-v2.0","revision":"9abf1c8aaeb82a2447809c20753ed0b106b76652","loading":"float32 / none","scoring_kind":"Adapted NLI","settings":{"dtype":"float32","quantization":null,"max_tokens":4096,"effective_max_tokens":512,"temperature":1.0,"scoring_batch_size":1,"trust_remote_code":false,"local_files_only":true,"use_safetensors":true,"guard_threshold":0.45,"seed":0,"tf32":false},"scoring_metadata":{"backend":"nli","protocol":"hf-local-sweep-backends-v1","probability_source":"adapted_entailment_softmax_not_calibrated_sixway","latency_scope":"synchronous_forward_scoring_including_transfer_excluding_tokenization","precision":"float32","equal_mass_convention":"rank bins with floor boundaries at floor(i*n/k), k=min(10,n//2); minimum two rows"},"load_ms":8228.46689983271,"peak_allocated_bytes":2324177920,"peak_reserved_bytes":2342518784,"panels":{"diagnostic43-no_samples":{"n":43,"raw":{"n":43,"acceptable":13,"unsafe":14,"review":0,"automated_count":43,"automated_acceptable":13,"automated_unsafe":14,"unnecessary_reviews":0,"accuracy":0.3023255813953488,"unsafe_rate":0.32558139534883723,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.3023255813953488,"conditional_unsafe":0.32558139534883723,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.1860200013584818,"upper":0.45105364177729607,"confidence_level":0.95}},"effective":{"n":43,"acceptable":5,"unsafe":0,"review":41,"automated_count":2,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":36,"accuracy":0.11627906976744186,"unsafe_rate":0.0,"review_rate":0.9534883720930233,"automated_coverage":0.046511627906976744,"conditional_correctness":0.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.8372093023255814,"accuracy_wilson_95":{"lower":0.05070427739804166,"upper":0.24479161810082128,"confidence_level":0.95}},"confusion":{"raw":{"keep":{"keep":4,"drop":4,"mask":3,"pseudonymize":0,"generalize":1,"review":0},"drop":{"keep":8,"drop":4,"mask":4,"pseudonymize":0,"generalize":0,"review":0},"mask":{"keep":1,"drop":0,"mask":1,"pseudonymize":0,"generalize":0,"review":0},"pseudonymize":{"keep":0,"drop":1,"mask":1,"pseudonymize":0,"generalize":0,"review":0},"generalize":{"keep":4,"drop":0,"mask":2,"pseudonymize":0,"generalize":0,"review":0},"review":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0}},"effective":{"keep":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":12},"drop":{"keep":0,"drop":0,"mask":2,"pseudonymize":0,"generalize":0,"review":14},"mask":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":2},"pseudonymize":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":2},"generalize":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":6},"review":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0}}},"probability_metrics":{"singleton_n":38,"multiacceptable_n":5,"multiacceptable_coverage":0.11627906976744186,"nll":1.7233632170944468,"nll_zero_mass_n":0,"brier":0.8106851661812166,"acceptable_set_n":43,"acceptable_set_nll":1.5996409985019735,"acceptable_set_zero_mass_n":0},"calibration_all_labels":{"equal_width":{"n":43,"bin_count":10,"ece":0.09576676404753397,"signed_mean_gap":-0.09576676404753397,"absolute_mean_gap":0.09576676404753397,"bins":[{"index":0,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.0,"upper":0.1,"upper_inclusive":false},{"index":1,"count":26,"confidence":0.19401217710513335,"empirical_correctness":0.2692307692307692,"signed_gap":-0.07521859212563586,"absolute_gap":0.07521859212563586,"lower":0.1,"upper":0.2,"upper_inclusive":false},{"index":2,"count":17,"confidence":0.22574779654250426,"empirical_correctness":0.35294117647058826,"signed_gap":-0.127193379928084,"absolute_gap":0.127193379928084,"lower":0.2,"upper":0.3,"upper_inclusive":false},{"index":3,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.3,"upper":0.4,"upper_inclusive":false},{"index":4,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.4,"upper":0.5,"upper_inclusive":false},{"index":5,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.5,"upper":0.6,"upper_inclusive":false},{"index":6,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.6,"upper":0.7,"upper_inclusive":false},{"index":7,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.7,"upper":0.8,"upper_inclusive":false},{"index":8,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.8,"upper":0.9,"upper_inclusive":false},{"index":9,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.9,"upper":1.0,"upper_inclusive":true}]},"equal_mass":{"n":43,"bin_count":10,"ece":0.21956678116044334,"signed_mean_gap":-0.09576676404753397,"absolute_mean_gap":0.09576676404753397,"bins":[{"index":0,"count":4,"confidence":0.18674414604902267,"empirical_correctness":0.25,"signed_gap":-0.06325585395097733,"absolute_gap":0.06325585395097733,"min_confidence":0.1858823299407959,"max_confidence":0.1871831864118576},{"index":1,"count":4,"confidence":0.1900048926472664,"empirical_correctness":0.0,"signed_gap":0.1900048926472664,"absolute_gap":0.1900048926472664,"min_confidence":0.18824279308319092,"max_confidence":0.19133463501930237},{"index":2,"count":4,"confidence":0.19295143336057663,"empirical_correctness":0.5,"signed_gap":-0.30704856663942337,"absolute_gap":0.30704856663942337,"min_confidence":0.19183777272701263,"max_confidence":0.19443963468074799},{"index":3,"count":5,"confidence":0.1955597758293152,"empirical_correctness":0.0,"signed_gap":0.1955597758293152,"absolute_gap":0.1955597758293152,"min_confidence":0.1946086436510086,"max_confidence":0.19623172283172607},{"index":4,"count":4,"confidence":0.19764301180839539,"empirical_correctness":0.5,"signed_gap":-0.3023569881916046,"absolute_gap":0.3023569881916046,"min_confidence":0.19686710834503174,"max_confidence":0.1988501250743866},{"index":5,"count":4,"confidence":0.1993352808058262,"empirical_correctness":0.5,"signed_gap":-0.3006647191941738,"absolute_gap":0.3006647191941738,"min_confidence":0.19902606308460236,"max_confidence":0.19978973269462585},{"index":6,"count":5,"confidence":0.20139050781726836,"empirical_correctness":0.2,"signed_gap":0.0013905078172683494,"absolute_gap":0.0013905078172683494,"min_confidence":0.19980266690254211,"max_confidence":0.2045423984527588},{"index":7,"count":4,"confidence":0.20790215954184532,"empirical_correctness":0.75,"signed_gap":-0.5420978404581547,"absolute_gap":0.5420978404581547,"min_confidence":0.20543403923511505,"max_confidence":0.20988649129867554},{"index":8,"count":4,"confidence":0.22923234477639198,"empirical_correctness":0.0,"signed_gap":0.22923234477639198,"absolute_gap":0.22923234477639198,"min_confidence":0.2193794846534729,"max_confidence":0.2421998679637909},{"index":9,"count":5,"confidence":0.2564049303531647,"empirical_correctness":0.4,"signed_gap":-0.14359506964683533,"absolute_gap":0.14359506964683533,"min_confidence":0.2537003457546234,"max_confidence":0.25905776023864746}]}},"calibration_singleton_labels":{"equal_width":{"n":38,"bin_count":10,"ece":0.05441671962800778,"signed_mean_gap":-0.03055032342672348,"absolute_mean_gap":0.03055032342672348,"bins":[{"index":0,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.0,"upper":0.1,"upper_inclusive":false},{"index":1,"count":23,"confidence":0.1936287620793218,"empirical_correctness":0.17391304347826086,"signed_gap":0.019715718601060944,"absolute_gap":0.019715718601060944,"lower":0.1,"upper":0.2,"upper_inclusive":false},{"index":2,"count":15,"confidence":0.22570841213067372,"empirical_correctness":0.3333333333333333,"signed_gap":-0.1076249212026596,"absolute_gap":0.1076249212026596,"lower":0.2,"upper":0.3,"upper_inclusive":false},{"index":3,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.3,"upper":0.4,"upper_inclusive":false},{"index":4,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.4,"upper":0.5,"upper_inclusive":false},{"index":5,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.5,"upper":0.6,"upper_inclusive":false},{"index":6,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.6,"upper":0.7,"upper_inclusive":false},{"index":7,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.7,"upper":0.8,"upper_inclusive":false},{"index":8,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.8,"upper":0.9,"upper_inclusive":false},{"index":9,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.9,"upper":1.0,"upper_inclusive":true}]},"equal_mass":{"n":38,"bin_count":10,"ece":0.16033269896319038,"signed_mean_gap":-0.03055032342672348,"absolute_mean_gap":0.03055032342672348,"bins":[{"index":0,"count":3,"confidence":0.18659779926141104,"empirical_correctness":0.3333333333333333,"signed_gap":-0.14673553407192227,"absolute_gap":0.14673553407192227,"min_confidence":0.1858823299407959,"max_confidence":0.18696166574954987},{"index":1,"count":4,"confidence":0.1889670304954052,"empirical_correctness":0.0,"signed_gap":0.1889670304954052,"absolute_gap":0.1889670304954052,"min_confidence":0.1871831864118576,"max_confidence":0.19128723442554474},{"index":2,"count":4,"confidence":0.1926083043217659,"empirical_correctness":0.25,"signed_gap":-0.0573916956782341,"absolute_gap":0.0573916956782341,"min_confidence":0.19133463501930237,"max_confidence":0.19443963468074799},{"index":3,"count":4,"confidence":0.19539178907871246,"empirical_correctness":0.0,"signed_gap":0.19539178907871246,"absolute_gap":0.19539178907871246,"min_confidence":0.1946086436510086,"max_confidence":0.1959558129310608},{"index":4,"count":4,"confidence":0.19698841124773026,"empirical_correctness":0.25,"signed_gap":-0.053011588752269745,"absolute_gap":0.053011588752269745,"min_confidence":0.19623172283172607,"max_confidence":0.1976429522037506},{"index":5,"count":3,"confidence":0.1993477741877238,"empirical_correctness":0.3333333333333333,"signed_gap":-0.13398555914560953,"absolute_gap":0.13398555914560953,"min_confidence":0.19902606308460236,"max_confidence":0.19978973269462585},{"index":6,"count":4,"confidence":0.20060253515839577,"empirical_correctness":0.25,"signed_gap":-0.04939746484160423,"absolute_gap":0.04939746484160423,"min_confidence":0.19980266690254211,"max_confidence":0.20163831114768982},{"index":7,"count":4,"confidence":0.20656613633036613,"empirical_correctness":0.5,"signed_gap":-0.29343386366963387,"absolute_gap":0.29343386366963387,"min_confidence":0.2045423984527588,"max_confidence":0.208316832780838},{"index":8,"count":4,"confidence":0.2321074642241001,"empirical_correctness":0.0,"signed_gap":0.2321074642241001,"absolute_gap":0.2321074642241001,"min_confidence":0.2193794846534729,"max_confidence":0.2537003457546234},{"index":9,"count":4,"confidence":0.2570810765028,"empirical_correctness":0.5,"signed_gap":-0.2429189234972,"absolute_gap":0.2429189234972,"min_confidence":0.2552732527256012,"max_confidence":0.25905776023864746}]}},"latency_ms":{"n":43,"median":386.9843001011759,"p95":395.84242997225374},"per_action":{"keep":{"n":12,"raw":{"n":12,"acceptable":4,"unsafe":0,"review":0,"automated_count":12,"automated_acceptable":4,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":0.3333333333333333,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.3333333333333333,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.13812009109121315,"upper":0.6093779111272004,"confidence_level":0.95}},"effective":{"n":12,"acceptable":0,"unsafe":0,"review":12,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":12,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":2.7755575615628914e-17,"upper":0.24249400665524085,"confidence_level":0.95}}},"drop":{"n":16,"raw":{"n":16,"acceptable":4,"unsafe":8,"review":0,"automated_count":16,"automated_acceptable":4,"automated_unsafe":8,"unnecessary_reviews":0,"accuracy":0.25,"unsafe_rate":0.5,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.25,"conditional_unsafe":0.5,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.10182067491213048,"upper":0.49498316535508774,"confidence_level":0.95}},"effective":{"n":16,"acceptable":0,"unsafe":0,"review":14,"automated_count":2,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":14,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":0.875,"automated_coverage":0.125,"conditional_correctness":0.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.875,"accuracy_wilson_95":{"lower":0.0,"upper":0.1936076805344365,"confidence_level":0.95}}},"mask":{"n":2,"raw":{"n":2,"acceptable":1,"unsafe":1,"review":0,"automated_count":2,"automated_acceptable":1,"automated_unsafe":1,"unnecessary_reviews":0,"accuracy":0.5,"unsafe_rate":0.5,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.5,"conditional_unsafe":0.5,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.09453120573423074,"upper":0.9054687942657693,"confidence_level":0.95}},"effective":{"n":2,"acceptable":0,"unsafe":0,"review":2,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":2,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.6576197724933469,"confidence_level":0.95}}},"pseudonymize":{"n":2,"raw":{"n":2,"acceptable":0,"unsafe":0,"review":0,"automated_count":2,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.6576197724933469,"confidence_level":0.95}},"effective":{"n":2,"acceptable":0,"unsafe":0,"review":2,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":2,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.6576197724933469,"confidence_level":0.95}}},"generalize":{"n":6,"raw":{"n":6,"acceptable":0,"unsafe":4,"review":0,"automated_count":6,"automated_acceptable":0,"automated_unsafe":4,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":0.6666666666666666,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":0.6666666666666666,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":2.7755575615628914e-17,"upper":0.3903342879021653,"confidence_level":0.95}},"effective":{"n":6,"acceptable":0,"unsafe":0,"review":6,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":6,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":2.7755575615628914e-17,"upper":0.3903342879021653,"confidence_level":0.95}}},"review":{"n":0,"raw":{"n":0,"acceptable":0,"unsafe":0,"review":0,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":null,"unsafe_rate":null,"review_rate":null,"automated_coverage":null,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":null},"effective":{"n":0,"acceptable":0,"unsafe":0,"review":0,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":null,"unsafe_rate":null,"review_rate":null,"automated_coverage":null,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":null}}}},"diagnostic43-compact_no_samples":{"n":43,"raw":{"n":43,"acceptable":17,"unsafe":8,"review":0,"automated_count":43,"automated_acceptable":17,"automated_unsafe":8,"unnecessary_reviews":0,"accuracy":0.3953488372093023,"unsafe_rate":0.18604651162790697,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.3953488372093023,"conditional_unsafe":0.18604651162790697,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.2636531475829744,"upper":0.5442093693712609,"confidence_level":0.95}},"effective":{"n":43,"acceptable":5,"unsafe":0,"review":43,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":38,"accuracy":0.11627906976744186,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":0.8837209302325582,"accuracy_wilson_95":{"lower":0.05070427739804166,"upper":0.24479161810082128,"confidence_level":0.95}},"confusion":{"raw":{"keep":{"keep":2,"drop":8,"mask":0,"pseudonymize":0,"generalize":2,"review":0},"drop":{"keep":4,"drop":10,"mask":0,"pseudonymize":0,"generalize":2,"review":0},"mask":{"keep":0,"drop":2,"mask":0,"pseudonymize":0,"generalize":0,"review":0},"pseudonymize":{"keep":1,"drop":0,"mask":1,"pseudonymize":0,"generalize":0,"review":0},"generalize":{"keep":3,"drop":3,"mask":0,"pseudonymize":0,"generalize":0,"review":0},"review":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0}},"effective":{"keep":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":12},"drop":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":16},"mask":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":2},"pseudonymize":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":2},"generalize":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":6},"review":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0}}},"probability_metrics":{"singleton_n":38,"multiacceptable_n":5,"multiacceptable_coverage":0.11627906976744186,"nll":1.6747958362360362,"nll_zero_mass_n":0,"brier":0.7921333605945329,"acceptable_set_n":43,"acceptable_set_nll":1.5517745144551192,"acceptable_set_zero_mass_n":0},"calibration_all_labels":{"equal_width":{"n":43,"bin_count":10,"ece":0.1805598850167075,"signed_mean_gap":-0.15413503279519636,"absolute_mean_gap":0.15413503279519636,"bins":[{"index":0,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.0,"upper":0.1,"upper_inclusive":false},{"index":1,"count":3,"confidence":0.18937810758749643,"empirical_correctness":0.0,"signed_gap":0.18937810758749643,"absolute_gap":0.18937810758749643,"lower":0.1,"upper":0.2,"upper_inclusive":false},{"index":2,"count":37,"confidence":0.24028309776976303,"empirical_correctness":0.43243243243243246,"signed_gap":-0.19214933466266942,"absolute_gap":0.19214933466266942,"lower":0.2,"upper":0.3,"upper_inclusive":false},{"index":3,"count":3,"confidence":0.30452821652094525,"empirical_correctness":0.3333333333333333,"signed_gap":-0.028805116812388065,"absolute_gap":0.028805116812388065,"lower":0.3,"upper":0.4,"upper_inclusive":false},{"index":4,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.4,"upper":0.5,"upper_inclusive":false},{"index":5,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.5,"upper":0.6,"upper_inclusive":false},{"index":6,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.6,"upper":0.7,"upper_inclusive":false},{"index":7,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.7,"upper":0.8,"upper_inclusive":false},{"index":8,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.8,"upper":0.9,"upper_inclusive":false},{"index":9,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.9,"upper":1.0,"upper_inclusive":true}]},"equal_mass":{"n":43,"bin_count":10,"ece":0.2469533671473348,"signed_mean_gap":-0.15413503279519636,"absolute_mean_gap":0.15413503279519636,"bins":[{"index":0,"count":4,"confidence":0.19224466755986214,"empirical_correctness":0.25,"signed_gap":-0.05775533244013786,"absolute_gap":0.05775533244013786,"min_confidence":0.18784070014953613,"max_confidence":0.20084434747695923},{"index":1,"count":4,"confidence":0.2110418602824211,"empirical_correctness":0.0,"signed_gap":0.2110418602824211,"absolute_gap":0.2110418602824211,"min_confidence":0.2023065984249115,"max_confidence":0.21554698050022125},{"index":2,"count":4,"confidence":0.22239667177200317,"empirical_correctness":0.5,"signed_gap":-0.2776033282279968,"absolute_gap":0.2776033282279968,"min_confidence":0.21906860172748566,"max_confidence":0.2277657687664032},{"index":3,"count":5,"confidence":0.23028534948825835,"empirical_correctness":0.0,"signed_gap":0.23028534948825835,"absolute_gap":0.23028534948825835,"min_confidence":0.22779427468776703,"max_confidence":0.2322407364845276},{"index":4,"count":4,"confidence":0.23706213012337685,"empirical_correctness":0.25,"signed_gap":-0.012937869876623154,"absolute_gap":0.012937869876623154,"min_confidence":0.23621580004692078,"max_confidence":0.23850664496421814},{"index":5,"count":4,"confidence":0.24254443868994713,"empirical_correctness":0.75,"signed_gap":-0.5074555613100529,"absolute_gap":0.5074555613100529,"min_confidence":0.2399386614561081,"max_confidence":0.24420887231826782},{"index":6,"count":5,"confidence":0.2486313372850418,"empirical_correctness":0.4,"signed_gap":-0.15136866271495822,"absolute_gap":0.15136866271495822,"min_confidence":0.24525023996829987,"max_confidence":0.25021645426750183},{"index":7,"count":4,"confidence":0.2523493319749832,"empirical_correctness":0.75,"signed_gap":-0.4976506680250168,"absolute_gap":0.4976506680250168,"min_confidence":0.2514280080795288,"max_confidence":0.25335150957107544},{"index":8,"count":4,"confidence":0.26079530268907547,"empirical_correctness":0.75,"signed_gap":-0.48920469731092453,"absolute_gap":0.48920469731092453,"min_confidence":0.2542838156223297,"max_confidence":0.2762167751789093},{"index":9,"count":5,"confidence":0.3007745087146759,"empirical_correctness":0.4,"signed_gap":-0.0992254912853241,"absolute_gap":0.0992254912853241,"min_confidence":0.29244571924209595,"max_confidence":0.3056949973106384}]}},"calibration_singleton_labels":{"equal_width":{"n":38,"bin_count":10,"ece":0.10369842930843956,"signed_mean_gap":-0.07379662284725591,"absolute_mean_gap":0.07379662284725591,"bins":[{"index":0,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.0,"upper":0.1,"upper_inclusive":false},{"index":1,"count":3,"confidence":0.18937810758749643,"empirical_correctness":0.0,"signed_gap":0.18937810758749643,"absolute_gap":0.18937810758749643,"lower":0.1,"upper":0.2,"upper_inclusive":false},{"index":2,"count":32,"confidence":0.2410627924837172,"empirical_correctness":0.34375,"signed_gap":-0.1026872075162828,"absolute_gap":0.1026872075162828,"lower":0.2,"upper":0.3,"upper_inclusive":false},{"index":3,"count":3,"confidence":0.30452821652094525,"empirical_correctness":0.3333333333333333,"signed_gap":-0.028805116812388065,"absolute_gap":0.028805116812388065,"lower":0.3,"upper":0.4,"upper_inclusive":false},{"index":4,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.4,"upper":0.5,"upper_inclusive":false},{"index":5,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.5,"upper":0.6,"upper_inclusive":false},{"index":6,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.6,"upper":0.7,"upper_inclusive":false},{"index":7,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.7,"upper":0.8,"upper_inclusive":false},{"index":8,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.8,"upper":0.9,"upper_inclusive":false},{"index":9,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.9,"upper":1.0,"upper_inclusive":true}]},"equal_mass":{"n":38,"bin_count":10,"ece":0.19674053003913478,"signed_mean_gap":-0.07379662284725591,"absolute_mean_gap":0.07379662284725591,"bins":[{"index":0,"count":3,"confidence":0.18937810758749643,"empirical_correctness":0.0,"signed_gap":0.18937810758749643,"absolute_gap":0.18937810758749643,"min_confidence":0.18784070014953613,"max_confidence":0.19064876437187195},{"index":1,"count":4,"confidence":0.2110418602824211,"empirical_correctness":0.0,"signed_gap":0.2110418602824211,"absolute_gap":0.2110418602824211,"min_confidence":0.2023065984249115,"max_confidence":0.21554698050022125},{"index":2,"count":4,"confidence":0.22457809001207352,"empirical_correctness":0.25,"signed_gap":-0.025421909987926483,"absolute_gap":0.025421909987926483,"min_confidence":0.2206566333770752,"max_confidence":0.22779427468776703},{"index":3,"count":4,"confidence":0.2309081181883812,"empirical_correctness":0.0,"signed_gap":0.2309081181883812,"absolute_gap":0.2309081181883812,"min_confidence":0.22796449065208435,"max_confidence":0.2322407364845276},{"index":4,"count":4,"confidence":0.23706213012337685,"empirical_correctness":0.25,"signed_gap":-0.012937869876623154,"absolute_gap":0.012937869876623154,"min_confidence":0.23621580004692078,"max_confidence":0.23850664496421814},{"index":5,"count":3,"confidence":0.2419896274805069,"empirical_correctness":0.6666666666666666,"signed_gap":-0.42467703918615973,"absolute_gap":0.42467703918615973,"min_confidence":0.2399386614561081,"max_confidence":0.24353531002998352},{"index":6,"count":4,"confidence":0.2467830367386341,"empirical_correctness":0.25,"signed_gap":-0.0032169632613658905,"absolute_gap":0.0032169632613658905,"min_confidence":0.24420887231826782,"max_confidence":0.2490157037973404},{"index":7,"count":4,"confidence":0.2518405616283417,"empirical_correctness":0.75,"signed_gap":-0.4981594383716583,"absolute_gap":0.4981594383716583,"min_confidence":0.2500169575214386,"max_confidence":0.25335150957107544},{"index":8,"count":4,"confidence":0.27033577859401703,"empirical_correctness":0.5,"signed_gap":-0.22966422140598297,"absolute_gap":0.22966422140598297,"min_confidence":0.25465938448905945,"max_confidence":0.29244571924209595},{"index":9,"count":4,"confidence":0.3028567060828209,"empirical_correctness":0.5,"signed_gap":-0.1971432939171791,"absolute_gap":0.1971432939171791,"min_confidence":0.2978421747684479,"max_confidence":0.3056949973106384}]}},"latency_ms":{"n":43,"median":394.06779990531504,"p95":401.8915599677712},"per_action":{"keep":{"n":12,"raw":{"n":12,"acceptable":2,"unsafe":0,"review":0,"automated_count":12,"automated_acceptable":2,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":0.16666666666666666,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.16666666666666666,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.04696514218385378,"upper":0.4480308622529734,"confidence_level":0.95}},"effective":{"n":12,"acceptable":0,"unsafe":0,"review":12,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":12,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":2.7755575615628914e-17,"upper":0.24249400665524085,"confidence_level":0.95}}},"drop":{"n":16,"raw":{"n":16,"acceptable":10,"unsafe":4,"review":0,"automated_count":16,"automated_acceptable":10,"automated_unsafe":4,"unnecessary_reviews":0,"accuracy":0.625,"unsafe_rate":0.25,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.625,"conditional_unsafe":0.25,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.38641040545502714,"upper":0.8151876744113636,"confidence_level":0.95}},"effective":{"n":16,"acceptable":0,"unsafe":0,"review":16,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":16,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.1936076805344365,"confidence_level":0.95}}},"mask":{"n":2,"raw":{"n":2,"acceptable":0,"unsafe":0,"review":0,"automated_count":2,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.6576197724933469,"confidence_level":0.95}},"effective":{"n":2,"acceptable":0,"unsafe":0,"review":2,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":2,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.6576197724933469,"confidence_level":0.95}}},"pseudonymize":{"n":2,"raw":{"n":2,"acceptable":0,"unsafe":1,"review":0,"automated_count":2,"automated_acceptable":0,"automated_unsafe":1,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":0.5,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":0.5,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.6576197724933469,"confidence_level":0.95}},"effective":{"n":2,"acceptable":0,"unsafe":0,"review":2,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":2,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.6576197724933469,"confidence_level":0.95}}},"generalize":{"n":6,"raw":{"n":6,"acceptable":0,"unsafe":3,"review":0,"automated_count":6,"automated_acceptable":0,"automated_unsafe":3,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":0.5,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":0.5,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":2.7755575615628914e-17,"upper":0.3903342879021653,"confidence_level":0.95}},"effective":{"n":6,"acceptable":0,"unsafe":0,"review":6,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":6,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":2.7755575615628914e-17,"upper":0.3903342879021653,"confidence_level":0.95}}},"review":{"n":0,"raw":{"n":0,"acceptable":0,"unsafe":0,"review":0,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":null,"unsafe_rate":null,"review_rate":null,"automated_coverage":null,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":null},"effective":{"n":0,"acceptable":0,"unsafe":0,"review":0,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":null,"unsafe_rate":null,"review_rate":null,"automated_coverage":null,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":null}}}},"challenge56-no_samples":{"n":56,"raw":{"n":56,"acceptable":22,"unsafe":33,"review":0,"automated_count":56,"automated_acceptable":22,"automated_unsafe":33,"unnecessary_reviews":0,"accuracy":0.39285714285714285,"unsafe_rate":0.5892857142857143,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.39285714285714285,"conditional_unsafe":0.5892857142857143,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.2758041573011051,"upper":0.5236659719957729,"confidence_level":0.95}},"effective":{"n":56,"acceptable":2,"unsafe":0,"review":56,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":54,"accuracy":0.03571428571428571,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":0.9642857142857143,"accuracy_wilson_95":{"lower":0.00984942514823639,"upper":0.12118780180490107,"confidence_level":0.95}},"confusion":{"raw":{"keep":{"keep":7,"drop":0,"mask":0,"pseudonymize":0,"generalize":1,"review":0},"drop":{"keep":21,"drop":0,"mask":0,"pseudonymize":0,"generalize":1,"review":0},"mask":{"keep":1,"drop":0,"mask":7,"pseudonymize":0,"generalize":0,"review":0},"pseudonymize":{"keep":8,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0},"generalize":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":8,"review":0},"review":{"keep":2,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0}},"effective":{"keep":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":8},"drop":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":22},"mask":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":8},"pseudonymize":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":8},"generalize":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":8},"review":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":2}}},"probability_metrics":{"singleton_n":56,"multiacceptable_n":0,"multiacceptable_coverage":0.0,"nll":1.6730769692384013,"nll_zero_mass_n":0,"brier":0.7916652537724619,"acceptable_set_n":56,"acceptable_set_nll":1.6730769692384013,"acceptable_set_zero_mass_n":0},"calibration_all_labels":{"equal_width":{"n":56,"bin_count":10,"ece":0.18186436433877265,"signed_mean_gap":-0.18186436433877265,"absolute_mean_gap":0.18186436433877265,"bins":[{"index":0,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.0,"upper":0.1,"upper_inclusive":false},{"index":1,"count":12,"confidence":0.18956991036732992,"empirical_correctness":0.25,"signed_gap":-0.060430089632670075,"absolute_gap":0.060430089632670075,"lower":0.1,"upper":0.2,"upper_inclusive":false},{"index":2,"count":44,"confidence":0.2168353789231994,"empirical_correctness":0.4318181818181818,"signed_gap":-0.21498280289498242,"absolute_gap":0.21498280289498242,"lower":0.2,"upper":0.3,"upper_inclusive":false},{"index":3,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.3,"upper":0.4,"upper_inclusive":false},{"index":4,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.4,"upper":0.5,"upper_inclusive":false},{"index":5,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.5,"upper":0.6,"upper_inclusive":false},{"index":6,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.6,"upper":0.7,"upper_inclusive":false},{"index":7,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.7,"upper":0.8,"upper_inclusive":false},{"index":8,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.8,"upper":0.9,"upper_inclusive":false},{"index":9,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.9,"upper":1.0,"upper_inclusive":true}]},"equal_mass":{"n":56,"bin_count":10,"ece":0.2637669699532645,"signed_mean_gap":-0.18186436433877265,"absolute_mean_gap":0.18186436433877265,"bins":[{"index":0,"count":5,"confidence":0.184028023481369,"empirical_correctness":0.0,"signed_gap":0.184028023481369,"absolute_gap":0.184028023481369,"min_confidence":0.1821223497390747,"max_confidence":0.1886010318994522},{"index":1,"count":6,"confidence":0.19263708343108496,"empirical_correctness":0.3333333333333333,"signed_gap":-0.14069624990224835,"absolute_gap":0.14069624990224835,"min_confidence":0.18904243409633636,"max_confidence":0.19780631363391876},{"index":2,"count":5,"confidence":0.20173980593681334,"empirical_correctness":0.4,"signed_gap":-0.19826019406318668,"absolute_gap":0.19826019406318668,"min_confidence":0.1988763064146042,"max_confidence":0.20352253317832947},{"index":3,"count":6,"confidence":0.20649103820323944,"empirical_correctness":0.6666666666666666,"signed_gap":-0.4601756284634272,"absolute_gap":0.4601756284634272,"min_confidence":0.20601750910282135,"max_confidence":0.20690464973449707},{"index":4,"count":6,"confidence":0.20938288668791452,"empirical_correctness":0.0,"signed_gap":0.20938288668791452,"absolute_gap":0.20938288668791452,"min_confidence":0.20841509103775024,"max_confidence":0.21098852157592773},{"index":5,"count":5,"confidence":0.21378858387470245,"empirical_correctness":0.4,"signed_gap":-0.18621141612529757,"absolute_gap":0.18621141612529757,"min_confidence":0.21152007579803467,"max_confidence":0.21526463329792023},{"index":6,"count":6,"confidence":0.21768759687741598,"empirical_correctness":0.8333333333333334,"signed_gap":-0.6156457364559174,"absolute_gap":0.6156457364559174,"min_confidence":0.21535137295722961,"max_confidence":0.22062475979328156},{"index":7,"count":5,"confidence":0.22336710393428802,"empirical_correctness":0.2,"signed_gap":0.023367103934288008,"absolute_gap":0.023367103934288008,"min_confidence":0.2218829244375229,"max_confidence":0.22426995635032654},{"index":8,"count":6,"confidence":0.22608056416114172,"empirical_correctness":0.5,"signed_gap":-0.27391943583885825,"absolute_gap":0.27391943583885825,"min_confidence":0.22438275814056396,"max_confidence":0.22784920036792755},{"index":9,"count":6,"confidence":0.23121716578801474,"empirical_correctness":0.5,"signed_gap":-0.26878283421198523,"absolute_gap":0.26878283421198523,"min_confidence":0.22861650586128235,"max_confidence":0.2342076599597931}]}},"calibration_singleton_labels":{"equal_width":{"n":56,"bin_count":10,"ece":0.18186436433877265,"signed_mean_gap":-0.18186436433877265,"absolute_mean_gap":0.18186436433877265,"bins":[{"index":0,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.0,"upper":0.1,"upper_inclusive":false},{"index":1,"count":12,"confidence":0.18956991036732992,"empirical_correctness":0.25,"signed_gap":-0.060430089632670075,"absolute_gap":0.060430089632670075,"lower":0.1,"upper":0.2,"upper_inclusive":false},{"index":2,"count":44,"confidence":0.2168353789231994,"empirical_correctness":0.4318181818181818,"signed_gap":-0.21498280289498242,"absolute_gap":0.21498280289498242,"lower":0.2,"upper":0.3,"upper_inclusive":false},{"index":3,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.3,"upper":0.4,"upper_inclusive":false},{"index":4,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.4,"upper":0.5,"upper_inclusive":false},{"index":5,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.5,"upper":0.6,"upper_inclusive":false},{"index":6,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.6,"upper":0.7,"upper_inclusive":false},{"index":7,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.7,"upper":0.8,"upper_inclusive":false},{"index":8,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.8,"upper":0.9,"upper_inclusive":false},{"index":9,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.9,"upper":1.0,"upper_inclusive":true}]},"equal_mass":{"n":56,"bin_count":10,"ece":0.2637669699532645,"signed_mean_gap":-0.18186436433877265,"absolute_mean_gap":0.18186436433877265,"bins":[{"index":0,"count":5,"confidence":0.184028023481369,"empirical_correctness":0.0,"signed_gap":0.184028023481369,"absolute_gap":0.184028023481369,"min_confidence":0.1821223497390747,"max_confidence":0.1886010318994522},{"index":1,"count":6,"confidence":0.19263708343108496,"empirical_correctness":0.3333333333333333,"signed_gap":-0.14069624990224835,"absolute_gap":0.14069624990224835,"min_confidence":0.18904243409633636,"max_confidence":0.19780631363391876},{"index":2,"count":5,"confidence":0.20173980593681334,"empirical_correctness":0.4,"signed_gap":-0.19826019406318668,"absolute_gap":0.19826019406318668,"min_confidence":0.1988763064146042,"max_confidence":0.20352253317832947},{"index":3,"count":6,"confidence":0.20649103820323944,"empirical_correctness":0.6666666666666666,"signed_gap":-0.4601756284634272,"absolute_gap":0.4601756284634272,"min_confidence":0.20601750910282135,"max_confidence":0.20690464973449707},{"index":4,"count":6,"confidence":0.20938288668791452,"empirical_correctness":0.0,"signed_gap":0.20938288668791452,"absolute_gap":0.20938288668791452,"min_confidence":0.20841509103775024,"max_confidence":0.21098852157592773},{"index":5,"count":5,"confidence":0.21378858387470245,"empirical_correctness":0.4,"signed_gap":-0.18621141612529757,"absolute_gap":0.18621141612529757,"min_confidence":0.21152007579803467,"max_confidence":0.21526463329792023},{"index":6,"count":6,"confidence":0.21768759687741598,"empirical_correctness":0.8333333333333334,"signed_gap":-0.6156457364559174,"absolute_gap":0.6156457364559174,"min_confidence":0.21535137295722961,"max_confidence":0.22062475979328156},{"index":7,"count":5,"confidence":0.22336710393428802,"empirical_correctness":0.2,"signed_gap":0.023367103934288008,"absolute_gap":0.023367103934288008,"min_confidence":0.2218829244375229,"max_confidence":0.22426995635032654},{"index":8,"count":6,"confidence":0.22608056416114172,"empirical_correctness":0.5,"signed_gap":-0.27391943583885825,"absolute_gap":0.27391943583885825,"min_confidence":0.22438275814056396,"max_confidence":0.22784920036792755},{"index":9,"count":6,"confidence":0.23121716578801474,"empirical_correctness":0.5,"signed_gap":-0.26878283421198523,"absolute_gap":0.26878283421198523,"min_confidence":0.22861650586128235,"max_confidence":0.2342076599597931}]}},"latency_ms":{"n":56,"median":340.13789996970445,"p95":350.7349999854341},"per_action":{"keep":{"n":8,"raw":{"n":8,"acceptable":7,"unsafe":0,"review":0,"automated_count":8,"automated_acceptable":7,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":0.875,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.875,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.5291118177871464,"upper":0.9775825085499433,"confidence_level":0.95}},"effective":{"n":8,"acceptable":0,"unsafe":0,"review":8,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":8,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.32440756488388023,"confidence_level":0.95}}},"drop":{"n":22,"raw":{"n":22,"acceptable":0,"unsafe":22,"review":0,"automated_count":22,"automated_acceptable":0,"automated_unsafe":22,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":1.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":1.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.14865487460862087,"confidence_level":0.95}},"effective":{"n":22,"acceptable":0,"unsafe":0,"review":22,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":22,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.14865487460862087,"confidence_level":0.95}}},"mask":{"n":8,"raw":{"n":8,"acceptable":7,"unsafe":1,"review":0,"automated_count":8,"automated_acceptable":7,"automated_unsafe":1,"unnecessary_reviews":0,"accuracy":0.875,"unsafe_rate":0.125,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.875,"conditional_unsafe":0.125,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.5291118177871464,"upper":0.9775825085499433,"confidence_level":0.95}},"effective":{"n":8,"acceptable":0,"unsafe":0,"review":8,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":8,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.32440756488388023,"confidence_level":0.95}}},"pseudonymize":{"n":8,"raw":{"n":8,"acceptable":0,"unsafe":8,"review":0,"automated_count":8,"automated_acceptable":0,"automated_unsafe":8,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":1.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":1.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.32440756488388023,"confidence_level":0.95}},"effective":{"n":8,"acceptable":0,"unsafe":0,"review":8,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":8,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.32440756488388023,"confidence_level":0.95}}},"generalize":{"n":8,"raw":{"n":8,"acceptable":8,"unsafe":0,"review":0,"automated_count":8,"automated_acceptable":8,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":1.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.6755924351161198,"upper":1.0,"confidence_level":0.95}},"effective":{"n":8,"acceptable":0,"unsafe":0,"review":8,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":8,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.32440756488388023,"confidence_level":0.95}}},"review":{"n":2,"raw":{"n":2,"acceptable":0,"unsafe":2,"review":0,"automated_count":2,"automated_acceptable":0,"automated_unsafe":2,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":1.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":1.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.6576197724933469,"confidence_level":0.95}},"effective":{"n":2,"acceptable":2,"unsafe":0,"review":2,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":1.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.34238022750665303,"upper":1.0,"confidence_level":0.95}}}}},"challenge56-compact_no_samples":{"n":56,"raw":{"n":56,"acceptable":10,"unsafe":33,"review":0,"automated_count":56,"automated_acceptable":10,"automated_unsafe":33,"unnecessary_reviews":0,"accuracy":0.17857142857142858,"unsafe_rate":0.5892857142857143,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.17857142857142858,"conditional_unsafe":0.5892857142857143,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.09999856752412305,"upper":0.2984118203665106,"confidence_level":0.95}},"effective":{"n":56,"acceptable":2,"unsafe":0,"review":56,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":54,"accuracy":0.03571428571428571,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":0.9642857142857143,"accuracy_wilson_95":{"lower":0.00984942514823639,"upper":0.12118780180490107,"confidence_level":0.95}},"confusion":{"raw":{"keep":{"keep":4,"drop":0,"mask":0,"pseudonymize":0,"generalize":4,"review":0},"drop":{"keep":16,"drop":1,"mask":0,"pseudonymize":0,"generalize":5,"review":0},"mask":{"keep":5,"drop":2,"mask":0,"pseudonymize":0,"generalize":1,"review":0},"pseudonymize":{"keep":7,"drop":0,"mask":0,"pseudonymize":0,"generalize":1,"review":0},"generalize":{"keep":3,"drop":0,"mask":0,"pseudonymize":0,"generalize":5,"review":0},"review":{"keep":1,"drop":0,"mask":0,"pseudonymize":0,"generalize":1,"review":0}},"effective":{"keep":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":8},"drop":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":22},"mask":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":8},"pseudonymize":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":8},"generalize":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":8},"review":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":2}}},"probability_metrics":{"singleton_n":56,"multiacceptable_n":0,"multiacceptable_coverage":0.0,"nll":1.7199812441851428,"nll_zero_mass_n":0,"brier":0.810124161788184,"acceptable_set_n":56,"acceptable_set_nll":1.7199812441851428,"acceptable_set_zero_mass_n":0},"calibration_all_labels":{"equal_width":{"n":56,"bin_count":10,"ece":0.05972321783857686,"signed_mean_gap":0.02745912836066314,"absolute_mean_gap":0.02745912836066314,"bins":[{"index":0,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.0,"upper":0.1,"upper_inclusive":false},{"index":1,"count":18,"confidence":0.1911725385321511,"empirical_correctness":0.05555555555555555,"signed_gap":0.13561698297659555,"absolute_gap":0.13561698297659555,"lower":0.1,"upper":0.2,"upper_inclusive":false},{"index":2,"count":38,"confidence":0.21306856564785304,"empirical_correctness":0.23684210526315788,"signed_gap":-0.023773539615304845,"absolute_gap":0.023773539615304845,"lower":0.2,"upper":0.3,"upper_inclusive":false},{"index":3,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.3,"upper":0.4,"upper_inclusive":false},{"index":4,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.4,"upper":0.5,"upper_inclusive":false},{"index":5,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.5,"upper":0.6,"upper_inclusive":false},{"index":6,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.6,"upper":0.7,"upper_inclusive":false},{"index":7,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.7,"upper":0.8,"upper_inclusive":false},{"index":8,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.8,"upper":0.9,"upper_inclusive":false},{"index":9,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.9,"upper":1.0,"upper_inclusive":true}]},"equal_mass":{"n":56,"bin_count":10,"ece":0.11202905513346195,"signed_mean_gap":0.027459128360663144,"absolute_mean_gap":0.027459128360663144,"bins":[{"index":0,"count":5,"confidence":0.18454301059246064,"empirical_correctness":0.0,"signed_gap":0.18454301059246064,"absolute_gap":0.18454301059246064,"min_confidence":0.1791381537914276,"max_confidence":0.1878383308649063},{"index":1,"count":6,"confidence":0.1910146102309227,"empirical_correctness":0.16666666666666666,"signed_gap":0.02434794356425604,"absolute_gap":0.02434794356425604,"min_confidence":0.18924683332443237,"max_confidence":0.19373533129692078},{"index":2,"count":5,"confidence":0.19502939581871032,"empirical_correctness":0.0,"signed_gap":0.19502939581871032,"absolute_gap":0.19502939581871032,"min_confidence":0.19375988841056824,"max_confidence":0.19649748504161835},{"index":3,"count":6,"confidence":0.20017449806133905,"empirical_correctness":0.16666666666666666,"signed_gap":0.033507831394672394,"absolute_gap":0.033507831394672394,"min_confidence":0.197170227766037,"max_confidence":0.20143993198871613},{"index":4,"count":6,"confidence":0.20347257206837335,"empirical_correctness":0.0,"signed_gap":0.20347257206837335,"absolute_gap":0.20347257206837335,"min_confidence":0.2015257626771927,"max_confidence":0.20496609807014465},{"index":5,"count":5,"confidence":0.205690199136734,"empirical_correctness":0.2,"signed_gap":0.005690199136733992,"absolute_gap":0.005690199136733992,"min_confidence":0.2050037980079651,"max_confidence":0.20641236007213593},{"index":6,"count":6,"confidence":0.208498053252697,"empirical_correctness":0.5,"signed_gap":-0.291501946747303,"absolute_gap":0.291501946747303,"min_confidence":0.20672443509101868,"max_confidence":0.21191446483135223},{"index":7,"count":5,"confidence":0.216013503074646,"empirical_correctness":0.2,"signed_gap":0.016013503074645996,"absolute_gap":0.016013503074645996,"min_confidence":0.21408453583717346,"max_confidence":0.217263862490654},{"index":8,"count":6,"confidence":0.22188641875982285,"empirical_correctness":0.16666666666666666,"signed_gap":0.05521975209315619,"absolute_gap":0.05521975209315619,"min_confidence":0.21755936741828918,"max_confidence":0.2254762351512909},{"index":9,"count":6,"confidence":0.23017562180757523,"empirical_correctness":0.3333333333333333,"signed_gap":-0.10315771152575809,"absolute_gap":0.10315771152575809,"min_confidence":0.2262996882200241,"max_confidence":0.24009551107883453}]}},"calibration_singleton_labels":{"equal_width":{"n":56,"bin_count":10,"ece":0.05972321783857686,"signed_mean_gap":0.02745912836066314,"absolute_mean_gap":0.02745912836066314,"bins":[{"index":0,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.0,"upper":0.1,"upper_inclusive":false},{"index":1,"count":18,"confidence":0.1911725385321511,"empirical_correctness":0.05555555555555555,"signed_gap":0.13561698297659555,"absolute_gap":0.13561698297659555,"lower":0.1,"upper":0.2,"upper_inclusive":false},{"index":2,"count":38,"confidence":0.21306856564785304,"empirical_correctness":0.23684210526315788,"signed_gap":-0.023773539615304845,"absolute_gap":0.023773539615304845,"lower":0.2,"upper":0.3,"upper_inclusive":false},{"index":3,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.3,"upper":0.4,"upper_inclusive":false},{"index":4,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.4,"upper":0.5,"upper_inclusive":false},{"index":5,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.5,"upper":0.6,"upper_inclusive":false},{"index":6,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.6,"upper":0.7,"upper_inclusive":false},{"index":7,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.7,"upper":0.8,"upper_inclusive":false},{"index":8,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.8,"upper":0.9,"upper_inclusive":false},{"index":9,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.9,"upper":1.0,"upper_inclusive":true}]},"equal_mass":{"n":56,"bin_count":10,"ece":0.11202905513346195,"signed_mean_gap":0.027459128360663144,"absolute_mean_gap":0.027459128360663144,"bins":[{"index":0,"count":5,"confidence":0.18454301059246064,"empirical_correctness":0.0,"signed_gap":0.18454301059246064,"absolute_gap":0.18454301059246064,"min_confidence":0.1791381537914276,"max_confidence":0.1878383308649063},{"index":1,"count":6,"confidence":0.1910146102309227,"empirical_correctness":0.16666666666666666,"signed_gap":0.02434794356425604,"absolute_gap":0.02434794356425604,"min_confidence":0.18924683332443237,"max_confidence":0.19373533129692078},{"index":2,"count":5,"confidence":0.19502939581871032,"empirical_correctness":0.0,"signed_gap":0.19502939581871032,"absolute_gap":0.19502939581871032,"min_confidence":0.19375988841056824,"max_confidence":0.19649748504161835},{"index":3,"count":6,"confidence":0.20017449806133905,"empirical_correctness":0.16666666666666666,"signed_gap":0.033507831394672394,"absolute_gap":0.033507831394672394,"min_confidence":0.197170227766037,"max_confidence":0.20143993198871613},{"index":4,"count":6,"confidence":0.20347257206837335,"empirical_correctness":0.0,"signed_gap":0.20347257206837335,"absolute_gap":0.20347257206837335,"min_confidence":0.2015257626771927,"max_confidence":0.20496609807014465},{"index":5,"count":5,"confidence":0.205690199136734,"empirical_correctness":0.2,"signed_gap":0.005690199136733992,"absolute_gap":0.005690199136733992,"min_confidence":0.2050037980079651,"max_confidence":0.20641236007213593},{"index":6,"count":6,"confidence":0.208498053252697,"empirical_correctness":0.5,"signed_gap":-0.291501946747303,"absolute_gap":0.291501946747303,"min_confidence":0.20672443509101868,"max_confidence":0.21191446483135223},{"index":7,"count":5,"confidence":0.216013503074646,"empirical_correctness":0.2,"signed_gap":0.016013503074645996,"absolute_gap":0.016013503074645996,"min_confidence":0.21408453583717346,"max_confidence":0.217263862490654},{"index":8,"count":6,"confidence":0.22188641875982285,"empirical_correctness":0.16666666666666666,"signed_gap":0.05521975209315619,"absolute_gap":0.05521975209315619,"min_confidence":0.21755936741828918,"max_confidence":0.2254762351512909},{"index":9,"count":6,"confidence":0.23017562180757523,"empirical_correctness":0.3333333333333333,"signed_gap":-0.10315771152575809,"absolute_gap":0.10315771152575809,"min_confidence":0.2262996882200241,"max_confidence":0.24009551107883453}]}},"latency_ms":{"n":56,"median":343.8922498608008,"p95":364.15522487368435},"per_action":{"keep":{"n":8,"raw":{"n":8,"acceptable":4,"unsafe":0,"review":0,"automated_count":8,"automated_acceptable":4,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":0.5,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.5,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.21521606221387757,"upper":0.7847839377861224,"confidence_level":0.95}},"effective":{"n":8,"acceptable":0,"unsafe":0,"review":8,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":8,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.32440756488388023,"confidence_level":0.95}}},"drop":{"n":22,"raw":{"n":22,"acceptable":1,"unsafe":16,"review":0,"automated_count":22,"automated_acceptable":1,"automated_unsafe":16,"unnecessary_reviews":0,"accuracy":0.045454545454545456,"unsafe_rate":0.7272727272727273,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.045454545454545456,"conditional_unsafe":0.7272727272727273,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.00806942762126385,"upper":0.21798045838657332,"confidence_level":0.95}},"effective":{"n":22,"acceptable":0,"unsafe":0,"review":22,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":22,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.14865487460862087,"confidence_level":0.95}}},"mask":{"n":8,"raw":{"n":8,"acceptable":0,"unsafe":5,"review":0,"automated_count":8,"automated_acceptable":0,"automated_unsafe":5,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":0.625,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":0.625,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.32440756488388023,"confidence_level":0.95}},"effective":{"n":8,"acceptable":0,"unsafe":0,"review":8,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":8,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.32440756488388023,"confidence_level":0.95}}},"pseudonymize":{"n":8,"raw":{"n":8,"acceptable":0,"unsafe":7,"review":0,"automated_count":8,"automated_acceptable":0,"automated_unsafe":7,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":0.875,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":0.875,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.32440756488388023,"confidence_level":0.95}},"effective":{"n":8,"acceptable":0,"unsafe":0,"review":8,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":8,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.32440756488388023,"confidence_level":0.95}}},"generalize":{"n":8,"raw":{"n":8,"acceptable":5,"unsafe":3,"review":0,"automated_count":8,"automated_acceptable":5,"automated_unsafe":3,"unnecessary_reviews":0,"accuracy":0.625,"unsafe_rate":0.375,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.625,"conditional_unsafe":0.375,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.30574239460262737,"upper":0.8631557141764026,"confidence_level":0.95}},"effective":{"n":8,"acceptable":0,"unsafe":0,"review":8,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":8,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.32440756488388023,"confidence_level":0.95}}},"review":{"n":2,"raw":{"n":2,"acceptable":0,"unsafe":2,"review":0,"automated_count":2,"automated_acceptable":0,"automated_unsafe":2,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":1.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":1.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.6576197724933469,"confidence_level":0.95}},"effective":{"n":2,"acceptable":2,"unsafe":0,"review":2,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":1.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.34238022750665303,"upper":1.0,"confidence_level":0.95}}}}}},"composite":{"n":198,"scope":"descriptive sum across four correlated panels","raw":{"n":198,"acceptable":62,"unsafe":88,"review":0,"automated_count":198,"automated_acceptable":62,"automated_unsafe":88,"unnecessary_reviews":0,"accuracy":0.31313131313131315,"unsafe_rate":0.4444444444444444,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.31313131313131315,"conditional_unsafe":0.4444444444444444,"unnecessary_review_rate":0.0},"effective":{"n":198,"acceptable":14,"unsafe":0,"review":196,"automated_count":2,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":182,"accuracy":0.0707070707070707,"unsafe_rate":0.0,"review_rate":0.98989898989899,"automated_coverage":0.010101010101010102,"conditional_correctness":0.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.9191919191919192}}},{"id":"MoritzLaurer--roberta-base-zeroshot-v2.0-c","platform":"Local","model_id":"MoritzLaurer/roberta-base-zeroshot-v2.0-c","revision":"d825e740e0c59881cf0b0b1481ccf726b6d65341","loading":"float32 / none","scoring_kind":"Adapted NLI","settings":{"dtype":"float32","quantization":null,"max_tokens":4096,"effective_max_tokens":512,"temperature":1.0,"scoring_batch_size":1,"trust_remote_code":false,"local_files_only":true,"use_safetensors":true,"guard_threshold":0.45,"seed":0,"tf32":false},"scoring_metadata":{"backend":"nli","protocol":"hf-local-sweep-backends-v1","probability_source":"adapted_entailment_softmax_not_calibrated_sixway","latency_scope":"synchronous_forward_scoring_including_transfer_excluding_tokenization","precision":"float32","equal_mass_convention":"rank bins with floor boundaries at floor(i*n/k), k=min(10,n//2); minimum two rows"},"load_ms":5641.1250999663025,"peak_allocated_bytes":550462464,"peak_reserved_bytes":595591168,"panels":{"diagnostic43-no_samples":{"n":43,"raw":{"n":43,"acceptable":12,"unsafe":31,"review":0,"automated_count":43,"automated_acceptable":12,"automated_unsafe":31,"unnecessary_reviews":0,"accuracy":0.27906976744186046,"unsafe_rate":0.7209302325581395,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.27906976744186046,"conditional_unsafe":0.7209302325581395,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.16746613786237488,"upper":0.42691028681878873,"confidence_level":0.95}},"effective":{"n":43,"acceptable":6,"unsafe":0,"review":38,"automated_count":5,"automated_acceptable":1,"automated_unsafe":0,"unnecessary_reviews":33,"accuracy":0.13953488372093023,"unsafe_rate":0.0,"review_rate":0.8837209302325582,"automated_coverage":0.11627906976744186,"conditional_correctness":0.2,"conditional_unsafe":0.0,"unnecessary_review_rate":0.7674418604651163,"accuracy_wilson_95":{"lower":0.06555723427964878,"upper":0.2726358796738286,"confidence_level":0.95}},"confusion":{"raw":{"keep":{"keep":12,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0},"drop":{"keep":16,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0},"mask":{"keep":2,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0},"pseudonymize":{"keep":2,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0},"generalize":{"keep":6,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0},"review":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0}},"effective":{"keep":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":12},"drop":{"keep":0,"drop":0,"mask":4,"pseudonymize":0,"generalize":0,"review":12},"mask":{"keep":0,"drop":0,"mask":1,"pseudonymize":0,"generalize":0,"review":1},"pseudonymize":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":2},"generalize":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":6},"review":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0}}},"probability_metrics":{"singleton_n":38,"multiacceptable_n":5,"multiacceptable_coverage":0.11627906976744186,"nll":1.6975052102505805,"nll_zero_mass_n":0,"brier":0.7996476939910291,"acceptable_set_n":43,"acceptable_set_nll":1.5927878033603493,"acceptable_set_zero_mass_n":0},"calibration_all_labels":{"equal_width":{"n":43,"bin_count":10,"ece":0.015535410406977623,"signed_mean_gap":0.002035749166510839,"absolute_mean_gap":0.002035749166510839,"bins":[{"index":0,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.0,"upper":0.1,"upper_inclusive":false},{"index":1,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.1,"upper":0.2,"upper_inclusive":false},{"index":2,"count":36,"confidence":0.269715480092499,"empirical_correctness":0.2777777777777778,"signed_gap":-0.008062297685278774,"absolute_gap":0.008062297685278774,"lower":0.2,"upper":0.3,"upper_inclusive":false},{"index":3,"count":7,"confidence":0.33968284726142883,"empirical_correctness":0.2857142857142857,"signed_gap":0.053968561547143135,"absolute_gap":0.053968561547143135,"lower":0.3,"upper":0.4,"upper_inclusive":false},{"index":4,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.4,"upper":0.5,"upper_inclusive":false},{"index":5,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.5,"upper":0.6,"upper_inclusive":false},{"index":6,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.6,"upper":0.7,"upper_inclusive":false},{"index":7,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.7,"upper":0.8,"upper_inclusive":false},{"index":8,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.8,"upper":0.9,"upper_inclusive":false},{"index":9,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.9,"upper":1.0,"upper_inclusive":true}]},"equal_mass":{"n":43,"bin_count":10,"ece":0.13565495229044625,"signed_mean_gap":0.002035749166510822,"absolute_mean_gap":0.002035749166510822,"bins":[{"index":0,"count":4,"confidence":0.24657363072037697,"empirical_correctness":0.0,"signed_gap":0.24657363072037697,"absolute_gap":0.24657363072037697,"min_confidence":0.243426114320755,"max_confidence":0.24842913448810577},{"index":1,"count":4,"confidence":0.2562430277466774,"empirical_correctness":0.25,"signed_gap":0.006243027746677399,"absolute_gap":0.006243027746677399,"min_confidence":0.2508445680141449,"max_confidence":0.2616894841194153},{"index":2,"count":4,"confidence":0.26398365944623947,"empirical_correctness":0.25,"signed_gap":0.013983659446239471,"absolute_gap":0.013983659446239471,"min_confidence":0.2631165683269501,"max_confidence":0.2644321918487549},{"index":3,"count":5,"confidence":0.2664669811725616,"empirical_correctness":0.6,"signed_gap":-0.33353301882743835,"absolute_gap":0.33353301882743835,"min_confidence":0.26469123363494873,"max_confidence":0.26940757036209106},{"index":4,"count":4,"confidence":0.2704498618841171,"empirical_correctness":0.5,"signed_gap":-0.22955013811588287,"absolute_gap":0.22955013811588287,"min_confidence":0.2696212828159332,"max_confidence":0.2718530297279358},{"index":5,"count":4,"confidence":0.27253276109695435,"empirical_correctness":0.25,"signed_gap":0.022532761096954346,"absolute_gap":0.022532761096954346,"min_confidence":0.27192890644073486,"max_confidence":0.2737933099269867},{"index":6,"count":5,"confidence":0.2768854320049286,"empirical_correctness":0.2,"signed_gap":0.07688543200492859,"absolute_gap":0.07688543200492859,"min_confidence":0.2738640010356903,"max_confidence":0.28171417117118835},{"index":7,"count":4,"confidence":0.28969675302505493,"empirical_correctness":0.0,"signed_gap":0.28969675302505493,"absolute_gap":0.28969675302505493,"min_confidence":0.2839583456516266,"max_confidence":0.2972065806388855},{"index":8,"count":4,"confidence":0.3149508982896805,"empirical_correctness":0.25,"signed_gap":0.06495089828968048,"absolute_gap":0.06495089828968048,"min_confidence":0.29720932245254517,"max_confidence":0.3363534212112427},{"index":9,"count":5,"confidence":0.3426105558872223,"empirical_correctness":0.4,"signed_gap":-0.05738944411277774,"absolute_gap":0.05738944411277774,"min_confidence":0.33891603350639343,"max_confidence":0.3465132713317871}]}},"calibration_singleton_labels":{"equal_width":{"n":38,"bin_count":10,"ece":0.052418221376444155,"signed_mean_gap":-0.03253506712223353,"absolute_mean_gap":0.03253506712223353,"bins":[{"index":0,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.0,"upper":0.1,"upper_inclusive":false},{"index":1,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.1,"upper":0.2,"upper_inclusive":false},{"index":2,"count":31,"confidence":0.27051250059758464,"empirical_correctness":0.3225806451612903,"signed_gap":-0.052068144563705676,"absolute_gap":0.052068144563705676,"lower":0.2,"upper":0.3,"upper_inclusive":false},{"index":3,"count":7,"confidence":0.33968284726142883,"empirical_correctness":0.2857142857142857,"signed_gap":0.053968561547143135,"absolute_gap":0.053968561547143135,"lower":0.3,"upper":0.4,"upper_inclusive":false},{"index":4,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.4,"upper":0.5,"upper_inclusive":false},{"index":5,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.5,"upper":0.6,"upper_inclusive":false},{"index":6,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.6,"upper":0.7,"upper_inclusive":false},{"index":7,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.7,"upper":0.8,"upper_inclusive":false},{"index":8,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.8,"upper":0.9,"upper_inclusive":false},{"index":9,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.9,"upper":1.0,"upper_inclusive":true}]},"equal_mass":{"n":38,"bin_count":10,"ece":0.19927669316530228,"signed_mean_gap":-0.032535067122233544,"absolute_mean_gap":0.032535067122233544,"bins":[{"index":0,"count":3,"confidence":0.2476228028535843,"empirical_correctness":0.0,"signed_gap":0.2476228028535843,"absolute_gap":0.2476228028535843,"min_confidence":0.24645952880382538,"max_confidence":0.24842913448810577},{"index":1,"count":4,"confidence":0.2565997987985611,"empirical_correctness":0.25,"signed_gap":0.006599798798561096,"absolute_gap":0.006599798798561096,"min_confidence":0.2508445680141449,"max_confidence":0.2631165683269501},{"index":2,"count":4,"confidence":0.2646310552954674,"empirical_correctness":0.75,"signed_gap":-0.4853689447045326,"absolute_gap":0.4853689447045326,"min_confidence":0.2639983594417572,"max_confidence":0.2654024362564087},{"index":3,"count":4,"confidence":0.26922303438186646,"empirical_correctness":0.5,"signed_gap":-0.23077696561813354,"absolute_gap":0.23077696561813354,"min_confidence":0.26783502101898193,"max_confidence":0.2700282633304596},{"index":4,"count":4,"confidence":0.27150586247444153,"empirical_correctness":0.5,"signed_gap":-0.22849413752555847,"absolute_gap":0.22849413752555847,"min_confidence":0.2702968716621399,"max_confidence":0.27194464206695557},{"index":5,"count":3,"confidence":0.2733738323052724,"empirical_correctness":0.0,"signed_gap":0.2733738323052724,"absolute_gap":0.2733738323052724,"min_confidence":0.27246418595314026,"max_confidence":0.2738640010356903},{"index":6,"count":4,"confidence":0.27764078974723816,"empirical_correctness":0.25,"signed_gap":0.02764078974723816,"absolute_gap":0.02764078974723816,"min_confidence":0.2742406129837036,"max_confidence":0.28171417117118835},{"index":7,"count":4,"confidence":0.29165708273649216,"empirical_correctness":0.25,"signed_gap":0.04165708273649216,"absolute_gap":0.04165708273649216,"min_confidence":0.2839583456516266,"max_confidence":0.29720932245254517},{"index":8,"count":4,"confidence":0.32537757605314255,"empirical_correctness":0.0,"signed_gap":0.32537757605314255,"absolute_gap":0.32537757605314255,"min_confidence":0.2978671193122864,"max_confidence":0.33891603350639343},{"index":9,"count":4,"confidence":0.3435341864824295,"empirical_correctness":0.5,"signed_gap":-0.1564658135175705,"absolute_gap":0.1564658135175705,"min_confidence":0.34100523591041565,"max_confidence":0.3465132713317871}]}},"latency_ms":{"n":43,"median":88.72759994119406,"p95":91.15130000282079},"per_action":{"keep":{"n":12,"raw":{"n":12,"acceptable":12,"unsafe":0,"review":0,"automated_count":12,"automated_acceptable":12,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":1.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.7575059933447592,"upper":1.0,"confidence_level":0.95}},"effective":{"n":12,"acceptable":0,"unsafe":0,"review":12,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":12,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":2.7755575615628914e-17,"upper":0.24249400665524085,"confidence_level":0.95}}},"drop":{"n":16,"raw":{"n":16,"acceptable":0,"unsafe":16,"review":0,"automated_count":16,"automated_acceptable":0,"automated_unsafe":16,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":1.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":1.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.1936076805344365,"confidence_level":0.95}},"effective":{"n":16,"acceptable":0,"unsafe":0,"review":12,"automated_count":4,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":12,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":0.75,"automated_coverage":0.25,"conditional_correctness":0.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.75,"accuracy_wilson_95":{"lower":0.0,"upper":0.1936076805344365,"confidence_level":0.95}}},"mask":{"n":2,"raw":{"n":2,"acceptable":0,"unsafe":2,"review":0,"automated_count":2,"automated_acceptable":0,"automated_unsafe":2,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":1.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":1.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.6576197724933469,"confidence_level":0.95}},"effective":{"n":2,"acceptable":1,"unsafe":0,"review":1,"automated_count":1,"automated_acceptable":1,"automated_unsafe":0,"unnecessary_reviews":1,"accuracy":0.5,"unsafe_rate":0.0,"review_rate":0.5,"automated_coverage":0.5,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.5,"accuracy_wilson_95":{"lower":0.09453120573423074,"upper":0.9054687942657693,"confidence_level":0.95}}},"pseudonymize":{"n":2,"raw":{"n":2,"acceptable":0,"unsafe":2,"review":0,"automated_count":2,"automated_acceptable":0,"automated_unsafe":2,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":1.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":1.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.6576197724933469,"confidence_level":0.95}},"effective":{"n":2,"acceptable":0,"unsafe":0,"review":2,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":2,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.6576197724933469,"confidence_level":0.95}}},"generalize":{"n":6,"raw":{"n":6,"acceptable":0,"unsafe":6,"review":0,"automated_count":6,"automated_acceptable":0,"automated_unsafe":6,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":1.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":1.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":2.7755575615628914e-17,"upper":0.3903342879021653,"confidence_level":0.95}},"effective":{"n":6,"acceptable":0,"unsafe":0,"review":6,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":6,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":2.7755575615628914e-17,"upper":0.3903342879021653,"confidence_level":0.95}}},"review":{"n":0,"raw":{"n":0,"acceptable":0,"unsafe":0,"review":0,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":null,"unsafe_rate":null,"review_rate":null,"automated_coverage":null,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":null},"effective":{"n":0,"acceptable":0,"unsafe":0,"review":0,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":null,"unsafe_rate":null,"review_rate":null,"automated_coverage":null,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":null}}}},"diagnostic43-compact_no_samples":{"n":43,"raw":{"n":43,"acceptable":12,"unsafe":31,"review":0,"automated_count":43,"automated_acceptable":12,"automated_unsafe":31,"unnecessary_reviews":0,"accuracy":0.27906976744186046,"unsafe_rate":0.7209302325581395,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.27906976744186046,"conditional_unsafe":0.7209302325581395,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.16746613786237488,"upper":0.42691028681878873,"confidence_level":0.95}},"effective":{"n":43,"acceptable":6,"unsafe":0,"review":38,"automated_count":5,"automated_acceptable":1,"automated_unsafe":0,"unnecessary_reviews":33,"accuracy":0.13953488372093023,"unsafe_rate":0.0,"review_rate":0.8837209302325582,"automated_coverage":0.11627906976744186,"conditional_correctness":0.2,"conditional_unsafe":0.0,"unnecessary_review_rate":0.7674418604651163,"accuracy_wilson_95":{"lower":0.06555723427964878,"upper":0.2726358796738286,"confidence_level":0.95}},"confusion":{"raw":{"keep":{"keep":12,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0},"drop":{"keep":16,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0},"mask":{"keep":2,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0},"pseudonymize":{"keep":2,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0},"generalize":{"keep":6,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0},"review":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0}},"effective":{"keep":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":12},"drop":{"keep":0,"drop":0,"mask":4,"pseudonymize":0,"generalize":0,"review":12},"mask":{"keep":0,"drop":0,"mask":1,"pseudonymize":0,"generalize":0,"review":1},"pseudonymize":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":2},"generalize":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":6},"review":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0}}},"probability_metrics":{"singleton_n":38,"multiacceptable_n":5,"multiacceptable_coverage":0.11627906976744186,"nll":1.7190831130260606,"nll_zero_mass_n":0,"brier":0.8072095613175464,"acceptable_set_n":43,"acceptable_set_nll":1.6118348275096288,"acceptable_set_zero_mass_n":0},"calibration_all_labels":{"equal_width":{"n":43,"bin_count":10,"ece":0.03890240469644235,"signed_mean_gap":-0.03890240469644235,"absolute_mean_gap":0.03890240469644235,"bins":[{"index":0,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.0,"upper":0.1,"upper_inclusive":false},{"index":1,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.1,"upper":0.2,"upper_inclusive":false},{"index":2,"count":43,"confidence":0.2401673627454181,"empirical_correctness":0.27906976744186046,"signed_gap":-0.03890240469644235,"absolute_gap":0.03890240469644235,"lower":0.2,"upper":0.3,"upper_inclusive":false},{"index":3,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.3,"upper":0.4,"upper_inclusive":false},{"index":4,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.4,"upper":0.5,"upper_inclusive":false},{"index":5,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.5,"upper":0.6,"upper_inclusive":false},{"index":6,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.6,"upper":0.7,"upper_inclusive":false},{"index":7,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.7,"upper":0.8,"upper_inclusive":false},{"index":8,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.8,"upper":0.9,"upper_inclusive":false},{"index":9,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.9,"upper":1.0,"upper_inclusive":true}]},"equal_mass":{"n":43,"bin_count":10,"ece":0.2368869809217231,"signed_mean_gap":-0.038902404696442364,"absolute_mean_gap":0.038902404696442364,"bins":[{"index":0,"count":4,"confidence":0.22510026767849922,"empirical_correctness":0.0,"signed_gap":0.22510026767849922,"absolute_gap":0.22510026767849922,"min_confidence":0.21974527835845947,"max_confidence":0.23024001717567444},{"index":1,"count":4,"confidence":0.23237470164895058,"empirical_correctness":0.5,"signed_gap":-0.2676252983510494,"absolute_gap":0.2676252983510494,"min_confidence":0.23049071431159973,"max_confidence":0.23365622758865356},{"index":2,"count":4,"confidence":0.23477906733751297,"empirical_correctness":0.5,"signed_gap":-0.26522093266248703,"absolute_gap":0.26522093266248703,"min_confidence":0.23385289311408997,"max_confidence":0.2360168993473053},{"index":3,"count":5,"confidence":0.23739563822746276,"empirical_correctness":0.0,"signed_gap":0.23739563822746276,"absolute_gap":0.23739563822746276,"min_confidence":0.2363278716802597,"max_confidence":0.2380242794752121},{"index":4,"count":4,"confidence":0.23872380703687668,"empirical_correctness":0.75,"signed_gap":-0.5112761929631233,"absolute_gap":0.5112761929631233,"min_confidence":0.23821943998336792,"max_confidence":0.23921440541744232},{"index":5,"count":4,"confidence":0.24000350385904312,"empirical_correctness":0.0,"signed_gap":0.24000350385904312,"absolute_gap":0.24000350385904312,"min_confidence":0.23940566182136536,"max_confidence":0.24045491218566895},{"index":6,"count":5,"confidence":0.24185502231121064,"empirical_correctness":0.0,"signed_gap":0.24185502231121064,"absolute_gap":0.24185502231121064,"min_confidence":0.24104009568691254,"max_confidence":0.24238860607147217},{"index":7,"count":4,"confidence":0.24406855553388596,"empirical_correctness":0.5,"signed_gap":-0.25593144446611404,"absolute_gap":0.25593144446611404,"min_confidence":0.2425134778022766,"max_confidence":0.24581998586654663},{"index":8,"count":4,"confidence":0.24763965234160423,"empirical_correctness":0.25,"signed_gap":-0.002360347658395767,"absolute_gap":0.002360347658395767,"min_confidence":0.2461419701576233,"max_confidence":0.2502816915512085},{"index":9,"count":5,"confidence":0.2560370147228241,"empirical_correctness":0.4,"signed_gap":-0.14396298527717594,"absolute_gap":0.14396298527717594,"min_confidence":0.251363605260849,"max_confidence":0.2634487450122833}]}},"calibration_singleton_labels":{"equal_width":{"n":38,"bin_count":10,"ece":0.07542694163949865,"signed_mean_gap":-0.07542694163949865,"absolute_mean_gap":0.07542694163949865,"bins":[{"index":0,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.0,"upper":0.1,"upper_inclusive":false},{"index":1,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.1,"upper":0.2,"upper_inclusive":false},{"index":2,"count":38,"confidence":0.24036253204471186,"empirical_correctness":0.3157894736842105,"signed_gap":-0.07542694163949865,"absolute_gap":0.07542694163949865,"lower":0.2,"upper":0.3,"upper_inclusive":false},{"index":3,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.3,"upper":0.4,"upper_inclusive":false},{"index":4,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.4,"upper":0.5,"upper_inclusive":false},{"index":5,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.5,"upper":0.6,"upper_inclusive":false},{"index":6,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.6,"upper":0.7,"upper_inclusive":false},{"index":7,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.7,"upper":0.8,"upper_inclusive":false},{"index":8,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.8,"upper":0.9,"upper_inclusive":false},{"index":9,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.9,"upper":1.0,"upper_inclusive":true}]},"equal_mass":{"n":38,"bin_count":10,"ece":0.20011273653883682,"signed_mean_gap":-0.07542694163949866,"absolute_mean_gap":0.07542694163949866,"bins":[{"index":0,"count":3,"confidence":0.22717841466267905,"empirical_correctness":0.0,"signed_gap":0.22717841466267905,"absolute_gap":0.22717841466267905,"min_confidence":0.22080451250076294,"max_confidence":0.23049071431159973},{"index":1,"count":4,"confidence":0.23321524634957314,"empirical_correctness":0.75,"signed_gap":-0.5167847536504269,"absolute_gap":0.5167847536504269,"min_confidence":0.2318660318851471,"max_confidence":0.23385289311408997},{"index":2,"count":4,"confidence":0.23565379530191422,"empirical_correctness":0.25,"signed_gap":-0.014346204698085785,"absolute_gap":0.014346204698085785,"min_confidence":0.2341737002134323,"max_confidence":0.23735180497169495},{"index":3,"count":4,"confidence":0.23787948861718178,"empirical_correctness":0.25,"signed_gap":-0.012120511382818222,"absolute_gap":0.012120511382818222,"min_confidence":0.23742058873176575,"max_confidence":0.23821943998336792},{"index":4,"count":4,"confidence":0.23902036249637604,"empirical_correctness":0.5,"signed_gap":-0.26097963750362396,"absolute_gap":0.26097963750362396,"min_confidence":0.2386356145143509,"max_confidence":0.23940566182136536},{"index":5,"count":3,"confidence":0.24020278453826904,"empirical_correctness":0.0,"signed_gap":0.24020278453826904,"absolute_gap":0.24020278453826904,"min_confidence":0.2399020493030548,"max_confidence":0.24045491218566895},{"index":6,"count":4,"confidence":0.24172162637114525,"empirical_correctness":0.0,"signed_gap":0.24172162637114525,"absolute_gap":0.24172162637114525,"min_confidence":0.24104009568691254,"max_confidence":0.2423844337463379},{"index":7,"count":4,"confidence":0.24359390139579773,"empirical_correctness":0.5,"signed_gap":-0.25640609860420227,"absolute_gap":0.25640609860420227,"min_confidence":0.24238860607147217,"max_confidence":0.24581998586654663},{"index":8,"count":4,"confidence":0.24763965234160423,"empirical_correctness":0.25,"signed_gap":-0.002360347658395767,"absolute_gap":0.002360347658395767,"min_confidence":0.2461419701576233,"max_confidence":0.2502816915512085},{"index":9,"count":4,"confidence":0.2541840821504593,"empirical_correctness":0.5,"signed_gap":-0.2458159178495407,"absolute_gap":0.2458159178495407,"min_confidence":0.251363605260849,"max_confidence":0.25789675116539}]}},"latency_ms":{"n":43,"median":108.54099993593991,"p95":111.92367991898209},"per_action":{"keep":{"n":12,"raw":{"n":12,"acceptable":12,"unsafe":0,"review":0,"automated_count":12,"automated_acceptable":12,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":1.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.7575059933447592,"upper":1.0,"confidence_level":0.95}},"effective":{"n":12,"acceptable":0,"unsafe":0,"review":12,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":12,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":2.7755575615628914e-17,"upper":0.24249400665524085,"confidence_level":0.95}}},"drop":{"n":16,"raw":{"n":16,"acceptable":0,"unsafe":16,"review":0,"automated_count":16,"automated_acceptable":0,"automated_unsafe":16,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":1.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":1.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.1936076805344365,"confidence_level":0.95}},"effective":{"n":16,"acceptable":0,"unsafe":0,"review":12,"automated_count":4,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":12,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":0.75,"automated_coverage":0.25,"conditional_correctness":0.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.75,"accuracy_wilson_95":{"lower":0.0,"upper":0.1936076805344365,"confidence_level":0.95}}},"mask":{"n":2,"raw":{"n":2,"acceptable":0,"unsafe":2,"review":0,"automated_count":2,"automated_acceptable":0,"automated_unsafe":2,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":1.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":1.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.6576197724933469,"confidence_level":0.95}},"effective":{"n":2,"acceptable":1,"unsafe":0,"review":1,"automated_count":1,"automated_acceptable":1,"automated_unsafe":0,"unnecessary_reviews":1,"accuracy":0.5,"unsafe_rate":0.0,"review_rate":0.5,"automated_coverage":0.5,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.5,"accuracy_wilson_95":{"lower":0.09453120573423074,"upper":0.9054687942657693,"confidence_level":0.95}}},"pseudonymize":{"n":2,"raw":{"n":2,"acceptable":0,"unsafe":2,"review":0,"automated_count":2,"automated_acceptable":0,"automated_unsafe":2,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":1.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":1.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.6576197724933469,"confidence_level":0.95}},"effective":{"n":2,"acceptable":0,"unsafe":0,"review":2,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":2,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.6576197724933469,"confidence_level":0.95}}},"generalize":{"n":6,"raw":{"n":6,"acceptable":0,"unsafe":6,"review":0,"automated_count":6,"automated_acceptable":0,"automated_unsafe":6,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":1.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":1.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":2.7755575615628914e-17,"upper":0.3903342879021653,"confidence_level":0.95}},"effective":{"n":6,"acceptable":0,"unsafe":0,"review":6,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":6,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":2.7755575615628914e-17,"upper":0.3903342879021653,"confidence_level":0.95}}},"review":{"n":0,"raw":{"n":0,"acceptable":0,"unsafe":0,"review":0,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":null,"unsafe_rate":null,"review_rate":null,"automated_coverage":null,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":null},"effective":{"n":0,"acceptable":0,"unsafe":0,"review":0,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":null,"unsafe_rate":null,"review_rate":null,"automated_coverage":null,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":null}}}},"challenge56-no_samples":{"n":56,"raw":{"n":56,"acceptable":8,"unsafe":48,"review":0,"automated_count":56,"automated_acceptable":8,"automated_unsafe":48,"unnecessary_reviews":0,"accuracy":0.14285714285714285,"unsafe_rate":0.8571428571428571,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.14285714285714285,"conditional_unsafe":0.8571428571428571,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.07420788536920624,"upper":0.25735921228705333,"confidence_level":0.95}},"effective":{"n":56,"acceptable":2,"unsafe":0,"review":56,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":54,"accuracy":0.03571428571428571,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":0.9642857142857143,"accuracy_wilson_95":{"lower":0.00984942514823639,"upper":0.12118780180490107,"confidence_level":0.95}},"confusion":{"raw":{"keep":{"keep":8,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0},"drop":{"keep":22,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0},"mask":{"keep":8,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0},"pseudonymize":{"keep":8,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0},"generalize":{"keep":8,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0},"review":{"keep":2,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0}},"effective":{"keep":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":8},"drop":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":22},"mask":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":8},"pseudonymize":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":8},"generalize":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":8},"review":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":2}}},"probability_metrics":{"singleton_n":56,"multiacceptable_n":0,"multiacceptable_coverage":0.0,"nll":1.8212604855836638,"nll_zero_mass_n":0,"brier":0.8485418998996407,"acceptable_set_n":56,"acceptable_set_nll":1.8212604855836638,"acceptable_set_zero_mass_n":0},"calibration_all_labels":{"equal_width":{"n":56,"bin_count":10,"ece":0.12396692324961935,"signed_mean_gap":0.12396692324961935,"absolute_mean_gap":0.12396692324961935,"bins":[{"index":0,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.0,"upper":0.1,"upper_inclusive":false},{"index":1,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.1,"upper":0.2,"upper_inclusive":false},{"index":2,"count":46,"confidence":0.2547721947016923,"empirical_correctness":0.13043478260869565,"signed_gap":0.12433741209299667,"absolute_gap":0.12433741209299667,"lower":0.2,"upper":0.3,"upper_inclusive":false},{"index":3,"count":10,"confidence":0.3222626745700836,"empirical_correctness":0.2,"signed_gap":0.12226267457008361,"absolute_gap":0.12226267457008361,"lower":0.3,"upper":0.4,"upper_inclusive":false},{"index":4,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.4,"upper":0.5,"upper_inclusive":false},{"index":5,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.5,"upper":0.6,"upper_inclusive":false},{"index":6,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.6,"upper":0.7,"upper_inclusive":false},{"index":7,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.7,"upper":0.8,"upper_inclusive":false},{"index":8,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.8,"upper":0.9,"upper_inclusive":false},{"index":9,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.9,"upper":1.0,"upper_inclusive":true}]},"equal_mass":{"n":56,"bin_count":10,"ece":0.14375575578638486,"signed_mean_gap":0.12396692324961936,"absolute_mean_gap":0.12396692324961936,"bins":[{"index":0,"count":5,"confidence":0.23621513247489928,"empirical_correctness":0.0,"signed_gap":0.23621513247489928,"absolute_gap":0.23621513247489928,"min_confidence":0.23122937977313995,"max_confidence":0.23995590209960938},{"index":1,"count":6,"confidence":0.24098544816176096,"empirical_correctness":0.3333333333333333,"signed_gap":-0.09234788517157236,"absolute_gap":0.09234788517157236,"min_confidence":0.24017418920993805,"max_confidence":0.241720050573349},{"index":2,"count":5,"confidence":0.24405753016471862,"empirical_correctness":0.2,"signed_gap":0.04405753016471861,"absolute_gap":0.04405753016471861,"min_confidence":0.24227575957775116,"max_confidence":0.2461279034614563},{"index":3,"count":6,"confidence":0.24805269141991934,"empirical_correctness":0.16666666666666666,"signed_gap":0.08138602475325268,"absolute_gap":0.08138602475325268,"min_confidence":0.24645903706550598,"max_confidence":0.2508872151374817},{"index":4,"count":6,"confidence":0.254348690311114,"empirical_correctness":0.16666666666666666,"signed_gap":0.08768202364444735,"absolute_gap":0.08768202364444735,"min_confidence":0.252998411655426,"max_confidence":0.25570085644721985},{"index":5,"count":5,"confidence":0.2596337556838989,"empirical_correctness":0.0,"signed_gap":0.2596337556838989,"absolute_gap":0.2596337556838989,"min_confidence":0.2564421594142914,"max_confidence":0.26139649748802185},{"index":6,"count":6,"confidence":0.26852932075659436,"empirical_correctness":0.0,"signed_gap":0.26852932075659436,"absolute_gap":0.26852932075659436,"min_confidence":0.2621934413909912,"max_confidence":0.27318471670150757},{"index":7,"count":5,"confidence":0.27694409489631655,"empirical_correctness":0.2,"signed_gap":0.07694409489631654,"absolute_gap":0.07694409489631654,"min_confidence":0.2741718590259552,"max_confidence":0.27960145473480225},{"index":8,"count":6,"confidence":0.30020102361838025,"empirical_correctness":0.16666666666666666,"signed_gap":0.1335343569517136,"absolute_gap":0.1335343569517136,"min_confidence":0.28067880868911743,"max_confidence":0.315065860748291},{"index":9,"count":6,"confidence":0.33086534837881726,"empirical_correctness":0.16666666666666666,"signed_gap":0.1641986817121506,"absolute_gap":0.1641986817121506,"min_confidence":0.31529584527015686,"max_confidence":0.3507052958011627}]}},"calibration_singleton_labels":{"equal_width":{"n":56,"bin_count":10,"ece":0.12396692324961935,"signed_mean_gap":0.12396692324961935,"absolute_mean_gap":0.12396692324961935,"bins":[{"index":0,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.0,"upper":0.1,"upper_inclusive":false},{"index":1,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.1,"upper":0.2,"upper_inclusive":false},{"index":2,"count":46,"confidence":0.2547721947016923,"empirical_correctness":0.13043478260869565,"signed_gap":0.12433741209299667,"absolute_gap":0.12433741209299667,"lower":0.2,"upper":0.3,"upper_inclusive":false},{"index":3,"count":10,"confidence":0.3222626745700836,"empirical_correctness":0.2,"signed_gap":0.12226267457008361,"absolute_gap":0.12226267457008361,"lower":0.3,"upper":0.4,"upper_inclusive":false},{"index":4,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.4,"upper":0.5,"upper_inclusive":false},{"index":5,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.5,"upper":0.6,"upper_inclusive":false},{"index":6,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.6,"upper":0.7,"upper_inclusive":false},{"index":7,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.7,"upper":0.8,"upper_inclusive":false},{"index":8,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.8,"upper":0.9,"upper_inclusive":false},{"index":9,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.9,"upper":1.0,"upper_inclusive":true}]},"equal_mass":{"n":56,"bin_count":10,"ece":0.14375575578638486,"signed_mean_gap":0.12396692324961936,"absolute_mean_gap":0.12396692324961936,"bins":[{"index":0,"count":5,"confidence":0.23621513247489928,"empirical_correctness":0.0,"signed_gap":0.23621513247489928,"absolute_gap":0.23621513247489928,"min_confidence":0.23122937977313995,"max_confidence":0.23995590209960938},{"index":1,"count":6,"confidence":0.24098544816176096,"empirical_correctness":0.3333333333333333,"signed_gap":-0.09234788517157236,"absolute_gap":0.09234788517157236,"min_confidence":0.24017418920993805,"max_confidence":0.241720050573349},{"index":2,"count":5,"confidence":0.24405753016471862,"empirical_correctness":0.2,"signed_gap":0.04405753016471861,"absolute_gap":0.04405753016471861,"min_confidence":0.24227575957775116,"max_confidence":0.2461279034614563},{"index":3,"count":6,"confidence":0.24805269141991934,"empirical_correctness":0.16666666666666666,"signed_gap":0.08138602475325268,"absolute_gap":0.08138602475325268,"min_confidence":0.24645903706550598,"max_confidence":0.2508872151374817},{"index":4,"count":6,"confidence":0.254348690311114,"empirical_correctness":0.16666666666666666,"signed_gap":0.08768202364444735,"absolute_gap":0.08768202364444735,"min_confidence":0.252998411655426,"max_confidence":0.25570085644721985},{"index":5,"count":5,"confidence":0.2596337556838989,"empirical_correctness":0.0,"signed_gap":0.2596337556838989,"absolute_gap":0.2596337556838989,"min_confidence":0.2564421594142914,"max_confidence":0.26139649748802185},{"index":6,"count":6,"confidence":0.26852932075659436,"empirical_correctness":0.0,"signed_gap":0.26852932075659436,"absolute_gap":0.26852932075659436,"min_confidence":0.2621934413909912,"max_confidence":0.27318471670150757},{"index":7,"count":5,"confidence":0.27694409489631655,"empirical_correctness":0.2,"signed_gap":0.07694409489631654,"absolute_gap":0.07694409489631654,"min_confidence":0.2741718590259552,"max_confidence":0.27960145473480225},{"index":8,"count":6,"confidence":0.30020102361838025,"empirical_correctness":0.16666666666666666,"signed_gap":0.1335343569517136,"absolute_gap":0.1335343569517136,"min_confidence":0.28067880868911743,"max_confidence":0.315065860748291},{"index":9,"count":6,"confidence":0.33086534837881726,"empirical_correctness":0.16666666666666666,"signed_gap":0.1641986817121506,"absolute_gap":0.1641986817121506,"min_confidence":0.31529584527015686,"max_confidence":0.3507052958011627}]}},"latency_ms":{"n":56,"median":78.42689985409379,"p95":89.96197499800473},"per_action":{"keep":{"n":8,"raw":{"n":8,"acceptable":8,"unsafe":0,"review":0,"automated_count":8,"automated_acceptable":8,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":1.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.6755924351161198,"upper":1.0,"confidence_level":0.95}},"effective":{"n":8,"acceptable":0,"unsafe":0,"review":8,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":8,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.32440756488388023,"confidence_level":0.95}}},"drop":{"n":22,"raw":{"n":22,"acceptable":0,"unsafe":22,"review":0,"automated_count":22,"automated_acceptable":0,"automated_unsafe":22,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":1.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":1.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.14865487460862087,"confidence_level":0.95}},"effective":{"n":22,"acceptable":0,"unsafe":0,"review":22,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":22,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.14865487460862087,"confidence_level":0.95}}},"mask":{"n":8,"raw":{"n":8,"acceptable":0,"unsafe":8,"review":0,"automated_count":8,"automated_acceptable":0,"automated_unsafe":8,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":1.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":1.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.32440756488388023,"confidence_level":0.95}},"effective":{"n":8,"acceptable":0,"unsafe":0,"review":8,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":8,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.32440756488388023,"confidence_level":0.95}}},"pseudonymize":{"n":8,"raw":{"n":8,"acceptable":0,"unsafe":8,"review":0,"automated_count":8,"automated_acceptable":0,"automated_unsafe":8,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":1.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":1.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.32440756488388023,"confidence_level":0.95}},"effective":{"n":8,"acceptable":0,"unsafe":0,"review":8,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":8,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.32440756488388023,"confidence_level":0.95}}},"generalize":{"n":8,"raw":{"n":8,"acceptable":0,"unsafe":8,"review":0,"automated_count":8,"automated_acceptable":0,"automated_unsafe":8,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":1.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":1.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.32440756488388023,"confidence_level":0.95}},"effective":{"n":8,"acceptable":0,"unsafe":0,"review":8,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":8,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.32440756488388023,"confidence_level":0.95}}},"review":{"n":2,"raw":{"n":2,"acceptable":0,"unsafe":2,"review":0,"automated_count":2,"automated_acceptable":0,"automated_unsafe":2,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":1.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":1.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.6576197724933469,"confidence_level":0.95}},"effective":{"n":2,"acceptable":2,"unsafe":0,"review":2,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":1.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.34238022750665303,"upper":1.0,"confidence_level":0.95}}}}},"challenge56-compact_no_samples":{"n":56,"raw":{"n":56,"acceptable":8,"unsafe":48,"review":0,"automated_count":56,"automated_acceptable":8,"automated_unsafe":48,"unnecessary_reviews":0,"accuracy":0.14285714285714285,"unsafe_rate":0.8571428571428571,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.14285714285714285,"conditional_unsafe":0.8571428571428571,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.07420788536920624,"upper":0.25735921228705333,"confidence_level":0.95}},"effective":{"n":56,"acceptable":2,"unsafe":0,"review":56,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":54,"accuracy":0.03571428571428571,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":0.9642857142857143,"accuracy_wilson_95":{"lower":0.00984942514823639,"upper":0.12118780180490107,"confidence_level":0.95}},"confusion":{"raw":{"keep":{"keep":8,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0},"drop":{"keep":22,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0},"mask":{"keep":8,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0},"pseudonymize":{"keep":8,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0},"generalize":{"keep":8,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0},"review":{"keep":2,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0}},"effective":{"keep":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":8},"drop":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":22},"mask":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":8},"pseudonymize":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":8},"generalize":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":8},"review":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":2}}},"probability_metrics":{"singleton_n":56,"multiacceptable_n":0,"multiacceptable_coverage":0.0,"nll":1.8097207930004302,"nll_zero_mass_n":0,"brier":0.8409271049384185,"acceptable_set_n":56,"acceptable_set_nll":1.8097207930004302,"acceptable_set_zero_mass_n":0},"calibration_all_labels":{"equal_width":{"n":56,"bin_count":10,"ece":0.0930721243577344,"signed_mean_gap":0.0930721243577344,"absolute_mean_gap":0.0930721243577344,"bins":[{"index":0,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.0,"upper":0.1,"upper_inclusive":false},{"index":1,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.1,"upper":0.2,"upper_inclusive":false},{"index":2,"count":56,"confidence":0.23592926721487725,"empirical_correctness":0.14285714285714285,"signed_gap":0.0930721243577344,"absolute_gap":0.0930721243577344,"lower":0.2,"upper":0.3,"upper_inclusive":false},{"index":3,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.3,"upper":0.4,"upper_inclusive":false},{"index":4,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.4,"upper":0.5,"upper_inclusive":false},{"index":5,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.5,"upper":0.6,"upper_inclusive":false},{"index":6,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.6,"upper":0.7,"upper_inclusive":false},{"index":7,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.7,"upper":0.8,"upper_inclusive":false},{"index":8,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.8,"upper":0.9,"upper_inclusive":false},{"index":9,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.9,"upper":1.0,"upper_inclusive":true}]},"equal_mass":{"n":56,"bin_count":10,"ece":0.11348928245050567,"signed_mean_gap":0.09307212435773442,"absolute_mean_gap":0.09307212435773442,"bins":[{"index":0,"count":5,"confidence":0.21705245077610016,"empirical_correctness":0.0,"signed_gap":0.21705245077610016,"absolute_gap":0.21705245077610016,"min_confidence":0.21512697637081146,"max_confidence":0.2179524451494217},{"index":1,"count":6,"confidence":0.22233585019906363,"empirical_correctness":0.16666666666666666,"signed_gap":0.05566918353239697,"absolute_gap":0.05566918353239697,"min_confidence":0.2199804037809372,"max_confidence":0.2247142195701599},{"index":2,"count":5,"confidence":0.2265852153301239,"empirical_correctness":0.0,"signed_gap":0.2265852153301239,"absolute_gap":0.2265852153301239,"min_confidence":0.22538870573043823,"max_confidence":0.22766560316085815},{"index":3,"count":6,"confidence":0.22976478189229965,"empirical_correctness":0.16666666666666666,"signed_gap":0.063098115225633,"absolute_gap":0.063098115225633,"min_confidence":0.22872479259967804,"max_confidence":0.23079507052898407},{"index":4,"count":6,"confidence":0.23208964864412943,"empirical_correctness":0.16666666666666666,"signed_gap":0.06542298197746277,"absolute_gap":0.06542298197746277,"min_confidence":0.23107671737670898,"max_confidence":0.23343317210674286},{"index":5,"count":5,"confidence":0.23486397564411163,"empirical_correctness":0.0,"signed_gap":0.23486397564411163,"absolute_gap":0.23486397564411163,"min_confidence":0.23355787992477417,"max_confidence":0.23580233752727509},{"index":6,"count":6,"confidence":0.23805326223373413,"empirical_correctness":0.3333333333333333,"signed_gap":-0.09528007109959918,"absolute_gap":0.09528007109959918,"min_confidence":0.23706461489200592,"max_confidence":0.24032357335090637},{"index":7,"count":5,"confidence":0.24505501687526704,"empirical_correctness":0.2,"signed_gap":0.04505501687526703,"absolute_gap":0.04505501687526703,"min_confidence":0.24071533977985382,"max_confidence":0.2468242049217224},{"index":8,"count":6,"confidence":0.25173349181811017,"empirical_correctness":0.16666666666666666,"signed_gap":0.08506682515144351,"absolute_gap":0.08506682515144351,"min_confidence":0.25005069375038147,"max_confidence":0.2537907660007477},{"index":9,"count":6,"confidence":0.2583989103635152,"empirical_correctness":0.16666666666666666,"signed_gap":0.09173224369684854,"absolute_gap":0.09173224369684854,"min_confidence":0.25494468212127686,"max_confidence":0.2615465819835663}]}},"calibration_singleton_labels":{"equal_width":{"n":56,"bin_count":10,"ece":0.0930721243577344,"signed_mean_gap":0.0930721243577344,"absolute_mean_gap":0.0930721243577344,"bins":[{"index":0,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.0,"upper":0.1,"upper_inclusive":false},{"index":1,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.1,"upper":0.2,"upper_inclusive":false},{"index":2,"count":56,"confidence":0.23592926721487725,"empirical_correctness":0.14285714285714285,"signed_gap":0.0930721243577344,"absolute_gap":0.0930721243577344,"lower":0.2,"upper":0.3,"upper_inclusive":false},{"index":3,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.3,"upper":0.4,"upper_inclusive":false},{"index":4,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.4,"upper":0.5,"upper_inclusive":false},{"index":5,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.5,"upper":0.6,"upper_inclusive":false},{"index":6,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.6,"upper":0.7,"upper_inclusive":false},{"index":7,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.7,"upper":0.8,"upper_inclusive":false},{"index":8,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.8,"upper":0.9,"upper_inclusive":false},{"index":9,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.9,"upper":1.0,"upper_inclusive":true}]},"equal_mass":{"n":56,"bin_count":10,"ece":0.11348928245050567,"signed_mean_gap":0.09307212435773442,"absolute_mean_gap":0.09307212435773442,"bins":[{"index":0,"count":5,"confidence":0.21705245077610016,"empirical_correctness":0.0,"signed_gap":0.21705245077610016,"absolute_gap":0.21705245077610016,"min_confidence":0.21512697637081146,"max_confidence":0.2179524451494217},{"index":1,"count":6,"confidence":0.22233585019906363,"empirical_correctness":0.16666666666666666,"signed_gap":0.05566918353239697,"absolute_gap":0.05566918353239697,"min_confidence":0.2199804037809372,"max_confidence":0.2247142195701599},{"index":2,"count":5,"confidence":0.2265852153301239,"empirical_correctness":0.0,"signed_gap":0.2265852153301239,"absolute_gap":0.2265852153301239,"min_confidence":0.22538870573043823,"max_confidence":0.22766560316085815},{"index":3,"count":6,"confidence":0.22976478189229965,"empirical_correctness":0.16666666666666666,"signed_gap":0.063098115225633,"absolute_gap":0.063098115225633,"min_confidence":0.22872479259967804,"max_confidence":0.23079507052898407},{"index":4,"count":6,"confidence":0.23208964864412943,"empirical_correctness":0.16666666666666666,"signed_gap":0.06542298197746277,"absolute_gap":0.06542298197746277,"min_confidence":0.23107671737670898,"max_confidence":0.23343317210674286},{"index":5,"count":5,"confidence":0.23486397564411163,"empirical_correctness":0.0,"signed_gap":0.23486397564411163,"absolute_gap":0.23486397564411163,"min_confidence":0.23355787992477417,"max_confidence":0.23580233752727509},{"index":6,"count":6,"confidence":0.23805326223373413,"empirical_correctness":0.3333333333333333,"signed_gap":-0.09528007109959918,"absolute_gap":0.09528007109959918,"min_confidence":0.23706461489200592,"max_confidence":0.24032357335090637},{"index":7,"count":5,"confidence":0.24505501687526704,"empirical_correctness":0.2,"signed_gap":0.04505501687526703,"absolute_gap":0.04505501687526703,"min_confidence":0.24071533977985382,"max_confidence":0.2468242049217224},{"index":8,"count":6,"confidence":0.25173349181811017,"empirical_correctness":0.16666666666666666,"signed_gap":0.08506682515144351,"absolute_gap":0.08506682515144351,"min_confidence":0.25005069375038147,"max_confidence":0.2537907660007477},{"index":9,"count":6,"confidence":0.2583989103635152,"empirical_correctness":0.16666666666666666,"signed_gap":0.09173224369684854,"absolute_gap":0.09173224369684854,"min_confidence":0.25494468212127686,"max_confidence":0.2615465819835663}]}},"latency_ms":{"n":56,"median":90.98654985427856,"p95":95.9560748306103},"per_action":{"keep":{"n":8,"raw":{"n":8,"acceptable":8,"unsafe":0,"review":0,"automated_count":8,"automated_acceptable":8,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":1.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.6755924351161198,"upper":1.0,"confidence_level":0.95}},"effective":{"n":8,"acceptable":0,"unsafe":0,"review":8,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":8,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.32440756488388023,"confidence_level":0.95}}},"drop":{"n":22,"raw":{"n":22,"acceptable":0,"unsafe":22,"review":0,"automated_count":22,"automated_acceptable":0,"automated_unsafe":22,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":1.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":1.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.14865487460862087,"confidence_level":0.95}},"effective":{"n":22,"acceptable":0,"unsafe":0,"review":22,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":22,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.14865487460862087,"confidence_level":0.95}}},"mask":{"n":8,"raw":{"n":8,"acceptable":0,"unsafe":8,"review":0,"automated_count":8,"automated_acceptable":0,"automated_unsafe":8,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":1.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":1.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.32440756488388023,"confidence_level":0.95}},"effective":{"n":8,"acceptable":0,"unsafe":0,"review":8,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":8,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.32440756488388023,"confidence_level":0.95}}},"pseudonymize":{"n":8,"raw":{"n":8,"acceptable":0,"unsafe":8,"review":0,"automated_count":8,"automated_acceptable":0,"automated_unsafe":8,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":1.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":1.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.32440756488388023,"confidence_level":0.95}},"effective":{"n":8,"acceptable":0,"unsafe":0,"review":8,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":8,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.32440756488388023,"confidence_level":0.95}}},"generalize":{"n":8,"raw":{"n":8,"acceptable":0,"unsafe":8,"review":0,"automated_count":8,"automated_acceptable":0,"automated_unsafe":8,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":1.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":1.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.32440756488388023,"confidence_level":0.95}},"effective":{"n":8,"acceptable":0,"unsafe":0,"review":8,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":8,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.32440756488388023,"confidence_level":0.95}}},"review":{"n":2,"raw":{"n":2,"acceptable":0,"unsafe":2,"review":0,"automated_count":2,"automated_acceptable":0,"automated_unsafe":2,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":1.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":1.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.6576197724933469,"confidence_level":0.95}},"effective":{"n":2,"acceptable":2,"unsafe":0,"review":2,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":1.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.34238022750665303,"upper":1.0,"confidence_level":0.95}}}}}},"composite":{"n":198,"scope":"descriptive sum across four correlated panels","raw":{"n":198,"acceptable":40,"unsafe":158,"review":0,"automated_count":198,"automated_acceptable":40,"automated_unsafe":158,"unnecessary_reviews":0,"accuracy":0.20202020202020202,"unsafe_rate":0.797979797979798,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.20202020202020202,"conditional_unsafe":0.797979797979798,"unnecessary_review_rate":0.0},"effective":{"n":198,"acceptable":16,"unsafe":0,"review":188,"automated_count":10,"automated_acceptable":2,"automated_unsafe":0,"unnecessary_reviews":174,"accuracy":0.08080808080808081,"unsafe_rate":0.0,"review_rate":0.9494949494949495,"automated_coverage":0.050505050505050504,"conditional_correctness":0.2,"conditional_unsafe":0.0,"unnecessary_review_rate":0.8787878787878788}}},{"id":"facebook--bart-large-mnli","platform":"Local","model_id":"facebook/bart-large-mnli","revision":"d7645e127eaf1aefc7862fd59a17a5aa8558b8ce","loading":"float32 / none","scoring_kind":"Adapted NLI","settings":{"dtype":"float32","quantization":null,"max_tokens":4096,"effective_max_tokens":1024,"temperature":1.0,"scoring_batch_size":1,"trust_remote_code":false,"local_files_only":true,"use_safetensors":true,"guard_threshold":0.45,"seed":0,"tf32":false},"scoring_metadata":{"backend":"nli","protocol":"hf-local-sweep-backends-v1","probability_source":"adapted_entailment_softmax_not_calibrated_sixway","latency_scope":"synchronous_forward_scoring_including_transfer_excluding_tokenization","precision":"float32","equal_mass_convention":"rank bins with floor boundaries at floor(i*n/k), k=min(10,n//2); minimum two rows"},"load_ms":6879.656400065869,"peak_allocated_bytes":1754362880,"peak_reserved_bytes":1835008000,"panels":{"diagnostic43-no_samples":{"n":43,"raw":{"n":43,"acceptable":12,"unsafe":29,"review":0,"automated_count":43,"automated_acceptable":12,"automated_unsafe":29,"unnecessary_reviews":0,"accuracy":0.27906976744186046,"unsafe_rate":0.6744186046511628,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.27906976744186046,"conditional_unsafe":0.6744186046511628,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.16746613786237488,"upper":0.42691028681878873,"confidence_level":0.95}},"effective":{"n":43,"acceptable":6,"unsafe":0,"review":38,"automated_count":5,"automated_acceptable":1,"automated_unsafe":0,"unnecessary_reviews":33,"accuracy":0.13953488372093023,"unsafe_rate":0.0,"review_rate":0.8837209302325582,"automated_coverage":0.11627906976744186,"conditional_correctness":0.2,"conditional_unsafe":0.0,"unnecessary_review_rate":0.7674418604651163,"accuracy_wilson_95":{"lower":0.06555723427964878,"upper":0.2726358796738286,"confidence_level":0.95}},"confusion":{"raw":{"keep":{"keep":12,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0},"drop":{"keep":16,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0},"mask":{"keep":2,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0},"pseudonymize":{"keep":2,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0},"generalize":{"keep":4,"drop":0,"mask":0,"pseudonymize":2,"generalize":0,"review":0},"review":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0}},"effective":{"keep":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":12},"drop":{"keep":0,"drop":0,"mask":4,"pseudonymize":0,"generalize":0,"review":12},"mask":{"keep":0,"drop":0,"mask":1,"pseudonymize":0,"generalize":0,"review":1},"pseudonymize":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":2},"generalize":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":6},"review":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0}}},"probability_metrics":{"singleton_n":38,"multiacceptable_n":5,"multiacceptable_coverage":0.11627906976744186,"nll":1.686509710014564,"nll_zero_mass_n":0,"brier":0.7932947588615412,"acceptable_set_n":43,"acceptable_set_nll":1.593354580552779,"acceptable_set_zero_mass_n":0},"calibration_all_labels":{"equal_width":{"n":43,"bin_count":10,"ece":0.05414038965868399,"signed_mean_gap":-0.0011328604332236304,"absolute_mean_gap":0.0011328604332236304,"bins":[{"index":0,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.0,"upper":0.1,"upper_inclusive":false},{"index":1,"count":3,"confidence":0.19696791470050812,"empirical_correctness":0.0,"signed_gap":0.19696791470050812,"absolute_gap":0.19696791470050812,"lower":0.1,"upper":0.2,"upper_inclusive":false},{"index":2,"count":25,"confidence":0.2619503253698349,"empirical_correctness":0.24,"signed_gap":0.021950325369834933,"absolute_gap":0.021950325369834933,"lower":0.2,"upper":0.3,"upper_inclusive":false},{"index":3,"count":15,"confidence":0.3207750082015991,"empirical_correctness":0.4,"signed_gap":-0.07922499179840092,"absolute_gap":0.07922499179840092,"lower":0.3,"upper":0.4,"upper_inclusive":false},{"index":4,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.4,"upper":0.5,"upper_inclusive":false},{"index":5,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.5,"upper":0.6,"upper_inclusive":false},{"index":6,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.6,"upper":0.7,"upper_inclusive":false},{"index":7,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.7,"upper":0.8,"upper_inclusive":false},{"index":8,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.8,"upper":0.9,"upper_inclusive":false},{"index":9,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.9,"upper":1.0,"upper_inclusive":true}]},"equal_mass":{"n":43,"bin_count":10,"ece":0.1730687292509301,"signed_mean_gap":-0.0011328604332236304,"absolute_mean_gap":0.0011328604332236304,"bins":[{"index":0,"count":4,"confidence":0.1980355717241764,"empirical_correctness":0.0,"signed_gap":0.1980355717241764,"absolute_gap":0.1980355717241764,"min_confidence":0.1955503672361374,"max_confidence":0.20123854279518127},{"index":1,"count":4,"confidence":0.22252532094717026,"empirical_correctness":0.25,"signed_gap":-0.027474679052829742,"absolute_gap":0.027474679052829742,"min_confidence":0.20461484789848328,"max_confidence":0.24417421221733093},{"index":2,"count":4,"confidence":0.25057728961110115,"empirical_correctness":0.0,"signed_gap":0.25057728961110115,"absolute_gap":0.25057728961110115,"min_confidence":0.2457350641489029,"max_confidence":0.2576126456260681},{"index":3,"count":5,"confidence":0.2617840886116028,"empirical_correctness":0.4,"signed_gap":-0.13821591138839723,"absolute_gap":0.13821591138839723,"min_confidence":0.2588233947753906,"max_confidence":0.2669726610183716},{"index":4,"count":4,"confidence":0.282136470079422,"empirical_correctness":0.5,"signed_gap":-0.217863529920578,"absolute_gap":0.217863529920578,"min_confidence":0.28043898940086365,"max_confidence":0.28379374742507935},{"index":5,"count":4,"confidence":0.28574924916028976,"empirical_correctness":0.25,"signed_gap":0.035749249160289764,"absolute_gap":0.035749249160289764,"min_confidence":0.2840487062931061,"max_confidence":0.2881137430667877},{"index":6,"count":5,"confidence":0.2960507094860077,"empirical_correctness":0.0,"signed_gap":0.2960507094860077,"absolute_gap":0.2960507094860077,"min_confidence":0.28821635246276855,"max_confidence":0.3031531870365143},{"index":7,"count":4,"confidence":0.3080833777785301,"empirical_correctness":0.5,"signed_gap":-0.19191662222146988,"absolute_gap":0.19191662222146988,"min_confidence":0.30338433384895325,"max_confidence":0.3109966516494751},{"index":8,"count":4,"confidence":0.3197297975420952,"empirical_correctness":0.25,"signed_gap":0.06972979754209518,"absolute_gap":0.06972979754209518,"min_confidence":0.3165181279182434,"max_confidence":0.3240015208721161},{"index":9,"count":5,"confidence":0.33895294070243837,"empirical_correctness":0.6,"signed_gap":-0.2610470592975616,"absolute_gap":0.2610470592975616,"min_confidence":0.3251131772994995,"max_confidence":0.35399994254112244}]}},"calibration_singleton_labels":{"equal_width":{"n":38,"bin_count":10,"ece":0.07032191243610884,"signed_mean_gap":-0.039221715378133876,"absolute_mean_gap":0.039221715378133876,"bins":[{"index":0,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.0,"upper":0.1,"upper_inclusive":false},{"index":1,"count":3,"confidence":0.19696791470050812,"empirical_correctness":0.0,"signed_gap":0.19696791470050812,"absolute_gap":0.19696791470050812,"lower":0.1,"upper":0.2,"upper_inclusive":false},{"index":2,"count":22,"confidence":0.2620090490037745,"empirical_correctness":0.2727272727272727,"signed_gap":-0.010718223723498232,"absolute_gap":0.010718223723498232,"lower":0.2,"upper":0.3,"upper_inclusive":false},{"index":3,"count":13,"confidence":0.3195747687266423,"empirical_correctness":0.46153846153846156,"signed_gap":-0.14196369281181925,"absolute_gap":0.14196369281181925,"lower":0.3,"upper":0.4,"upper_inclusive":false},{"index":4,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.4,"upper":0.5,"upper_inclusive":false},{"index":5,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.5,"upper":0.6,"upper_inclusive":false},{"index":6,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.6,"upper":0.7,"upper_inclusive":false},{"index":7,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.7,"upper":0.8,"upper_inclusive":false},{"index":8,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.8,"upper":0.9,"upper_inclusive":false},{"index":9,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.9,"upper":1.0,"upper_inclusive":true}]},"equal_mass":{"n":38,"bin_count":10,"ece":0.19965434427324094,"signed_mean_gap":-0.039221715378133876,"absolute_mean_gap":0.039221715378133876,"bins":[{"index":0,"count":3,"confidence":0.19696791470050812,"empirical_correctness":0.0,"signed_gap":0.19696791470050812,"absolute_gap":0.19696791470050812,"min_confidence":0.1955503672361374,"max_confidence":0.19777356088161469},{"index":1,"count":4,"confidence":0.2136826030910015,"empirical_correctness":0.25,"signed_gap":-0.03631739690899849,"absolute_gap":0.03631739690899849,"min_confidence":0.20123854279518127,"max_confidence":0.24417421221733093},{"index":2,"count":4,"confidence":0.25057728961110115,"empirical_correctness":0.0,"signed_gap":0.25057728961110115,"absolute_gap":0.25057728961110115,"min_confidence":0.2457350641489029,"max_confidence":0.2576126456260681},{"index":3,"count":4,"confidence":0.2625242620706558,"empirical_correctness":0.5,"signed_gap":-0.23747573792934418,"absolute_gap":0.23747573792934418,"min_confidence":0.2597385346889496,"max_confidence":0.2669726610183716},{"index":4,"count":4,"confidence":0.282136470079422,"empirical_correctness":0.5,"signed_gap":-0.217863529920578,"absolute_gap":0.217863529920578,"min_confidence":0.28043898940086365,"max_confidence":0.28379374742507935},{"index":5,"count":3,"confidence":0.28496108452479046,"empirical_correctness":0.3333333333333333,"signed_gap":-0.04837224880854285,"absolute_gap":0.04837224880854285,"min_confidence":0.2840487062931061,"max_confidence":0.2867467403411865},{"index":6,"count":4,"confidence":0.29402196407318115,"empirical_correctness":0.0,"signed_gap":0.29402196407318115,"absolute_gap":0.29402196407318115,"min_confidence":0.2881137430667877,"max_confidence":0.30245453119277954},{"index":7,"count":4,"confidence":0.3080833777785301,"empirical_correctness":0.5,"signed_gap":-0.19191662222146988,"absolute_gap":0.19191662222146988,"min_confidence":0.30338433384895325,"max_confidence":0.3109966516494751},{"index":8,"count":4,"confidence":0.3197297975420952,"empirical_correctness":0.25,"signed_gap":0.06972979754209518,"absolute_gap":0.06972979754209518,"min_confidence":0.3165181279182434,"max_confidence":0.3240015208721161},{"index":9,"count":4,"confidence":0.33519119024276733,"empirical_correctness":0.75,"signed_gap":-0.41480880975723267,"absolute_gap":0.41480880975723267,"min_confidence":0.3251131772994995,"max_confidence":0.3454703986644745}]}},"latency_ms":{"n":43,"median":373.09079989790916,"p95":378.791890013963},"per_action":{"keep":{"n":12,"raw":{"n":12,"acceptable":12,"unsafe":0,"review":0,"automated_count":12,"automated_acceptable":12,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":1.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.7575059933447592,"upper":1.0,"confidence_level":0.95}},"effective":{"n":12,"acceptable":0,"unsafe":0,"review":12,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":12,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":2.7755575615628914e-17,"upper":0.24249400665524085,"confidence_level":0.95}}},"drop":{"n":16,"raw":{"n":16,"acceptable":0,"unsafe":16,"review":0,"automated_count":16,"automated_acceptable":0,"automated_unsafe":16,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":1.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":1.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.1936076805344365,"confidence_level":0.95}},"effective":{"n":16,"acceptable":0,"unsafe":0,"review":12,"automated_count":4,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":12,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":0.75,"automated_coverage":0.25,"conditional_correctness":0.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.75,"accuracy_wilson_95":{"lower":0.0,"upper":0.1936076805344365,"confidence_level":0.95}}},"mask":{"n":2,"raw":{"n":2,"acceptable":0,"unsafe":2,"review":0,"automated_count":2,"automated_acceptable":0,"automated_unsafe":2,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":1.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":1.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.6576197724933469,"confidence_level":0.95}},"effective":{"n":2,"acceptable":1,"unsafe":0,"review":1,"automated_count":1,"automated_acceptable":1,"automated_unsafe":0,"unnecessary_reviews":1,"accuracy":0.5,"unsafe_rate":0.0,"review_rate":0.5,"automated_coverage":0.5,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.5,"accuracy_wilson_95":{"lower":0.09453120573423074,"upper":0.9054687942657693,"confidence_level":0.95}}},"pseudonymize":{"n":2,"raw":{"n":2,"acceptable":0,"unsafe":2,"review":0,"automated_count":2,"automated_acceptable":0,"automated_unsafe":2,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":1.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":1.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.6576197724933469,"confidence_level":0.95}},"effective":{"n":2,"acceptable":0,"unsafe":0,"review":2,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":2,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.6576197724933469,"confidence_level":0.95}}},"generalize":{"n":6,"raw":{"n":6,"acceptable":0,"unsafe":4,"review":0,"automated_count":6,"automated_acceptable":0,"automated_unsafe":4,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":0.6666666666666666,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":0.6666666666666666,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":2.7755575615628914e-17,"upper":0.3903342879021653,"confidence_level":0.95}},"effective":{"n":6,"acceptable":0,"unsafe":0,"review":6,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":6,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":2.7755575615628914e-17,"upper":0.3903342879021653,"confidence_level":0.95}}},"review":{"n":0,"raw":{"n":0,"acceptable":0,"unsafe":0,"review":0,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":null,"unsafe_rate":null,"review_rate":null,"automated_coverage":null,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":null},"effective":{"n":0,"acceptable":0,"unsafe":0,"review":0,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":null,"unsafe_rate":null,"review_rate":null,"automated_coverage":null,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":null}}}},"diagnostic43-compact_no_samples":{"n":43,"raw":{"n":43,"acceptable":12,"unsafe":30,"review":1,"automated_count":42,"automated_acceptable":12,"automated_unsafe":30,"unnecessary_reviews":1,"accuracy":0.27906976744186046,"unsafe_rate":0.6976744186046512,"review_rate":0.023255813953488372,"automated_coverage":0.9767441860465116,"conditional_correctness":0.2857142857142857,"conditional_unsafe":0.7142857142857143,"unnecessary_review_rate":0.023255813953488372,"accuracy_wilson_95":{"lower":0.16746613786237488,"upper":0.42691028681878873,"confidence_level":0.95}},"effective":{"n":43,"acceptable":6,"unsafe":0,"review":39,"automated_count":4,"automated_acceptable":1,"automated_unsafe":0,"unnecessary_reviews":34,"accuracy":0.13953488372093023,"unsafe_rate":0.0,"review_rate":0.9069767441860465,"automated_coverage":0.09302325581395349,"conditional_correctness":0.25,"conditional_unsafe":0.0,"unnecessary_review_rate":0.7906976744186046,"accuracy_wilson_95":{"lower":0.06555723427964878,"upper":0.2726358796738286,"confidence_level":0.95}},"confusion":{"raw":{"keep":{"keep":12,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0},"drop":{"keep":15,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":1},"mask":{"keep":2,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0},"pseudonymize":{"keep":2,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0},"generalize":{"keep":6,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0},"review":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0}},"effective":{"keep":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":12},"drop":{"keep":0,"drop":0,"mask":3,"pseudonymize":0,"generalize":0,"review":13},"mask":{"keep":0,"drop":0,"mask":1,"pseudonymize":0,"generalize":0,"review":1},"pseudonymize":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":2},"generalize":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":6},"review":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0}}},"probability_metrics":{"singleton_n":38,"multiacceptable_n":5,"multiacceptable_coverage":0.11627906976744186,"nll":1.72668016878498,"nll_zero_mass_n":0,"brier":0.8083729754278489,"acceptable_set_n":43,"acceptable_set_nll":1.6181429123741333,"acceptable_set_zero_mass_n":0},"calibration_all_labels":{"equal_width":{"n":43,"bin_count":10,"ece":0.029302212734555078,"signed_mean_gap":-0.01769854960053465,"absolute_mean_gap":0.01769854960053465,"bins":[{"index":0,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.0,"upper":0.1,"upper_inclusive":false},{"index":1,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.1,"upper":0.2,"upper_inclusive":false},{"index":2,"count":39,"confidence":0.2561406053793736,"empirical_correctness":0.28205128205128205,"signed_gap":-0.02591067667190844,"absolute_gap":0.02591067667190844,"lower":0.2,"upper":0.3,"upper_inclusive":false},{"index":3,"count":4,"confidence":0.3123696893453598,"empirical_correctness":0.25,"signed_gap":0.0623696893453598,"absolute_gap":0.0623696893453598,"lower":0.3,"upper":0.4,"upper_inclusive":false},{"index":4,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.4,"upper":0.5,"upper_inclusive":false},{"index":5,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.5,"upper":0.6,"upper_inclusive":false},{"index":6,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.6,"upper":0.7,"upper_inclusive":false},{"index":7,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.7,"upper":0.8,"upper_inclusive":false},{"index":8,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.8,"upper":0.9,"upper_inclusive":false},{"index":9,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.9,"upper":1.0,"upper_inclusive":true}]},"equal_mass":{"n":43,"bin_count":10,"ece":0.17171617060206656,"signed_mean_gap":-0.017698549600534668,"absolute_mean_gap":0.017698549600534668,"bins":[{"index":0,"count":4,"confidence":0.21051955595612526,"empirical_correctness":0.0,"signed_gap":0.21051955595612526,"absolute_gap":0.21051955595612526,"min_confidence":0.20527908205986023,"max_confidence":0.213308647274971},{"index":1,"count":4,"confidence":0.22046280652284622,"empirical_correctness":0.5,"signed_gap":-0.2795371934771538,"absolute_gap":0.2795371934771538,"min_confidence":0.21899688243865967,"max_confidence":0.2213781774044037},{"index":2,"count":4,"confidence":0.22586382925510406,"empirical_correctness":0.25,"signed_gap":-0.024136170744895935,"absolute_gap":0.024136170744895935,"min_confidence":0.2218332588672638,"max_confidence":0.2326601892709732},{"index":3,"count":5,"confidence":0.25107147097587584,"empirical_correctness":0.2,"signed_gap":0.05107147097587583,"absolute_gap":0.05107147097587583,"min_confidence":0.24129801988601685,"max_confidence":0.2555440664291382},{"index":4,"count":4,"confidence":0.26334019005298615,"empirical_correctness":0.5,"signed_gap":-0.23665980994701385,"absolute_gap":0.23665980994701385,"min_confidence":0.2599692940711975,"max_confidence":0.2655298411846161},{"index":5,"count":4,"confidence":0.27067459374666214,"empirical_correctness":0.0,"signed_gap":0.27067459374666214,"absolute_gap":0.27067459374666214,"min_confidence":0.2674434185028076,"max_confidence":0.27391746640205383},{"index":6,"count":5,"confidence":0.2783550679683685,"empirical_correctness":0.4,"signed_gap":-0.12164493203163151,"absolute_gap":0.12164493203163151,"min_confidence":0.2756539285182953,"max_confidence":0.28140097856521606},{"index":7,"count":4,"confidence":0.2828112244606018,"empirical_correctness":0.0,"signed_gap":0.2828112244606018,"absolute_gap":0.2828112244606018,"min_confidence":0.2817404270172119,"max_confidence":0.2844172716140747},{"index":8,"count":4,"confidence":0.2874823734164238,"empirical_correctness":0.5,"signed_gap":-0.2125176265835762,"absolute_gap":0.2125176265835762,"min_confidence":0.28560763597488403,"max_confidence":0.2916690409183502},{"index":9,"count":5,"confidence":0.309442275762558,"empirical_correctness":0.4,"signed_gap":-0.09055772423744202,"absolute_gap":0.09055772423744202,"min_confidence":0.2977326214313507,"max_confidence":0.3295866549015045}]}},"calibration_singleton_labels":{"equal_width":{"n":38,"bin_count":10,"ece":0.05470620213370574,"signed_mean_gap":-0.05470620213370574,"absolute_mean_gap":0.05470620213370574,"bins":[{"index":0,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.0,"upper":0.1,"upper_inclusive":false},{"index":1,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.1,"upper":0.2,"upper_inclusive":false},{"index":2,"count":35,"confidence":0.2567830319915499,"empirical_correctness":0.3142857142857143,"signed_gap":-0.05750268229416439,"absolute_gap":0.05750268229416439,"lower":0.2,"upper":0.3,"upper_inclusive":false},{"index":3,"count":3,"confidence":0.3112527330716451,"empirical_correctness":0.3333333333333333,"signed_gap":-0.022080600261688232,"absolute_gap":0.022080600261688232,"lower":0.3,"upper":0.4,"upper_inclusive":false},{"index":4,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.4,"upper":0.5,"upper_inclusive":false},{"index":5,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.5,"upper":0.6,"upper_inclusive":false},{"index":6,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.6,"upper":0.7,"upper_inclusive":false},{"index":7,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.7,"upper":0.8,"upper_inclusive":false},{"index":8,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.8,"upper":0.9,"upper_inclusive":false},{"index":9,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.9,"upper":1.0,"upper_inclusive":true}]},"equal_mass":{"n":38,"bin_count":10,"ece":0.19117542826815656,"signed_mean_gap":-0.05470620213370574,"absolute_mean_gap":0.05470620213370574,"bins":[{"index":0,"count":3,"confidence":0.20958985884984335,"empirical_correctness":0.0,"signed_gap":0.20958985884984335,"absolute_gap":0.20958985884984335,"min_confidence":0.20527908205986023,"max_confidence":0.21296808123588562},{"index":1,"count":4,"confidence":0.21847280487418175,"empirical_correctness":0.5,"signed_gap":-0.28152719512581825,"absolute_gap":0.28152719512581825,"min_confidence":0.213308647274971,"max_confidence":0.2213781774044037},{"index":2,"count":4,"confidence":0.22586382925510406,"empirical_correctness":0.25,"signed_gap":-0.024136170744895935,"absolute_gap":0.024136170744895935,"min_confidence":0.2218332588672638,"max_confidence":0.2326601892709732},{"index":3,"count":4,"confidence":0.2535148337483406,"empirical_correctness":0.25,"signed_gap":0.0035148337483406067,"absolute_gap":0.0035148337483406067,"min_confidence":0.250076025724411,"max_confidence":0.2555440664291382},{"index":4,"count":4,"confidence":0.26334019005298615,"empirical_correctness":0.5,"signed_gap":-0.23665980994701385,"absolute_gap":0.23665980994701385,"min_confidence":0.2599692940711975,"max_confidence":0.2655298411846161},{"index":5,"count":3,"confidence":0.27294716238975525,"empirical_correctness":0.0,"signed_gap":0.27294716238975525,"absolute_gap":0.27294716238975525,"min_confidence":0.2692700922489166,"max_confidence":0.2756539285182953},{"index":6,"count":4,"confidence":0.27903035283088684,"empirical_correctness":0.5,"signed_gap":-0.22096964716911316,"absolute_gap":0.22096964716911316,"min_confidence":0.27740612626075745,"max_confidence":0.28140097856521606},{"index":7,"count":4,"confidence":0.2828112244606018,"empirical_correctness":0.0,"signed_gap":0.2828112244606018,"absolute_gap":0.2828112244606018,"min_confidence":0.2817404270172119,"max_confidence":0.2844172716140747},{"index":8,"count":4,"confidence":0.2874823734164238,"empirical_correctness":0.5,"signed_gap":-0.2125176265835762,"absolute_gap":0.2125176265835762,"min_confidence":0.28560763597488403,"max_confidence":0.2916690409183502},{"index":9,"count":4,"confidence":0.3078727051615715,"empirical_correctness":0.5,"signed_gap":-0.1921272948384285,"absolute_gap":0.1921272948384285,"min_confidence":0.2977326214313507,"max_confidence":0.3295866549015045}]}},"latency_ms":{"n":43,"median":415.87569983676076,"p95":423.68664008099586},"per_action":{"keep":{"n":12,"raw":{"n":12,"acceptable":12,"unsafe":0,"review":0,"automated_count":12,"automated_acceptable":12,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":1.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.7575059933447592,"upper":1.0,"confidence_level":0.95}},"effective":{"n":12,"acceptable":0,"unsafe":0,"review":12,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":12,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":2.7755575615628914e-17,"upper":0.24249400665524085,"confidence_level":0.95}}},"drop":{"n":16,"raw":{"n":16,"acceptable":0,"unsafe":15,"review":1,"automated_count":15,"automated_acceptable":0,"automated_unsafe":15,"unnecessary_reviews":1,"accuracy":0.0,"unsafe_rate":0.9375,"review_rate":0.0625,"automated_coverage":0.9375,"conditional_correctness":0.0,"conditional_unsafe":1.0,"unnecessary_review_rate":0.0625,"accuracy_wilson_95":{"lower":0.0,"upper":0.1936076805344365,"confidence_level":0.95}},"effective":{"n":16,"acceptable":0,"unsafe":0,"review":13,"automated_count":3,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":13,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":0.8125,"automated_coverage":0.1875,"conditional_correctness":0.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.8125,"accuracy_wilson_95":{"lower":0.0,"upper":0.1936076805344365,"confidence_level":0.95}}},"mask":{"n":2,"raw":{"n":2,"acceptable":0,"unsafe":2,"review":0,"automated_count":2,"automated_acceptable":0,"automated_unsafe":2,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":1.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":1.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.6576197724933469,"confidence_level":0.95}},"effective":{"n":2,"acceptable":1,"unsafe":0,"review":1,"automated_count":1,"automated_acceptable":1,"automated_unsafe":0,"unnecessary_reviews":1,"accuracy":0.5,"unsafe_rate":0.0,"review_rate":0.5,"automated_coverage":0.5,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.5,"accuracy_wilson_95":{"lower":0.09453120573423074,"upper":0.9054687942657693,"confidence_level":0.95}}},"pseudonymize":{"n":2,"raw":{"n":2,"acceptable":0,"unsafe":2,"review":0,"automated_count":2,"automated_acceptable":0,"automated_unsafe":2,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":1.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":1.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.6576197724933469,"confidence_level":0.95}},"effective":{"n":2,"acceptable":0,"unsafe":0,"review":2,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":2,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.6576197724933469,"confidence_level":0.95}}},"generalize":{"n":6,"raw":{"n":6,"acceptable":0,"unsafe":6,"review":0,"automated_count":6,"automated_acceptable":0,"automated_unsafe":6,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":1.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":1.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":2.7755575615628914e-17,"upper":0.3903342879021653,"confidence_level":0.95}},"effective":{"n":6,"acceptable":0,"unsafe":0,"review":6,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":6,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":2.7755575615628914e-17,"upper":0.3903342879021653,"confidence_level":0.95}}},"review":{"n":0,"raw":{"n":0,"acceptable":0,"unsafe":0,"review":0,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":null,"unsafe_rate":null,"review_rate":null,"automated_coverage":null,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":null},"effective":{"n":0,"acceptable":0,"unsafe":0,"review":0,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":null,"unsafe_rate":null,"review_rate":null,"automated_coverage":null,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":null}}}},"challenge56-no_samples":{"n":56,"raw":{"n":56,"acceptable":15,"unsafe":40,"review":0,"automated_count":56,"automated_acceptable":15,"automated_unsafe":40,"unnecessary_reviews":0,"accuracy":0.26785714285714285,"unsafe_rate":0.7142857142857143,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.26785714285714285,"conditional_unsafe":0.7142857142857143,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.16957305418501728,"upper":0.39594555929155156,"confidence_level":0.95}},"effective":{"n":56,"acceptable":3,"unsafe":0,"review":55,"automated_count":1,"automated_acceptable":1,"automated_unsafe":0,"unnecessary_reviews":53,"accuracy":0.05357142857142857,"unsafe_rate":0.0,"review_rate":0.9821428571428571,"automated_coverage":0.017857142857142856,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.9464285714285714,"accuracy_wilson_95":{"lower":0.018385781382109126,"upper":0.14607309068821533,"confidence_level":0.95}},"confusion":{"raw":{"keep":{"keep":8,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0},"drop":{"keep":21,"drop":0,"mask":0,"pseudonymize":0,"generalize":1,"review":0},"mask":{"keep":2,"drop":0,"mask":6,"pseudonymize":0,"generalize":0,"review":0},"pseudonymize":{"keep":7,"drop":0,"mask":0,"pseudonymize":1,"generalize":0,"review":0},"generalize":{"keep":8,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0},"review":{"keep":2,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0}},"effective":{"keep":{"keep":1,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":7},"drop":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":22},"mask":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":8},"pseudonymize":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":8},"generalize":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":8},"review":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":2}}},"probability_metrics":{"singleton_n":56,"multiacceptable_n":0,"multiacceptable_coverage":0.0,"nll":1.6482939587885415,"nll_zero_mass_n":0,"brier":0.7837024316584158,"acceptable_set_n":56,"acceptable_set_nll":1.6482939587885415,"acceptable_set_zero_mass_n":0},"calibration_all_labels":{"equal_width":{"n":56,"bin_count":10,"ece":0.12938931291656836,"signed_mean_gap":0.045589369588664594,"absolute_mean_gap":0.045589369588664594,"bins":[{"index":0,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.0,"upper":0.1,"upper_inclusive":false},{"index":1,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.1,"upper":0.2,"upper_inclusive":false},{"index":2,"count":28,"confidence":0.265940073877573,"empirical_correctness":0.25,"signed_gap":0.015940073877573013,"absolute_gap":0.015940073877573013,"lower":0.2,"upper":0.3,"upper_inclusive":false},{"index":3,"count":22,"confidence":0.3387764109806581,"empirical_correctness":0.13636363636363635,"signed_gap":0.20241277461702173,"absolute_gap":0.20241277461702173,"lower":0.3,"upper":0.4,"upper_inclusive":false},{"index":4,"count":6,"confidence":0.44226693113644916,"empirical_correctness":0.8333333333333334,"signed_gap":-0.3910664021968842,"absolute_gap":0.3910664021968842,"lower":0.4,"upper":0.5,"upper_inclusive":false},{"index":5,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.5,"upper":0.6,"upper_inclusive":false},{"index":6,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.6,"upper":0.7,"upper_inclusive":false},{"index":7,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.7,"upper":0.8,"upper_inclusive":false},{"index":8,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.8,"upper":0.9,"upper_inclusive":false},{"index":9,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.9,"upper":1.0,"upper_inclusive":true}]},"equal_mass":{"n":56,"bin_count":10,"ece":0.23682719442461217,"signed_mean_gap":0.04558936958866459,"absolute_mean_gap":0.04558936958866459,"bins":[{"index":0,"count":5,"confidence":0.22040956914424897,"empirical_correctness":0.4,"signed_gap":-0.17959043085575105,"absolute_gap":0.17959043085575105,"min_confidence":0.20425711572170258,"max_confidence":0.23378176987171173},{"index":1,"count":6,"confidence":0.2533874462048213,"empirical_correctness":0.5,"signed_gap":-0.2466125537951787,"absolute_gap":0.2466125537951787,"min_confidence":0.2404872179031372,"max_confidence":0.2655189037322998},{"index":2,"count":5,"confidence":0.27387335896492004,"empirical_correctness":0.4,"signed_gap":-0.12612664103507998,"absolute_gap":0.12612664103507998,"min_confidence":0.26856157183647156,"max_confidence":0.2771475613117218},{"index":3,"count":6,"confidence":0.28342414398988086,"empirical_correctness":0.0,"signed_gap":0.28342414398988086,"absolute_gap":0.28342414398988086,"min_confidence":0.277995765209198,"max_confidence":0.28740575909614563},{"index":4,"count":6,"confidence":0.2923396478096644,"empirical_correctness":0.0,"signed_gap":0.2923396478096644,"absolute_gap":0.2923396478096644,"min_confidence":0.28903594613075256,"max_confidence":0.29685401916503906},{"index":5,"count":5,"confidence":0.303915935754776,"empirical_correctness":0.0,"signed_gap":0.303915935754776,"absolute_gap":0.303915935754776,"min_confidence":0.3001289963722229,"max_confidence":0.30627232789993286},{"index":6,"count":6,"confidence":0.3188281903664271,"empirical_correctness":0.0,"signed_gap":0.3188281903664271,"absolute_gap":0.3188281903664271,"min_confidence":0.3115270137786865,"max_confidence":0.330157071352005},{"index":7,"count":5,"confidence":0.3487002313137054,"empirical_correctness":0.2,"signed_gap":0.1487002313137054,"absolute_gap":0.1487002313137054,"min_confidence":0.3349592685699463,"max_confidence":0.3637102246284485},{"index":8,"count":6,"confidence":0.37950517733891803,"empirical_correctness":0.3333333333333333,"signed_gap":0.04617184400558472,"absolute_gap":0.04617184400558472,"min_confidence":0.36373159289360046,"max_confidence":0.3965893089771271},{"index":9,"count":6,"confidence":0.44226693113644916,"empirical_correctness":0.8333333333333334,"signed_gap":-0.3910664021968842,"absolute_gap":0.3910664021968842,"min_confidence":0.416454941034317,"max_confidence":0.49231311678886414}]}},"calibration_singleton_labels":{"equal_width":{"n":56,"bin_count":10,"ece":0.12938931291656836,"signed_mean_gap":0.045589369588664594,"absolute_mean_gap":0.045589369588664594,"bins":[{"index":0,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.0,"upper":0.1,"upper_inclusive":false},{"index":1,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.1,"upper":0.2,"upper_inclusive":false},{"index":2,"count":28,"confidence":0.265940073877573,"empirical_correctness":0.25,"signed_gap":0.015940073877573013,"absolute_gap":0.015940073877573013,"lower":0.2,"upper":0.3,"upper_inclusive":false},{"index":3,"count":22,"confidence":0.3387764109806581,"empirical_correctness":0.13636363636363635,"signed_gap":0.20241277461702173,"absolute_gap":0.20241277461702173,"lower":0.3,"upper":0.4,"upper_inclusive":false},{"index":4,"count":6,"confidence":0.44226693113644916,"empirical_correctness":0.8333333333333334,"signed_gap":-0.3910664021968842,"absolute_gap":0.3910664021968842,"lower":0.4,"upper":0.5,"upper_inclusive":false},{"index":5,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.5,"upper":0.6,"upper_inclusive":false},{"index":6,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.6,"upper":0.7,"upper_inclusive":false},{"index":7,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.7,"upper":0.8,"upper_inclusive":false},{"index":8,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.8,"upper":0.9,"upper_inclusive":false},{"index":9,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.9,"upper":1.0,"upper_inclusive":true}]},"equal_mass":{"n":56,"bin_count":10,"ece":0.23682719442461217,"signed_mean_gap":0.04558936958866459,"absolute_mean_gap":0.04558936958866459,"bins":[{"index":0,"count":5,"confidence":0.22040956914424897,"empirical_correctness":0.4,"signed_gap":-0.17959043085575105,"absolute_gap":0.17959043085575105,"min_confidence":0.20425711572170258,"max_confidence":0.23378176987171173},{"index":1,"count":6,"confidence":0.2533874462048213,"empirical_correctness":0.5,"signed_gap":-0.2466125537951787,"absolute_gap":0.2466125537951787,"min_confidence":0.2404872179031372,"max_confidence":0.2655189037322998},{"index":2,"count":5,"confidence":0.27387335896492004,"empirical_correctness":0.4,"signed_gap":-0.12612664103507998,"absolute_gap":0.12612664103507998,"min_confidence":0.26856157183647156,"max_confidence":0.2771475613117218},{"index":3,"count":6,"confidence":0.28342414398988086,"empirical_correctness":0.0,"signed_gap":0.28342414398988086,"absolute_gap":0.28342414398988086,"min_confidence":0.277995765209198,"max_confidence":0.28740575909614563},{"index":4,"count":6,"confidence":0.2923396478096644,"empirical_correctness":0.0,"signed_gap":0.2923396478096644,"absolute_gap":0.2923396478096644,"min_confidence":0.28903594613075256,"max_confidence":0.29685401916503906},{"index":5,"count":5,"confidence":0.303915935754776,"empirical_correctness":0.0,"signed_gap":0.303915935754776,"absolute_gap":0.303915935754776,"min_confidence":0.3001289963722229,"max_confidence":0.30627232789993286},{"index":6,"count":6,"confidence":0.3188281903664271,"empirical_correctness":0.0,"signed_gap":0.3188281903664271,"absolute_gap":0.3188281903664271,"min_confidence":0.3115270137786865,"max_confidence":0.330157071352005},{"index":7,"count":5,"confidence":0.3487002313137054,"empirical_correctness":0.2,"signed_gap":0.1487002313137054,"absolute_gap":0.1487002313137054,"min_confidence":0.3349592685699463,"max_confidence":0.3637102246284485},{"index":8,"count":6,"confidence":0.37950517733891803,"empirical_correctness":0.3333333333333333,"signed_gap":0.04617184400558472,"absolute_gap":0.04617184400558472,"min_confidence":0.36373159289360046,"max_confidence":0.3965893089771271},{"index":9,"count":6,"confidence":0.44226693113644916,"empirical_correctness":0.8333333333333334,"signed_gap":-0.3910664021968842,"absolute_gap":0.3910664021968842,"min_confidence":0.416454941034317,"max_confidence":0.49231311678886414}]}},"latency_ms":{"n":56,"median":304.485350032337,"p95":376.269549888093},"per_action":{"keep":{"n":8,"raw":{"n":8,"acceptable":8,"unsafe":0,"review":0,"automated_count":8,"automated_acceptable":8,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":1.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.6755924351161198,"upper":1.0,"confidence_level":0.95}},"effective":{"n":8,"acceptable":1,"unsafe":0,"review":7,"automated_count":1,"automated_acceptable":1,"automated_unsafe":0,"unnecessary_reviews":7,"accuracy":0.125,"unsafe_rate":0.0,"review_rate":0.875,"automated_coverage":0.125,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.875,"accuracy_wilson_95":{"lower":0.02241749145005667,"upper":0.4708881822128535,"confidence_level":0.95}}},"drop":{"n":22,"raw":{"n":22,"acceptable":0,"unsafe":21,"review":0,"automated_count":22,"automated_acceptable":0,"automated_unsafe":21,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":0.9545454545454546,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":0.9545454545454546,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.14865487460862087,"confidence_level":0.95}},"effective":{"n":22,"acceptable":0,"unsafe":0,"review":22,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":22,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.14865487460862087,"confidence_level":0.95}}},"mask":{"n":8,"raw":{"n":8,"acceptable":6,"unsafe":2,"review":0,"automated_count":8,"automated_acceptable":6,"automated_unsafe":2,"unnecessary_reviews":0,"accuracy":0.75,"unsafe_rate":0.25,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.75,"conditional_unsafe":0.25,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.40927543031016883,"upper":0.9285207872478909,"confidence_level":0.95}},"effective":{"n":8,"acceptable":0,"unsafe":0,"review":8,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":8,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.32440756488388023,"confidence_level":0.95}}},"pseudonymize":{"n":8,"raw":{"n":8,"acceptable":1,"unsafe":7,"review":0,"automated_count":8,"automated_acceptable":1,"automated_unsafe":7,"unnecessary_reviews":0,"accuracy":0.125,"unsafe_rate":0.875,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.125,"conditional_unsafe":0.875,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.02241749145005667,"upper":0.4708881822128535,"confidence_level":0.95}},"effective":{"n":8,"acceptable":0,"unsafe":0,"review":8,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":8,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.32440756488388023,"confidence_level":0.95}}},"generalize":{"n":8,"raw":{"n":8,"acceptable":0,"unsafe":8,"review":0,"automated_count":8,"automated_acceptable":0,"automated_unsafe":8,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":1.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":1.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.32440756488388023,"confidence_level":0.95}},"effective":{"n":8,"acceptable":0,"unsafe":0,"review":8,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":8,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.32440756488388023,"confidence_level":0.95}}},"review":{"n":2,"raw":{"n":2,"acceptable":0,"unsafe":2,"review":0,"automated_count":2,"automated_acceptable":0,"automated_unsafe":2,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":1.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":1.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.6576197724933469,"confidence_level":0.95}},"effective":{"n":2,"acceptable":2,"unsafe":0,"review":2,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":1.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.34238022750665303,"upper":1.0,"confidence_level":0.95}}}}},"challenge56-compact_no_samples":{"n":56,"raw":{"n":56,"acceptable":16,"unsafe":32,"review":1,"automated_count":55,"automated_acceptable":16,"automated_unsafe":32,"unnecessary_reviews":1,"accuracy":0.2857142857142857,"unsafe_rate":0.5714285714285714,"review_rate":0.017857142857142856,"automated_coverage":0.9821428571428571,"conditional_correctness":0.2909090909090909,"conditional_unsafe":0.5818181818181818,"unnecessary_review_rate":0.017857142857142856,"accuracy_wilson_95":{"lower":0.18418775143858906,"upper":0.4147525071551667,"confidence_level":0.95}},"effective":{"n":56,"acceptable":2,"unsafe":0,"review":56,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":54,"accuracy":0.03571428571428571,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":0.9642857142857143,"accuracy_wilson_95":{"lower":0.00984942514823639,"upper":0.12118780180490107,"confidence_level":0.95}},"confusion":{"raw":{"keep":{"keep":8,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0},"drop":{"keep":16,"drop":0,"mask":6,"pseudonymize":0,"generalize":0,"review":0},"mask":{"keep":0,"drop":0,"mask":8,"pseudonymize":0,"generalize":0,"review":0},"pseudonymize":{"keep":7,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":1},"generalize":{"keep":7,"drop":0,"mask":1,"pseudonymize":0,"generalize":0,"review":0},"review":{"keep":2,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0}},"effective":{"keep":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":8},"drop":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":22},"mask":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":8},"pseudonymize":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":8},"generalize":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":8},"review":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":2}}},"probability_metrics":{"singleton_n":56,"multiacceptable_n":0,"multiacceptable_coverage":0.0,"nll":1.8534177433136103,"nll_zero_mass_n":0,"brier":0.851623406367355,"acceptable_set_n":56,"acceptable_set_nll":1.8534177433136103,"acceptable_set_zero_mass_n":0},"calibration_all_labels":{"equal_width":{"n":56,"bin_count":10,"ece":0.10117753329021593,"signed_mean_gap":0.020645134683166224,"absolute_mean_gap":0.020645134683166224,"bins":[{"index":0,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.0,"upper":0.1,"upper_inclusive":false},{"index":1,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.1,"upper":0.2,"upper_inclusive":false},{"index":2,"count":28,"confidence":0.27661045853580746,"empirical_correctness":0.35714285714285715,"signed_gap":-0.0805323986070497,"absolute_gap":0.0805323986070497,"lower":0.2,"upper":0.3,"upper_inclusive":false},{"index":3,"count":27,"confidence":0.3334147643159937,"empirical_correctness":0.2222222222222222,"signed_gap":0.1111925420937715,"absolute_gap":0.1111925420937715,"lower":0.3,"upper":0.4,"upper_inclusive":false},{"index":4,"count":1,"confidence":0.4088360667228699,"empirical_correctness":0.0,"signed_gap":0.4088360667228699,"absolute_gap":0.4088360667228699,"lower":0.4,"upper":0.5,"upper_inclusive":false},{"index":5,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.5,"upper":0.6,"upper_inclusive":false},{"index":6,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.6,"upper":0.7,"upper_inclusive":false},{"index":7,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.7,"upper":0.8,"upper_inclusive":false},{"index":8,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.8,"upper":0.9,"upper_inclusive":false},{"index":9,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.9,"upper":1.0,"upper_inclusive":true}]},"equal_mass":{"n":56,"bin_count":10,"ece":0.1607531580541815,"signed_mean_gap":0.020645134683166238,"absolute_mean_gap":0.020645134683166238,"bins":[{"index":0,"count":5,"confidence":0.2414686858654022,"empirical_correctness":0.0,"signed_gap":0.2414686858654022,"absolute_gap":0.2414686858654022,"min_confidence":0.23196542263031006,"max_confidence":0.24947847425937653},{"index":1,"count":6,"confidence":0.2703113208214442,"empirical_correctness":0.3333333333333333,"signed_gap":-0.0630220125118891,"absolute_gap":0.0630220125118891,"min_confidence":0.26293644309043884,"max_confidence":0.27707985043525696},{"index":2,"count":5,"confidence":0.28001485466957093,"empirical_correctness":0.4,"signed_gap":-0.11998514533042909,"absolute_gap":0.11998514533042909,"min_confidence":0.2782643735408783,"max_confidence":0.2832319140434265},{"index":3,"count":6,"confidence":0.28843415280183154,"empirical_correctness":0.5,"signed_gap":-0.21156584719816846,"absolute_gap":0.21156584719816846,"min_confidence":0.28325551748275757,"max_confidence":0.29473891854286194},{"index":4,"count":6,"confidence":0.2975337157646815,"empirical_correctness":0.5,"signed_gap":-0.20246628423531848,"absolute_gap":0.20246628423531848,"min_confidence":0.2952788770198822,"max_confidence":0.2999013662338257},{"index":5,"count":5,"confidence":0.30784518718719484,"empirical_correctness":0.4,"signed_gap":-0.09215481281280519,"absolute_gap":0.09215481281280519,"min_confidence":0.3000047504901886,"max_confidence":0.3112491965293884},{"index":6,"count":6,"confidence":0.3199282189210256,"empirical_correctness":0.16666666666666666,"signed_gap":0.15326155225435892,"absolute_gap":0.15326155225435892,"min_confidence":0.3161699175834656,"max_confidence":0.3241409957408905},{"index":7,"count":5,"confidence":0.3259990870952606,"empirical_correctness":0.0,"signed_gap":0.3259990870952606,"absolute_gap":0.3259990870952606,"min_confidence":0.3244802951812744,"max_confidence":0.32859063148498535},{"index":8,"count":6,"confidence":0.34779661397139233,"empirical_correctness":0.3333333333333333,"signed_gap":0.014463280638059017,"absolute_gap":0.014463280638059017,"min_confidence":0.3434607982635498,"max_confidence":0.352001428604126},{"index":9,"count":6,"confidence":0.37257738908131915,"empirical_correctness":0.16666666666666666,"signed_gap":0.2059107224146525,"absolute_gap":0.2059107224146525,"min_confidence":0.3528831899166107,"max_confidence":0.4088360667228699}]}},"calibration_singleton_labels":{"equal_width":{"n":56,"bin_count":10,"ece":0.10117753329021593,"signed_mean_gap":0.020645134683166224,"absolute_mean_gap":0.020645134683166224,"bins":[{"index":0,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.0,"upper":0.1,"upper_inclusive":false},{"index":1,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.1,"upper":0.2,"upper_inclusive":false},{"index":2,"count":28,"confidence":0.27661045853580746,"empirical_correctness":0.35714285714285715,"signed_gap":-0.0805323986070497,"absolute_gap":0.0805323986070497,"lower":0.2,"upper":0.3,"upper_inclusive":false},{"index":3,"count":27,"confidence":0.3334147643159937,"empirical_correctness":0.2222222222222222,"signed_gap":0.1111925420937715,"absolute_gap":0.1111925420937715,"lower":0.3,"upper":0.4,"upper_inclusive":false},{"index":4,"count":1,"confidence":0.4088360667228699,"empirical_correctness":0.0,"signed_gap":0.4088360667228699,"absolute_gap":0.4088360667228699,"lower":0.4,"upper":0.5,"upper_inclusive":false},{"index":5,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.5,"upper":0.6,"upper_inclusive":false},{"index":6,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.6,"upper":0.7,"upper_inclusive":false},{"index":7,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.7,"upper":0.8,"upper_inclusive":false},{"index":8,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.8,"upper":0.9,"upper_inclusive":false},{"index":9,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.9,"upper":1.0,"upper_inclusive":true}]},"equal_mass":{"n":56,"bin_count":10,"ece":0.1607531580541815,"signed_mean_gap":0.020645134683166238,"absolute_mean_gap":0.020645134683166238,"bins":[{"index":0,"count":5,"confidence":0.2414686858654022,"empirical_correctness":0.0,"signed_gap":0.2414686858654022,"absolute_gap":0.2414686858654022,"min_confidence":0.23196542263031006,"max_confidence":0.24947847425937653},{"index":1,"count":6,"confidence":0.2703113208214442,"empirical_correctness":0.3333333333333333,"signed_gap":-0.0630220125118891,"absolute_gap":0.0630220125118891,"min_confidence":0.26293644309043884,"max_confidence":0.27707985043525696},{"index":2,"count":5,"confidence":0.28001485466957093,"empirical_correctness":0.4,"signed_gap":-0.11998514533042909,"absolute_gap":0.11998514533042909,"min_confidence":0.2782643735408783,"max_confidence":0.2832319140434265},{"index":3,"count":6,"confidence":0.28843415280183154,"empirical_correctness":0.5,"signed_gap":-0.21156584719816846,"absolute_gap":0.21156584719816846,"min_confidence":0.28325551748275757,"max_confidence":0.29473891854286194},{"index":4,"count":6,"confidence":0.2975337157646815,"empirical_correctness":0.5,"signed_gap":-0.20246628423531848,"absolute_gap":0.20246628423531848,"min_confidence":0.2952788770198822,"max_confidence":0.2999013662338257},{"index":5,"count":5,"confidence":0.30784518718719484,"empirical_correctness":0.4,"signed_gap":-0.09215481281280519,"absolute_gap":0.09215481281280519,"min_confidence":0.3000047504901886,"max_confidence":0.3112491965293884},{"index":6,"count":6,"confidence":0.3199282189210256,"empirical_correctness":0.16666666666666666,"signed_gap":0.15326155225435892,"absolute_gap":0.15326155225435892,"min_confidence":0.3161699175834656,"max_confidence":0.3241409957408905},{"index":7,"count":5,"confidence":0.3259990870952606,"empirical_correctness":0.0,"signed_gap":0.3259990870952606,"absolute_gap":0.3259990870952606,"min_confidence":0.3244802951812744,"max_confidence":0.32859063148498535},{"index":8,"count":6,"confidence":0.34779661397139233,"empirical_correctness":0.3333333333333333,"signed_gap":0.014463280638059017,"absolute_gap":0.014463280638059017,"min_confidence":0.3434607982635498,"max_confidence":0.352001428604126},{"index":9,"count":6,"confidence":0.37257738908131915,"empirical_correctness":0.16666666666666666,"signed_gap":0.2059107224146525,"absolute_gap":0.2059107224146525,"min_confidence":0.3528831899166107,"max_confidence":0.4088360667228699}]}},"latency_ms":{"n":56,"median":377.2568500135094,"p95":385.5715999379754},"per_action":{"keep":{"n":8,"raw":{"n":8,"acceptable":8,"unsafe":0,"review":0,"automated_count":8,"automated_acceptable":8,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":1.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.6755924351161198,"upper":1.0,"confidence_level":0.95}},"effective":{"n":8,"acceptable":0,"unsafe":0,"review":8,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":8,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.32440756488388023,"confidence_level":0.95}}},"drop":{"n":22,"raw":{"n":22,"acceptable":0,"unsafe":16,"review":0,"automated_count":22,"automated_acceptable":0,"automated_unsafe":16,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":0.7272727272727273,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":0.7272727272727273,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.14865487460862087,"confidence_level":0.95}},"effective":{"n":22,"acceptable":0,"unsafe":0,"review":22,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":22,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.14865487460862087,"confidence_level":0.95}}},"mask":{"n":8,"raw":{"n":8,"acceptable":8,"unsafe":0,"review":0,"automated_count":8,"automated_acceptable":8,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":1.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.6755924351161198,"upper":1.0,"confidence_level":0.95}},"effective":{"n":8,"acceptable":0,"unsafe":0,"review":8,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":8,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.32440756488388023,"confidence_level":0.95}}},"pseudonymize":{"n":8,"raw":{"n":8,"acceptable":0,"unsafe":7,"review":1,"automated_count":7,"automated_acceptable":0,"automated_unsafe":7,"unnecessary_reviews":1,"accuracy":0.0,"unsafe_rate":0.875,"review_rate":0.125,"automated_coverage":0.875,"conditional_correctness":0.0,"conditional_unsafe":1.0,"unnecessary_review_rate":0.125,"accuracy_wilson_95":{"lower":0.0,"upper":0.32440756488388023,"confidence_level":0.95}},"effective":{"n":8,"acceptable":0,"unsafe":0,"review":8,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":8,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.32440756488388023,"confidence_level":0.95}}},"generalize":{"n":8,"raw":{"n":8,"acceptable":0,"unsafe":7,"review":0,"automated_count":8,"automated_acceptable":0,"automated_unsafe":7,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":0.875,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":0.875,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.32440756488388023,"confidence_level":0.95}},"effective":{"n":8,"acceptable":0,"unsafe":0,"review":8,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":8,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.32440756488388023,"confidence_level":0.95}}},"review":{"n":2,"raw":{"n":2,"acceptable":0,"unsafe":2,"review":0,"automated_count":2,"automated_acceptable":0,"automated_unsafe":2,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":1.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":1.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.6576197724933469,"confidence_level":0.95}},"effective":{"n":2,"acceptable":2,"unsafe":0,"review":2,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":1.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.34238022750665303,"upper":1.0,"confidence_level":0.95}}}}}},"composite":{"n":198,"scope":"descriptive sum across four correlated panels","raw":{"n":198,"acceptable":55,"unsafe":131,"review":2,"automated_count":196,"automated_acceptable":55,"automated_unsafe":131,"unnecessary_reviews":2,"accuracy":0.2777777777777778,"unsafe_rate":0.6616161616161617,"review_rate":0.010101010101010102,"automated_coverage":0.98989898989899,"conditional_correctness":0.28061224489795916,"conditional_unsafe":0.6683673469387755,"unnecessary_review_rate":0.010101010101010102},"effective":{"n":198,"acceptable":17,"unsafe":0,"review":188,"automated_count":10,"automated_acceptable":3,"automated_unsafe":0,"unnecessary_reviews":174,"accuracy":0.08585858585858586,"unsafe_rate":0.0,"review_rate":0.9494949494949495,"automated_coverage":0.050505050505050504,"conditional_correctness":0.3,"conditional_unsafe":0.0,"unnecessary_review_rate":0.8787878787878788}}},{"id":"cross-encoder--nli-MiniLM2-L6-H768","platform":"Local","model_id":"cross-encoder/nli-MiniLM2-L6-H768","revision":"b95119ce93d3e065de6214e38cd4a97b0f2f2c6d","loading":"float32 / none","scoring_kind":"Adapted NLI","settings":{"dtype":"float32","quantization":null,"max_tokens":4096,"effective_max_tokens":512,"temperature":1.0,"scoring_batch_size":1,"trust_remote_code":false,"local_files_only":true,"use_safetensors":true,"guard_threshold":0.45,"seed":0,"tf32":false},"scoring_metadata":{"backend":"nli","protocol":"hf-local-sweep-backends-v1","probability_source":"adapted_entailment_softmax_not_calibrated_sixway","latency_scope":"synchronous_forward_scoring_including_transfer_excluding_tokenization","precision":"float32","equal_mass_convention":"rank bins with floor boundaries at floor(i*n/k), k=min(10,n//2); minimum two rows"},"load_ms":5579.127999953926,"peak_allocated_bytes":380356608,"peak_reserved_bytes":406847488,"panels":{"diagnostic43-no_samples":{"n":43,"raw":{"n":43,"acceptable":6,"unsafe":23,"review":0,"automated_count":43,"automated_acceptable":6,"automated_unsafe":23,"unnecessary_reviews":0,"accuracy":0.13953488372093023,"unsafe_rate":0.5348837209302325,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.13953488372093023,"conditional_unsafe":0.5348837209302325,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.06555723427964878,"upper":0.2726358796738286,"confidence_level":0.95}},"effective":{"n":43,"acceptable":5,"unsafe":0,"review":40,"automated_count":3,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":35,"accuracy":0.11627906976744186,"unsafe_rate":0.0,"review_rate":0.9302325581395349,"automated_coverage":0.06976744186046512,"conditional_correctness":0.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.813953488372093,"accuracy_wilson_95":{"lower":0.05070427739804166,"upper":0.24479161810082128,"confidence_level":0.95}},"confusion":{"raw":{"keep":{"keep":5,"drop":0,"mask":0,"pseudonymize":0,"generalize":7,"review":0},"drop":{"keep":9,"drop":0,"mask":0,"pseudonymize":0,"generalize":7,"review":0},"mask":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":2,"review":0},"pseudonymize":{"keep":2,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0},"generalize":{"keep":5,"drop":0,"mask":0,"pseudonymize":0,"generalize":1,"review":0},"review":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0}},"effective":{"keep":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":12},"drop":{"keep":0,"drop":0,"mask":3,"pseudonymize":0,"generalize":0,"review":13},"mask":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":2},"pseudonymize":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":2},"generalize":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":6},"review":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0}}},"probability_metrics":{"singleton_n":38,"multiacceptable_n":5,"multiacceptable_coverage":0.11627906976744186,"nll":1.8103892677841202,"nll_zero_mass_n":0,"brier":0.8384916577708235,"acceptable_set_n":43,"acceptable_set_nll":1.6971540219049297,"acceptable_set_zero_mass_n":0},"calibration_all_labels":{"equal_width":{"n":43,"bin_count":10,"ece":0.06846364119718241,"signed_mean_gap":0.06846364119718241,"absolute_mean_gap":0.06846364119718241,"bins":[{"index":0,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.0,"upper":0.1,"upper_inclusive":false},{"index":1,"count":28,"confidence":0.1959335862525872,"empirical_correctness":0.14285714285714285,"signed_gap":0.053076443395444345,"absolute_gap":0.053076443395444345,"lower":0.1,"upper":0.2,"upper_inclusive":false},{"index":2,"count":15,"confidence":0.23051974376042683,"empirical_correctness":0.13333333333333333,"signed_gap":0.0971864104270935,"absolute_gap":0.0971864104270935,"lower":0.2,"upper":0.3,"upper_inclusive":false},{"index":3,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.3,"upper":0.4,"upper_inclusive":false},{"index":4,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.4,"upper":0.5,"upper_inclusive":false},{"index":5,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.5,"upper":0.6,"upper_inclusive":false},{"index":6,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.6,"upper":0.7,"upper_inclusive":false},{"index":7,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.7,"upper":0.8,"upper_inclusive":false},{"index":8,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.8,"upper":0.9,"upper_inclusive":false},{"index":9,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.9,"upper":1.0,"upper_inclusive":true}]},"equal_mass":{"n":43,"bin_count":10,"ece":0.14242446942384854,"signed_mean_gap":0.0684636411971824,"absolute_mean_gap":0.0684636411971824,"bins":[{"index":0,"count":4,"confidence":0.19060437753796577,"empirical_correctness":0.0,"signed_gap":0.19060437753796577,"absolute_gap":0.19060437753796577,"min_confidence":0.1889580488204956,"max_confidence":0.19207648932933807},{"index":1,"count":4,"confidence":0.1942865364253521,"empirical_correctness":0.0,"signed_gap":0.1942865364253521,"absolute_gap":0.1942865364253521,"min_confidence":0.19359080493450165,"max_confidence":0.19490459561347961},{"index":2,"count":4,"confidence":0.1955743208527565,"empirical_correctness":0.0,"signed_gap":0.1955743208527565,"absolute_gap":0.1955743208527565,"min_confidence":0.19533219933509827,"max_confidence":0.19590455293655396},{"index":3,"count":5,"confidence":0.19655172824859618,"empirical_correctness":0.2,"signed_gap":-0.003448271751403831,"absolute_gap":0.003448271751403831,"min_confidence":0.1963728666305542,"max_confidence":0.1970541924238205},{"index":4,"count":4,"confidence":0.19764419272542,"empirical_correctness":0.25,"signed_gap":-0.05235580727458,"absolute_gap":0.05235580727458,"min_confidence":0.19714364409446716,"max_confidence":0.19789911806583405},{"index":5,"count":4,"confidence":0.19827743619680405,"empirical_correctness":0.5,"signed_gap":-0.30172256380319595,"absolute_gap":0.30172256380319595,"min_confidence":0.19803431630134583,"max_confidence":0.1985841989517212},{"index":6,"count":5,"confidence":0.20116993486881257,"empirical_correctness":0.0,"signed_gap":0.20116993486881257,"absolute_gap":0.20116993486881257,"min_confidence":0.1987207978963852,"max_confidence":0.2074664980173111},{"index":7,"count":4,"confidence":0.21084925904870033,"empirical_correctness":0.25,"signed_gap":-0.03915074095129967,"absolute_gap":0.03915074095129967,"min_confidence":0.2098189741373062,"max_confidence":0.21142591536045074},{"index":8,"count":4,"confidence":0.22594524919986725,"empirical_correctness":0.0,"signed_gap":0.22594524919986725,"absolute_gap":0.22594524919986725,"min_confidence":0.2124982625246048,"max_confidence":0.2503819763660431},{"index":9,"count":5,"confidence":0.26052055358886717,"empirical_correctness":0.2,"signed_gap":0.060520553588867154,"absolute_gap":0.060520553588867154,"min_confidence":0.250845342874527,"max_confidence":0.26671892404556274}]}},"calibration_singleton_labels":{"equal_width":{"n":38,"bin_count":10,"ece":0.049610077550536695,"signed_mean_gap":0.049610077550536695,"absolute_mean_gap":0.049610077550536695,"bins":[{"index":0,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.0,"upper":0.1,"upper_inclusive":false},{"index":1,"count":25,"confidence":0.19601384103298186,"empirical_correctness":0.16,"signed_gap":0.03601384103298186,"absolute_gap":0.03601384103298186,"lower":0.1,"upper":0.2,"upper_inclusive":false},{"index":2,"count":13,"confidence":0.22960284008429602,"empirical_correctness":0.15384615384615385,"signed_gap":0.07575668623814216,"absolute_gap":0.07575668623814216,"lower":0.2,"upper":0.3,"upper_inclusive":false},{"index":3,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.3,"upper":0.4,"upper_inclusive":false},{"index":4,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.4,"upper":0.5,"upper_inclusive":false},{"index":5,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.5,"upper":0.6,"upper_inclusive":false},{"index":6,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.6,"upper":0.7,"upper_inclusive":false},{"index":7,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.7,"upper":0.8,"upper_inclusive":false},{"index":8,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.8,"upper":0.9,"upper_inclusive":false},{"index":9,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.9,"upper":1.0,"upper_inclusive":true}]},"equal_mass":{"n":38,"bin_count":10,"ece":0.15401398900308108,"signed_mean_gap":0.04961007755053671,"absolute_mean_gap":0.04961007755053671,"bins":[{"index":0,"count":3,"confidence":0.19011367360750833,"empirical_correctness":0.0,"signed_gap":0.19011367360750833,"absolute_gap":0.19011367360750833,"min_confidence":0.1889580488204956,"max_confidence":0.19074435532093048},{"index":1,"count":4,"confidence":0.1939760223031044,"empirical_correctness":0.0,"signed_gap":0.1939760223031044,"absolute_gap":0.1939760223031044,"min_confidence":0.19207648932933807,"max_confidence":0.19533219933509827},{"index":2,"count":4,"confidence":0.19583448767662048,"empirical_correctness":0.0,"signed_gap":0.19583448767662048,"absolute_gap":0.19583448767662048,"min_confidence":0.19550193846225739,"max_confidence":0.1963728666305542},{"index":3,"count":4,"confidence":0.1965964436531067,"empirical_correctness":0.25,"signed_gap":-0.05340355634689331,"absolute_gap":0.05340355634689331,"min_confidence":0.19641833007335663,"max_confidence":0.1970541924238205},{"index":4,"count":4,"confidence":0.19786686077713966,"empirical_correctness":0.5,"signed_gap":-0.30213313922286034,"absolute_gap":0.30213313922286034,"min_confidence":0.19768311083316803,"max_confidence":0.19803431630134583},{"index":5,"count":3,"confidence":0.1983584761619568,"empirical_correctness":0.3333333333333333,"signed_gap":-0.13497485717137653,"absolute_gap":0.13497485717137653,"min_confidence":0.198060542345047,"max_confidence":0.1985841989517212},{"index":6,"count":4,"confidence":0.19959579408168793,"empirical_correctness":0.0,"signed_gap":0.19959579408168793,"absolute_gap":0.19959579408168793,"min_confidence":0.1987207978963852,"max_confidence":0.20054885745048523},{"index":7,"count":4,"confidence":0.21084925904870033,"empirical_correctness":0.25,"signed_gap":-0.03915074095129967,"absolute_gap":0.03915074095129967,"min_confidence":0.2098189741373062,"max_confidence":0.21142591536045074},{"index":8,"count":4,"confidence":0.22594524919986725,"empirical_correctness":0.0,"signed_gap":0.22594524919986725,"absolute_gap":0.22594524919986725,"min_confidence":0.2124982625246048,"max_confidence":0.2503819763660431},{"index":9,"count":4,"confidence":0.25927750766277313,"empirical_correctness":0.25,"signed_gap":0.009277507662773132,"absolute_gap":0.009277507662773132,"min_confidence":0.250845342874527,"max_confidence":0.26671892404556274}]}},"latency_ms":{"n":43,"median":47.58929996751249,"p95":50.62578010838479},"per_action":{"keep":{"n":12,"raw":{"n":12,"acceptable":5,"unsafe":0,"review":0,"automated_count":12,"automated_acceptable":5,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":0.4166666666666667,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.4166666666666667,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.19326031365875665,"upper":0.6804886874504503,"confidence_level":0.95}},"effective":{"n":12,"acceptable":0,"unsafe":0,"review":12,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":12,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":2.7755575615628914e-17,"upper":0.24249400665524085,"confidence_level":0.95}}},"drop":{"n":16,"raw":{"n":16,"acceptable":0,"unsafe":11,"review":0,"automated_count":16,"automated_acceptable":0,"automated_unsafe":11,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":0.6875,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":0.6875,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.1936076805344365,"confidence_level":0.95}},"effective":{"n":16,"acceptable":0,"unsafe":0,"review":13,"automated_count":3,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":13,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":0.8125,"automated_coverage":0.1875,"conditional_correctness":0.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.8125,"accuracy_wilson_95":{"lower":0.0,"upper":0.1936076805344365,"confidence_level":0.95}}},"mask":{"n":2,"raw":{"n":2,"acceptable":0,"unsafe":0,"review":0,"automated_count":2,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.6576197724933469,"confidence_level":0.95}},"effective":{"n":2,"acceptable":0,"unsafe":0,"review":2,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":2,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.6576197724933469,"confidence_level":0.95}}},"pseudonymize":{"n":2,"raw":{"n":2,"acceptable":0,"unsafe":2,"review":0,"automated_count":2,"automated_acceptable":0,"automated_unsafe":2,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":1.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":1.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.6576197724933469,"confidence_level":0.95}},"effective":{"n":2,"acceptable":0,"unsafe":0,"review":2,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":2,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.6576197724933469,"confidence_level":0.95}}},"generalize":{"n":6,"raw":{"n":6,"acceptable":1,"unsafe":5,"review":0,"automated_count":6,"automated_acceptable":1,"automated_unsafe":5,"unnecessary_reviews":0,"accuracy":0.16666666666666666,"unsafe_rate":0.8333333333333334,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.16666666666666666,"conditional_unsafe":0.8333333333333334,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.030053369748306635,"upper":0.5635028221864702,"confidence_level":0.95}},"effective":{"n":6,"acceptable":0,"unsafe":0,"review":6,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":6,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":2.7755575615628914e-17,"upper":0.3903342879021653,"confidence_level":0.95}}},"review":{"n":0,"raw":{"n":0,"acceptable":0,"unsafe":0,"review":0,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":null,"unsafe_rate":null,"review_rate":null,"automated_coverage":null,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":null},"effective":{"n":0,"acceptable":0,"unsafe":0,"review":0,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":null,"unsafe_rate":null,"review_rate":null,"automated_coverage":null,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":null}}}},"diagnostic43-compact_no_samples":{"n":43,"raw":{"n":43,"acceptable":11,"unsafe":26,"review":2,"automated_count":41,"automated_acceptable":11,"automated_unsafe":26,"unnecessary_reviews":2,"accuracy":0.2558139534883721,"unsafe_rate":0.6046511627906976,"review_rate":0.046511627906976744,"automated_coverage":0.9534883720930233,"conditional_correctness":0.2682926829268293,"conditional_unsafe":0.6341463414634146,"unnecessary_review_rate":0.046511627906976744,"accuracy_wilson_95":{"lower":0.14929517123808245,"upper":0.40238403498846675,"confidence_level":0.95}},"effective":{"n":43,"acceptable":6,"unsafe":0,"review":38,"automated_count":5,"automated_acceptable":1,"automated_unsafe":0,"unnecessary_reviews":33,"accuracy":0.13953488372093023,"unsafe_rate":0.0,"review_rate":0.8837209302325582,"automated_coverage":0.11627906976744186,"conditional_correctness":0.2,"conditional_unsafe":0.0,"unnecessary_review_rate":0.7674418604651163,"accuracy_wilson_95":{"lower":0.06555723427964878,"upper":0.2726358796738286,"confidence_level":0.95}},"confusion":{"raw":{"keep":{"keep":10,"drop":0,"mask":0,"pseudonymize":0,"generalize":2,"review":0},"drop":{"keep":13,"drop":0,"mask":0,"pseudonymize":0,"generalize":2,"review":1},"mask":{"keep":2,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0},"pseudonymize":{"keep":1,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":1},"generalize":{"keep":5,"drop":0,"mask":0,"pseudonymize":0,"generalize":1,"review":0},"review":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0}},"effective":{"keep":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":12},"drop":{"keep":0,"drop":0,"mask":4,"pseudonymize":0,"generalize":0,"review":12},"mask":{"keep":0,"drop":0,"mask":1,"pseudonymize":0,"generalize":0,"review":1},"pseudonymize":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":2},"generalize":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":6},"review":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0}}},"probability_metrics":{"singleton_n":38,"multiacceptable_n":5,"multiacceptable_coverage":0.11627906976744186,"nll":1.7864051580244962,"nll_zero_mass_n":0,"brier":0.8299352351997851,"acceptable_set_n":43,"acceptable_set_nll":1.671190263058543,"acceptable_set_zero_mass_n":0},"calibration_all_labels":{"equal_width":{"n":43,"bin_count":10,"ece":0.07391137820343639,"signed_mean_gap":-0.05496647642102352,"absolute_mean_gap":0.05496647642102352,"bins":[{"index":0,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.0,"upper":0.1,"upper_inclusive":false},{"index":1,"count":18,"confidence":0.18929529935121536,"empirical_correctness":0.16666666666666666,"signed_gap":0.022628632684548705,"absolute_gap":0.022628632684548705,"lower":0.1,"upper":0.2,"upper_inclusive":false},{"index":2,"count":25,"confidence":0.20916504502296449,"empirical_correctness":0.32,"signed_gap":-0.11083495497703552,"absolute_gap":0.11083495497703552,"lower":0.2,"upper":0.3,"upper_inclusive":false},{"index":3,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.3,"upper":0.4,"upper_inclusive":false},{"index":4,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.4,"upper":0.5,"upper_inclusive":false},{"index":5,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.5,"upper":0.6,"upper_inclusive":false},{"index":6,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.6,"upper":0.7,"upper_inclusive":false},{"index":7,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.7,"upper":0.8,"upper_inclusive":false},{"index":8,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.8,"upper":0.9,"upper_inclusive":false},{"index":9,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.9,"upper":1.0,"upper_inclusive":true}]},"equal_mass":{"n":43,"bin_count":10,"ece":0.09211810556955116,"signed_mean_gap":-0.05496647642102352,"absolute_mean_gap":0.05496647642102352,"bins":[{"index":0,"count":4,"confidence":0.1760535053908825,"empirical_correctness":0.0,"signed_gap":0.1760535053908825,"absolute_gap":0.1760535053908825,"min_confidence":0.17483405768871307,"max_confidence":0.17843307554721832},{"index":1,"count":4,"confidence":0.18239224329590797,"empirical_correctness":0.25,"signed_gap":-0.06760775670409203,"absolute_gap":0.06760775670409203,"min_confidence":0.17962250113487244,"max_confidence":0.18764424324035645},{"index":2,"count":4,"confidence":0.19615687429904938,"empirical_correctness":0.25,"signed_gap":-0.05384312570095062,"absolute_gap":0.05384312570095062,"min_confidence":0.19549134373664856,"max_confidence":0.19667492806911469},{"index":3,"count":5,"confidence":0.19797174334526063,"empirical_correctness":0.2,"signed_gap":-0.00202825665473938,"absolute_gap":0.00202825665473938,"min_confidence":0.19674263894557953,"max_confidence":0.19898119568824768},{"index":4,"count":4,"confidence":0.20075567811727524,"empirical_correctness":0.25,"signed_gap":-0.04924432188272476,"absolute_gap":0.04924432188272476,"min_confidence":0.19904617965221405,"max_confidence":0.20196183025836945},{"index":5,"count":4,"confidence":0.20442494377493858,"empirical_correctness":0.25,"signed_gap":-0.04557505622506142,"absolute_gap":0.04557505622506142,"min_confidence":0.20317931473255157,"max_confidence":0.20549209415912628},{"index":6,"count":5,"confidence":0.2064925789833069,"empirical_correctness":0.4,"signed_gap":-0.19350742101669313,"absolute_gap":0.19350742101669313,"min_confidence":0.20551927387714386,"max_confidence":0.20765234529972076},{"index":7,"count":4,"confidence":0.2088373452425003,"empirical_correctness":0.5,"signed_gap":-0.2911626547574997,"absolute_gap":0.2911626547574997,"min_confidence":0.2084052860736847,"max_confidence":0.2096947282552719},{"index":8,"count":4,"confidence":0.21127288416028023,"empirical_correctness":0.25,"signed_gap":-0.03872711583971977,"absolute_gap":0.03872711583971977,"min_confidence":0.20985078811645508,"max_confidence":0.2124340832233429},{"index":9,"count":5,"confidence":0.21890920102596284,"empirical_correctness":0.2,"signed_gap":0.01890920102596283,"absolute_gap":0.01890920102596283,"min_confidence":0.21709541976451874,"max_confidence":0.2217775285243988}]}},"calibration_singleton_labels":{"equal_width":{"n":38,"bin_count":10,"ece":0.09002171692095305,"signed_mean_gap":-0.08888353642664458,"absolute_mean_gap":0.08888353642664458,"bins":[{"index":0,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.0,"upper":0.1,"upper_inclusive":false},{"index":1,"count":16,"confidence":0.1888515893369913,"empirical_correctness":0.1875,"signed_gap":0.0013515893369913101,"absolute_gap":0.0013515893369913101,"lower":0.1,"upper":0.2,"upper_inclusive":false},{"index":2,"count":22,"confidence":0.20912728119980206,"empirical_correctness":0.36363636363636365,"signed_gap":-0.15450908243656158,"absolute_gap":0.15450908243656158,"lower":0.2,"upper":0.3,"upper_inclusive":false},{"index":3,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.3,"upper":0.4,"upper_inclusive":false},{"index":4,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.4,"upper":0.5,"upper_inclusive":false},{"index":5,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.5,"upper":0.6,"upper_inclusive":false},{"index":6,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.6,"upper":0.7,"upper_inclusive":false},{"index":7,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.7,"upper":0.8,"upper_inclusive":false},{"index":8,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.8,"upper":0.9,"upper_inclusive":false},{"index":9,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.9,"upper":1.0,"upper_inclusive":true}]},"equal_mass":{"n":38,"bin_count":10,"ece":0.11655621779592414,"signed_mean_gap":-0.08888353642664458,"absolute_mean_gap":0.08888353642664458,"bins":[{"index":0,"count":3,"confidence":0.17526031533877054,"empirical_correctness":0.0,"signed_gap":0.17526031533877054,"absolute_gap":0.17526031533877054,"min_confidence":0.17483405768871307,"max_confidence":0.17586074769496918},{"index":1,"count":4,"confidence":0.18008945137262344,"empirical_correctness":0.25,"signed_gap":-0.06991054862737656,"absolute_gap":0.06991054862737656,"min_confidence":0.17843307554721832,"max_confidence":0.18227650225162506},{"index":2,"count":4,"confidence":0.19615687429904938,"empirical_correctness":0.25,"signed_gap":-0.05384312570095062,"absolute_gap":0.05384312570095062,"min_confidence":0.19549134373664856,"max_confidence":0.19667492806911469},{"index":3,"count":4,"confidence":0.197953250259161,"empirical_correctness":0.25,"signed_gap":-0.052046749740839005,"absolute_gap":0.052046749740839005,"min_confidence":0.19674263894557953,"max_confidence":0.19898119568824768},{"index":4,"count":4,"confidence":0.20075567811727524,"empirical_correctness":0.25,"signed_gap":-0.04924432188272476,"absolute_gap":0.04924432188272476,"min_confidence":0.19904617965221405,"max_confidence":0.20196183025836945},{"index":5,"count":3,"confidence":0.20406922698020935,"empirical_correctness":0.3333333333333333,"signed_gap":-0.12926410635312396,"absolute_gap":0.12926410635312396,"min_confidence":0.20317931473255157,"max_confidence":0.2053578794002533},{"index":6,"count":4,"confidence":0.20619584247469902,"empirical_correctness":0.5,"signed_gap":-0.293804157525301,"absolute_gap":0.293804157525301,"min_confidence":0.20549209415912628,"max_confidence":0.2073839008808136},{"index":7,"count":4,"confidence":0.20832674950361252,"empirical_correctness":0.25,"signed_gap":-0.04167325049638748,"absolute_gap":0.04167325049638748,"min_confidence":0.20765234529972076,"max_confidence":0.20867519080638885},{"index":8,"count":4,"confidence":0.21226875483989716,"empirical_correctness":0.5,"signed_gap":-0.28773124516010284,"absolute_gap":0.28773124516010284,"min_confidence":0.2096947282552719,"max_confidence":0.21709541976451874},{"index":9,"count":4,"confidence":0.21936264634132385,"empirical_correctness":0.25,"signed_gap":-0.030637353658676147,"absolute_gap":0.030637353658676147,"min_confidence":0.21715180575847626,"max_confidence":0.2217775285243988}]}},"latency_ms":{"n":43,"median":57.14499996975064,"p95":61.55062986072153},"per_action":{"keep":{"n":12,"raw":{"n":12,"acceptable":10,"unsafe":0,"review":0,"automated_count":12,"automated_acceptable":10,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":0.8333333333333334,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.8333333333333334,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.5519691377470266,"upper":0.9530348578161463,"confidence_level":0.95}},"effective":{"n":12,"acceptable":0,"unsafe":0,"review":12,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":12,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":2.7755575615628914e-17,"upper":0.24249400665524085,"confidence_level":0.95}}},"drop":{"n":16,"raw":{"n":16,"acceptable":0,"unsafe":13,"review":1,"automated_count":15,"automated_acceptable":0,"automated_unsafe":13,"unnecessary_reviews":1,"accuracy":0.0,"unsafe_rate":0.8125,"review_rate":0.0625,"automated_coverage":0.9375,"conditional_correctness":0.0,"conditional_unsafe":0.8666666666666667,"unnecessary_review_rate":0.0625,"accuracy_wilson_95":{"lower":0.0,"upper":0.1936076805344365,"confidence_level":0.95}},"effective":{"n":16,"acceptable":0,"unsafe":0,"review":12,"automated_count":4,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":12,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":0.75,"automated_coverage":0.25,"conditional_correctness":0.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.75,"accuracy_wilson_95":{"lower":0.0,"upper":0.1936076805344365,"confidence_level":0.95}}},"mask":{"n":2,"raw":{"n":2,"acceptable":0,"unsafe":2,"review":0,"automated_count":2,"automated_acceptable":0,"automated_unsafe":2,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":1.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":1.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.6576197724933469,"confidence_level":0.95}},"effective":{"n":2,"acceptable":1,"unsafe":0,"review":1,"automated_count":1,"automated_acceptable":1,"automated_unsafe":0,"unnecessary_reviews":1,"accuracy":0.5,"unsafe_rate":0.0,"review_rate":0.5,"automated_coverage":0.5,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.5,"accuracy_wilson_95":{"lower":0.09453120573423074,"upper":0.9054687942657693,"confidence_level":0.95}}},"pseudonymize":{"n":2,"raw":{"n":2,"acceptable":0,"unsafe":1,"review":1,"automated_count":1,"automated_acceptable":0,"automated_unsafe":1,"unnecessary_reviews":1,"accuracy":0.0,"unsafe_rate":0.5,"review_rate":0.5,"automated_coverage":0.5,"conditional_correctness":0.0,"conditional_unsafe":1.0,"unnecessary_review_rate":0.5,"accuracy_wilson_95":{"lower":0.0,"upper":0.6576197724933469,"confidence_level":0.95}},"effective":{"n":2,"acceptable":0,"unsafe":0,"review":2,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":2,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.6576197724933469,"confidence_level":0.95}}},"generalize":{"n":6,"raw":{"n":6,"acceptable":1,"unsafe":5,"review":0,"automated_count":6,"automated_acceptable":1,"automated_unsafe":5,"unnecessary_reviews":0,"accuracy":0.16666666666666666,"unsafe_rate":0.8333333333333334,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.16666666666666666,"conditional_unsafe":0.8333333333333334,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.030053369748306635,"upper":0.5635028221864702,"confidence_level":0.95}},"effective":{"n":6,"acceptable":0,"unsafe":0,"review":6,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":6,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":2.7755575615628914e-17,"upper":0.3903342879021653,"confidence_level":0.95}}},"review":{"n":0,"raw":{"n":0,"acceptable":0,"unsafe":0,"review":0,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":null,"unsafe_rate":null,"review_rate":null,"automated_coverage":null,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":null},"effective":{"n":0,"acceptable":0,"unsafe":0,"review":0,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":null,"unsafe_rate":null,"review_rate":null,"automated_coverage":null,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":null}}}},"challenge56-no_samples":{"n":56,"raw":{"n":56,"acceptable":17,"unsafe":31,"review":0,"automated_count":56,"automated_acceptable":17,"automated_unsafe":31,"unnecessary_reviews":0,"accuracy":0.30357142857142855,"unsafe_rate":0.5535714285714286,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.30357142857142855,"conditional_unsafe":0.5535714285714286,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.19900325085369447,"upper":0.43335865285724834,"confidence_level":0.95}},"effective":{"n":56,"acceptable":2,"unsafe":0,"review":56,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":54,"accuracy":0.03571428571428571,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":0.9642857142857143,"accuracy_wilson_95":{"lower":0.00984942514823639,"upper":0.12118780180490107,"confidence_level":0.95}},"confusion":{"raw":{"keep":{"keep":2,"drop":0,"mask":0,"pseudonymize":0,"generalize":6,"review":0},"drop":{"keep":19,"drop":0,"mask":0,"pseudonymize":0,"generalize":3,"review":0},"mask":{"keep":0,"drop":0,"mask":7,"pseudonymize":1,"generalize":0,"review":0},"pseudonymize":{"keep":6,"drop":0,"mask":0,"pseudonymize":2,"generalize":0,"review":0},"generalize":{"keep":2,"drop":0,"mask":0,"pseudonymize":0,"generalize":6,"review":0},"review":{"keep":2,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0}},"effective":{"keep":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":8},"drop":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":22},"mask":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":8},"pseudonymize":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":8},"generalize":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":8},"review":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":2}}},"probability_metrics":{"singleton_n":56,"multiacceptable_n":0,"multiacceptable_coverage":0.0,"nll":1.79275602419974,"nll_zero_mass_n":0,"brier":0.8298603741488869,"acceptable_set_n":56,"acceptable_set_nll":1.79275602419974,"acceptable_set_zero_mass_n":0},"calibration_all_labels":{"equal_width":{"n":56,"bin_count":10,"ece":0.09916563225643975,"signed_mean_gap":-0.09916563225643975,"absolute_mean_gap":0.09916563225643975,"bins":[{"index":0,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.0,"upper":0.1,"upper_inclusive":false},{"index":1,"count":22,"confidence":0.19311055541038513,"empirical_correctness":0.2727272727272727,"signed_gap":-0.07961671731688758,"absolute_gap":0.07961671731688758,"lower":0.1,"upper":0.2,"upper_inclusive":false},{"index":2,"count":34,"confidence":0.21171448160620296,"empirical_correctness":0.3235294117647059,"signed_gap":-0.11181493015850294,"absolute_gap":0.11181493015850294,"lower":0.2,"upper":0.3,"upper_inclusive":false},{"index":3,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.3,"upper":0.4,"upper_inclusive":false},{"index":4,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.4,"upper":0.5,"upper_inclusive":false},{"index":5,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.5,"upper":0.6,"upper_inclusive":false},{"index":6,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.6,"upper":0.7,"upper_inclusive":false},{"index":7,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.7,"upper":0.8,"upper_inclusive":false},{"index":8,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.8,"upper":0.9,"upper_inclusive":false},{"index":9,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.9,"upper":1.0,"upper_inclusive":true}]},"equal_mass":{"n":56,"bin_count":10,"ece":0.11345951791320529,"signed_mean_gap":-0.09916563225643975,"absolute_mean_gap":0.09916563225643975,"bins":[{"index":0,"count":5,"confidence":0.18757677376270293,"empirical_correctness":0.2,"signed_gap":-0.01242322623729708,"absolute_gap":0.01242322623729708,"min_confidence":0.1855587512254715,"max_confidence":0.18853823840618134},{"index":1,"count":6,"confidence":0.18989035487174988,"empirical_correctness":0.16666666666666666,"signed_gap":0.02322368820508322,"absolute_gap":0.02322368820508322,"min_confidence":0.1886741667985916,"max_confidence":0.19246028363704681},{"index":2,"count":5,"confidence":0.19558023512363434,"empirical_correctness":0.2,"signed_gap":-0.004419764876365667,"absolute_gap":0.004419764876365667,"min_confidence":0.19320163130760193,"max_confidence":0.19753246009349823},{"index":3,"count":6,"confidence":0.19888417422771454,"empirical_correctness":0.5,"signed_gap":-0.30111582577228546,"absolute_gap":0.30111582577228546,"min_confidence":0.19771067798137665,"max_confidence":0.19989626109600067},{"index":4,"count":6,"confidence":0.20132158199946085,"empirical_correctness":0.16666666666666666,"signed_gap":0.03465491533279419,"absolute_gap":0.03465491533279419,"min_confidence":0.20022453367710114,"max_confidence":0.2022218108177185},{"index":5,"count":5,"confidence":0.20361389219760895,"empirical_correctness":0.4,"signed_gap":-0.19638610780239107,"absolute_gap":0.19638610780239107,"min_confidence":0.20229090750217438,"max_confidence":0.2044082134962082},{"index":6,"count":6,"confidence":0.20589091628789902,"empirical_correctness":0.3333333333333333,"signed_gap":-0.1274424170454343,"absolute_gap":0.1274424170454343,"min_confidence":0.2044876366853714,"max_confidence":0.20661887526512146},{"index":7,"count":5,"confidence":0.21059143543243408,"empirical_correctness":0.2,"signed_gap":0.010591435432434071,"absolute_gap":0.010591435432434071,"min_confidence":0.20719115436077118,"max_confidence":0.21325640380382538},{"index":8,"count":6,"confidence":0.21653600285450617,"empirical_correctness":0.3333333333333333,"signed_gap":-0.11679733047882715,"absolute_gap":0.11679733047882715,"min_confidence":0.21483053267002106,"max_confidence":0.2175399661064148},{"index":9,"count":6,"confidence":0.23079578826824823,"empirical_correctness":0.5,"signed_gap":-0.2692042117317518,"absolute_gap":0.2692042117317518,"min_confidence":0.21823906898498535,"max_confidence":0.25114721059799194}]}},"calibration_singleton_labels":{"equal_width":{"n":56,"bin_count":10,"ece":0.09916563225643975,"signed_mean_gap":-0.09916563225643975,"absolute_mean_gap":0.09916563225643975,"bins":[{"index":0,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.0,"upper":0.1,"upper_inclusive":false},{"index":1,"count":22,"confidence":0.19311055541038513,"empirical_correctness":0.2727272727272727,"signed_gap":-0.07961671731688758,"absolute_gap":0.07961671731688758,"lower":0.1,"upper":0.2,"upper_inclusive":false},{"index":2,"count":34,"confidence":0.21171448160620296,"empirical_correctness":0.3235294117647059,"signed_gap":-0.11181493015850294,"absolute_gap":0.11181493015850294,"lower":0.2,"upper":0.3,"upper_inclusive":false},{"index":3,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.3,"upper":0.4,"upper_inclusive":false},{"index":4,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.4,"upper":0.5,"upper_inclusive":false},{"index":5,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.5,"upper":0.6,"upper_inclusive":false},{"index":6,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.6,"upper":0.7,"upper_inclusive":false},{"index":7,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.7,"upper":0.8,"upper_inclusive":false},{"index":8,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.8,"upper":0.9,"upper_inclusive":false},{"index":9,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.9,"upper":1.0,"upper_inclusive":true}]},"equal_mass":{"n":56,"bin_count":10,"ece":0.11345951791320529,"signed_mean_gap":-0.09916563225643975,"absolute_mean_gap":0.09916563225643975,"bins":[{"index":0,"count":5,"confidence":0.18757677376270293,"empirical_correctness":0.2,"signed_gap":-0.01242322623729708,"absolute_gap":0.01242322623729708,"min_confidence":0.1855587512254715,"max_confidence":0.18853823840618134},{"index":1,"count":6,"confidence":0.18989035487174988,"empirical_correctness":0.16666666666666666,"signed_gap":0.02322368820508322,"absolute_gap":0.02322368820508322,"min_confidence":0.1886741667985916,"max_confidence":0.19246028363704681},{"index":2,"count":5,"confidence":0.19558023512363434,"empirical_correctness":0.2,"signed_gap":-0.004419764876365667,"absolute_gap":0.004419764876365667,"min_confidence":0.19320163130760193,"max_confidence":0.19753246009349823},{"index":3,"count":6,"confidence":0.19888417422771454,"empirical_correctness":0.5,"signed_gap":-0.30111582577228546,"absolute_gap":0.30111582577228546,"min_confidence":0.19771067798137665,"max_confidence":0.19989626109600067},{"index":4,"count":6,"confidence":0.20132158199946085,"empirical_correctness":0.16666666666666666,"signed_gap":0.03465491533279419,"absolute_gap":0.03465491533279419,"min_confidence":0.20022453367710114,"max_confidence":0.2022218108177185},{"index":5,"count":5,"confidence":0.20361389219760895,"empirical_correctness":0.4,"signed_gap":-0.19638610780239107,"absolute_gap":0.19638610780239107,"min_confidence":0.20229090750217438,"max_confidence":0.2044082134962082},{"index":6,"count":6,"confidence":0.20589091628789902,"empirical_correctness":0.3333333333333333,"signed_gap":-0.1274424170454343,"absolute_gap":0.1274424170454343,"min_confidence":0.2044876366853714,"max_confidence":0.20661887526512146},{"index":7,"count":5,"confidence":0.21059143543243408,"empirical_correctness":0.2,"signed_gap":0.010591435432434071,"absolute_gap":0.010591435432434071,"min_confidence":0.20719115436077118,"max_confidence":0.21325640380382538},{"index":8,"count":6,"confidence":0.21653600285450617,"empirical_correctness":0.3333333333333333,"signed_gap":-0.11679733047882715,"absolute_gap":0.11679733047882715,"min_confidence":0.21483053267002106,"max_confidence":0.2175399661064148},{"index":9,"count":6,"confidence":0.23079578826824823,"empirical_correctness":0.5,"signed_gap":-0.2692042117317518,"absolute_gap":0.2692042117317518,"min_confidence":0.21823906898498535,"max_confidence":0.25114721059799194}]}},"latency_ms":{"n":56,"median":42.55344998091459,"p95":47.908199892845005},"per_action":{"keep":{"n":8,"raw":{"n":8,"acceptable":2,"unsafe":0,"review":0,"automated_count":8,"automated_acceptable":2,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":0.25,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.25,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.071479212752109,"upper":0.5907245696898311,"confidence_level":0.95}},"effective":{"n":8,"acceptable":0,"unsafe":0,"review":8,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":8,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.32440756488388023,"confidence_level":0.95}}},"drop":{"n":22,"raw":{"n":22,"acceptable":0,"unsafe":20,"review":0,"automated_count":22,"automated_acceptable":0,"automated_unsafe":20,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":0.9090909090909091,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":0.9090909090909091,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.14865487460862087,"confidence_level":0.95}},"effective":{"n":22,"acceptable":0,"unsafe":0,"review":22,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":22,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.14865487460862087,"confidence_level":0.95}}},"mask":{"n":8,"raw":{"n":8,"acceptable":7,"unsafe":1,"review":0,"automated_count":8,"automated_acceptable":7,"automated_unsafe":1,"unnecessary_reviews":0,"accuracy":0.875,"unsafe_rate":0.125,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.875,"conditional_unsafe":0.125,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.5291118177871464,"upper":0.9775825085499433,"confidence_level":0.95}},"effective":{"n":8,"acceptable":0,"unsafe":0,"review":8,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":8,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.32440756488388023,"confidence_level":0.95}}},"pseudonymize":{"n":8,"raw":{"n":8,"acceptable":2,"unsafe":6,"review":0,"automated_count":8,"automated_acceptable":2,"automated_unsafe":6,"unnecessary_reviews":0,"accuracy":0.25,"unsafe_rate":0.75,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.25,"conditional_unsafe":0.75,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.071479212752109,"upper":0.5907245696898311,"confidence_level":0.95}},"effective":{"n":8,"acceptable":0,"unsafe":0,"review":8,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":8,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.32440756488388023,"confidence_level":0.95}}},"generalize":{"n":8,"raw":{"n":8,"acceptable":6,"unsafe":2,"review":0,"automated_count":8,"automated_acceptable":6,"automated_unsafe":2,"unnecessary_reviews":0,"accuracy":0.75,"unsafe_rate":0.25,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.75,"conditional_unsafe":0.25,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.40927543031016883,"upper":0.9285207872478909,"confidence_level":0.95}},"effective":{"n":8,"acceptable":0,"unsafe":0,"review":8,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":8,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.32440756488388023,"confidence_level":0.95}}},"review":{"n":2,"raw":{"n":2,"acceptable":0,"unsafe":2,"review":0,"automated_count":2,"automated_acceptable":0,"automated_unsafe":2,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":1.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":1.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.6576197724933469,"confidence_level":0.95}},"effective":{"n":2,"acceptable":2,"unsafe":0,"review":2,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":1.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.34238022750665303,"upper":1.0,"confidence_level":0.95}}}}},"challenge56-compact_no_samples":{"n":56,"raw":{"n":56,"acceptable":9,"unsafe":42,"review":2,"automated_count":54,"automated_acceptable":9,"automated_unsafe":42,"unnecessary_reviews":2,"accuracy":0.16071428571428573,"unsafe_rate":0.75,"review_rate":0.03571428571428571,"automated_coverage":0.9642857142857143,"conditional_correctness":0.16666666666666666,"conditional_unsafe":0.7777777777777778,"unnecessary_review_rate":0.03571428571428571,"accuracy_wilson_95":{"lower":0.08692667607595139,"upper":0.2780620666974952,"confidence_level":0.95}},"effective":{"n":56,"acceptable":2,"unsafe":0,"review":56,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":54,"accuracy":0.03571428571428571,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":0.9642857142857143,"accuracy_wilson_95":{"lower":0.00984942514823639,"upper":0.12118780180490107,"confidence_level":0.95}},"confusion":{"raw":{"keep":{"keep":8,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0},"drop":{"keep":18,"drop":0,"mask":0,"pseudonymize":0,"generalize":4,"review":0},"mask":{"keep":8,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0},"pseudonymize":{"keep":3,"drop":0,"mask":0,"pseudonymize":0,"generalize":3,"review":2},"generalize":{"keep":7,"drop":0,"mask":0,"pseudonymize":0,"generalize":1,"review":0},"review":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":2,"review":0}},"effective":{"keep":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":8},"drop":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":22},"mask":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":8},"pseudonymize":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":8},"generalize":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":8},"review":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":2}}},"probability_metrics":{"singleton_n":56,"multiacceptable_n":0,"multiacceptable_coverage":0.0,"nll":1.8173503923996475,"nll_zero_mass_n":0,"brier":0.8418606190979928,"acceptable_set_n":56,"acceptable_set_nll":1.8173503923996475,"acceptable_set_zero_mass_n":0},"calibration_all_labels":{"equal_width":{"n":56,"bin_count":10,"ece":0.03695237343864781,"signed_mean_gap":0.03695237343864781,"absolute_mean_gap":0.03695237343864781,"bins":[{"index":0,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.0,"upper":0.1,"upper_inclusive":false},{"index":1,"count":32,"confidence":0.1889310791157186,"empirical_correctness":0.125,"signed_gap":0.0639310791157186,"absolute_gap":0.0639310791157186,"lower":0.1,"upper":0.2,"upper_inclusive":false},{"index":2,"count":24,"confidence":0.20931409920255342,"empirical_correctness":0.20833333333333334,"signed_gap":0.0009807658692200794,"absolute_gap":0.0009807658692200794,"lower":0.2,"upper":0.3,"upper_inclusive":false},{"index":3,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.3,"upper":0.4,"upper_inclusive":false},{"index":4,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.4,"upper":0.5,"upper_inclusive":false},{"index":5,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.5,"upper":0.6,"upper_inclusive":false},{"index":6,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.6,"upper":0.7,"upper_inclusive":false},{"index":7,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.7,"upper":0.8,"upper_inclusive":false},{"index":8,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.8,"upper":0.9,"upper_inclusive":false},{"index":9,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.9,"upper":1.0,"upper_inclusive":true}]},"equal_mass":{"n":56,"bin_count":10,"ece":0.13149245216378144,"signed_mean_gap":0.03695237343864782,"absolute_mean_gap":0.03695237343864782,"bins":[{"index":0,"count":5,"confidence":0.17946510016918182,"empirical_correctness":0.0,"signed_gap":0.17946510016918182,"absolute_gap":0.17946510016918182,"min_confidence":0.17725731432437897,"max_confidence":0.18180620670318604},{"index":1,"count":6,"confidence":0.1837302272518476,"empirical_correctness":0.3333333333333333,"signed_gap":-0.14960310608148572,"absolute_gap":0.14960310608148572,"min_confidence":0.18267197906970978,"max_confidence":0.18453963100910187},{"index":2,"count":5,"confidence":0.1862068623304367,"empirical_correctness":0.0,"signed_gap":0.1862068623304367,"absolute_gap":0.1862068623304367,"min_confidence":0.18487781286239624,"max_confidence":0.18712982535362244},{"index":3,"count":6,"confidence":0.19071033596992493,"empirical_correctness":0.16666666666666666,"signed_gap":0.02404366930325827,"absolute_gap":0.02404366930325827,"min_confidence":0.18858884274959564,"max_confidence":0.19242578744888306},{"index":4,"count":6,"confidence":0.19588864346345267,"empirical_correctness":0.16666666666666666,"signed_gap":0.02922197679678601,"absolute_gap":0.02922197679678601,"min_confidence":0.19259615242481232,"max_confidence":0.1972087025642395},{"index":5,"count":5,"confidence":0.1991294413805008,"empirical_correctness":0.0,"signed_gap":0.1991294413805008,"absolute_gap":0.1991294413805008,"min_confidence":0.19804982841014862,"max_confidence":0.20018772780895233},{"index":6,"count":6,"confidence":0.2031528726220131,"empirical_correctness":0.3333333333333333,"signed_gap":-0.13018046071132022,"absolute_gap":0.13018046071132022,"min_confidence":0.20081792771816254,"max_confidence":0.205613374710083},{"index":7,"count":5,"confidence":0.2063158392906189,"empirical_correctness":0.4,"signed_gap":-0.19368416070938113,"absolute_gap":0.19368416070938113,"min_confidence":0.20566993951797485,"max_confidence":0.2074689269065857},{"index":8,"count":6,"confidence":0.21002753327290216,"empirical_correctness":0.16666666666666666,"signed_gap":0.043360866606235504,"absolute_gap":0.043360866606235504,"min_confidence":0.20811237394809723,"max_confidence":0.21276620030403137},{"index":9,"count":6,"confidence":0.21878150353829065,"empirical_correctness":0.0,"signed_gap":0.21878150353829065,"absolute_gap":0.21878150353829065,"min_confidence":0.21293655037879944,"max_confidence":0.22340820729732513}]}},"calibration_singleton_labels":{"equal_width":{"n":56,"bin_count":10,"ece":0.03695237343864781,"signed_mean_gap":0.03695237343864781,"absolute_mean_gap":0.03695237343864781,"bins":[{"index":0,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.0,"upper":0.1,"upper_inclusive":false},{"index":1,"count":32,"confidence":0.1889310791157186,"empirical_correctness":0.125,"signed_gap":0.0639310791157186,"absolute_gap":0.0639310791157186,"lower":0.1,"upper":0.2,"upper_inclusive":false},{"index":2,"count":24,"confidence":0.20931409920255342,"empirical_correctness":0.20833333333333334,"signed_gap":0.0009807658692200794,"absolute_gap":0.0009807658692200794,"lower":0.2,"upper":0.3,"upper_inclusive":false},{"index":3,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.3,"upper":0.4,"upper_inclusive":false},{"index":4,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.4,"upper":0.5,"upper_inclusive":false},{"index":5,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.5,"upper":0.6,"upper_inclusive":false},{"index":6,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.6,"upper":0.7,"upper_inclusive":false},{"index":7,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.7,"upper":0.8,"upper_inclusive":false},{"index":8,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.8,"upper":0.9,"upper_inclusive":false},{"index":9,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.9,"upper":1.0,"upper_inclusive":true}]},"equal_mass":{"n":56,"bin_count":10,"ece":0.13149245216378144,"signed_mean_gap":0.03695237343864782,"absolute_mean_gap":0.03695237343864782,"bins":[{"index":0,"count":5,"confidence":0.17946510016918182,"empirical_correctness":0.0,"signed_gap":0.17946510016918182,"absolute_gap":0.17946510016918182,"min_confidence":0.17725731432437897,"max_confidence":0.18180620670318604},{"index":1,"count":6,"confidence":0.1837302272518476,"empirical_correctness":0.3333333333333333,"signed_gap":-0.14960310608148572,"absolute_gap":0.14960310608148572,"min_confidence":0.18267197906970978,"max_confidence":0.18453963100910187},{"index":2,"count":5,"confidence":0.1862068623304367,"empirical_correctness":0.0,"signed_gap":0.1862068623304367,"absolute_gap":0.1862068623304367,"min_confidence":0.18487781286239624,"max_confidence":0.18712982535362244},{"index":3,"count":6,"confidence":0.19071033596992493,"empirical_correctness":0.16666666666666666,"signed_gap":0.02404366930325827,"absolute_gap":0.02404366930325827,"min_confidence":0.18858884274959564,"max_confidence":0.19242578744888306},{"index":4,"count":6,"confidence":0.19588864346345267,"empirical_correctness":0.16666666666666666,"signed_gap":0.02922197679678601,"absolute_gap":0.02922197679678601,"min_confidence":0.19259615242481232,"max_confidence":0.1972087025642395},{"index":5,"count":5,"confidence":0.1991294413805008,"empirical_correctness":0.0,"signed_gap":0.1991294413805008,"absolute_gap":0.1991294413805008,"min_confidence":0.19804982841014862,"max_confidence":0.20018772780895233},{"index":6,"count":6,"confidence":0.2031528726220131,"empirical_correctness":0.3333333333333333,"signed_gap":-0.13018046071132022,"absolute_gap":0.13018046071132022,"min_confidence":0.20081792771816254,"max_confidence":0.205613374710083},{"index":7,"count":5,"confidence":0.2063158392906189,"empirical_correctness":0.4,"signed_gap":-0.19368416070938113,"absolute_gap":0.19368416070938113,"min_confidence":0.20566993951797485,"max_confidence":0.2074689269065857},{"index":8,"count":6,"confidence":0.21002753327290216,"empirical_correctness":0.16666666666666666,"signed_gap":0.043360866606235504,"absolute_gap":0.043360866606235504,"min_confidence":0.20811237394809723,"max_confidence":0.21276620030403137},{"index":9,"count":6,"confidence":0.21878150353829065,"empirical_correctness":0.0,"signed_gap":0.21878150353829065,"absolute_gap":0.21878150353829065,"min_confidence":0.21293655037879944,"max_confidence":0.22340820729732513}]}},"latency_ms":{"n":56,"median":48.31380001269281,"p95":50.6937499740161},"per_action":{"keep":{"n":8,"raw":{"n":8,"acceptable":8,"unsafe":0,"review":0,"automated_count":8,"automated_acceptable":8,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":1.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.6755924351161198,"upper":1.0,"confidence_level":0.95}},"effective":{"n":8,"acceptable":0,"unsafe":0,"review":8,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":8,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.32440756488388023,"confidence_level":0.95}}},"drop":{"n":22,"raw":{"n":22,"acceptable":0,"unsafe":22,"review":0,"automated_count":22,"automated_acceptable":0,"automated_unsafe":22,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":1.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":1.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.14865487460862087,"confidence_level":0.95}},"effective":{"n":22,"acceptable":0,"unsafe":0,"review":22,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":22,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.14865487460862087,"confidence_level":0.95}}},"mask":{"n":8,"raw":{"n":8,"acceptable":0,"unsafe":8,"review":0,"automated_count":8,"automated_acceptable":0,"automated_unsafe":8,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":1.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":1.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.32440756488388023,"confidence_level":0.95}},"effective":{"n":8,"acceptable":0,"unsafe":0,"review":8,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":8,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.32440756488388023,"confidence_level":0.95}}},"pseudonymize":{"n":8,"raw":{"n":8,"acceptable":0,"unsafe":3,"review":2,"automated_count":6,"automated_acceptable":0,"automated_unsafe":3,"unnecessary_reviews":2,"accuracy":0.0,"unsafe_rate":0.375,"review_rate":0.25,"automated_coverage":0.75,"conditional_correctness":0.0,"conditional_unsafe":0.5,"unnecessary_review_rate":0.25,"accuracy_wilson_95":{"lower":0.0,"upper":0.32440756488388023,"confidence_level":0.95}},"effective":{"n":8,"acceptable":0,"unsafe":0,"review":8,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":8,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.32440756488388023,"confidence_level":0.95}}},"generalize":{"n":8,"raw":{"n":8,"acceptable":1,"unsafe":7,"review":0,"automated_count":8,"automated_acceptable":1,"automated_unsafe":7,"unnecessary_reviews":0,"accuracy":0.125,"unsafe_rate":0.875,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.125,"conditional_unsafe":0.875,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.02241749145005667,"upper":0.4708881822128535,"confidence_level":0.95}},"effective":{"n":8,"acceptable":0,"unsafe":0,"review":8,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":8,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.32440756488388023,"confidence_level":0.95}}},"review":{"n":2,"raw":{"n":2,"acceptable":0,"unsafe":2,"review":0,"automated_count":2,"automated_acceptable":0,"automated_unsafe":2,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":1.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":1.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.6576197724933469,"confidence_level":0.95}},"effective":{"n":2,"acceptable":2,"unsafe":0,"review":2,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":1.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.34238022750665303,"upper":1.0,"confidence_level":0.95}}}}}},"composite":{"n":198,"scope":"descriptive sum across four correlated panels","raw":{"n":198,"acceptable":43,"unsafe":122,"review":4,"automated_count":194,"automated_acceptable":43,"automated_unsafe":122,"unnecessary_reviews":4,"accuracy":0.21717171717171718,"unsafe_rate":0.6161616161616161,"review_rate":0.020202020202020204,"automated_coverage":0.9797979797979798,"conditional_correctness":0.22164948453608246,"conditional_unsafe":0.6288659793814433,"unnecessary_review_rate":0.020202020202020204},"effective":{"n":198,"acceptable":15,"unsafe":0,"review":190,"automated_count":8,"automated_acceptable":1,"automated_unsafe":0,"unnecessary_reviews":176,"accuracy":0.07575757575757576,"unsafe_rate":0.0,"review_rate":0.9595959595959596,"automated_coverage":0.04040404040404041,"conditional_correctness":0.125,"conditional_unsafe":0.0,"unnecessary_review_rate":0.8888888888888888}}},{"id":"Qwen--Qwen3-0.6B","platform":"Local","model_id":"Qwen/Qwen3-0.6B","revision":"c1899de289a04d12100db370d81485cdf75e47ca","loading":"float16 / none","scoring_kind":"Adapted label scores","settings":{"dtype":"float16","quantization":null,"max_tokens":4096,"effective_max_tokens":4096,"temperature":1.0,"trust_remote_code":false,"local_files_only":true,"use_safetensors":true,"guard_threshold":0.45,"seed":0,"tf32":false},"scoring_metadata":{"backend":"causal","protocol":"hf-local-sweep-backends-v1","probability_source":"adapted_conditional_label_softmax_not_calibrated_sixway","latency_scope":"synchronous_forward_scoring_including_transfer_excluding_tokenization","precision":"float16","equal_mass_convention":"rank bins with floor boundaries at floor(i*n/k), k=min(10,n//2); minimum two rows"},"load_ms":7044.552800012752,"peak_allocated_bytes":1238082048,"peak_reserved_bytes":1258291200,"panels":{"diagnostic43-no_samples":{"n":43,"raw":{"n":43,"acceptable":13,"unsafe":30,"review":0,"automated_count":43,"automated_acceptable":13,"automated_unsafe":30,"unnecessary_reviews":0,"accuracy":0.3023255813953488,"unsafe_rate":0.6976744186046512,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.3023255813953488,"conditional_unsafe":0.6976744186046512,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.1860200013584818,"upper":0.45105364177729607,"confidence_level":0.95}},"effective":{"n":43,"acceptable":18,"unsafe":3,"review":24,"automated_count":19,"automated_acceptable":13,"automated_unsafe":3,"unnecessary_reviews":19,"accuracy":0.4186046511627907,"unsafe_rate":0.06976744186046512,"review_rate":0.5581395348837209,"automated_coverage":0.4418604651162791,"conditional_correctness":0.6842105263157895,"conditional_unsafe":0.15789473684210525,"unnecessary_review_rate":0.4418604651162791,"accuracy_wilson_95":{"lower":0.28384550927849195,"upper":0.5667142261303578,"confidence_level":0.95}},"confusion":{"raw":{"keep":{"keep":12,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0},"drop":{"keep":15,"drop":1,"mask":0,"pseudonymize":0,"generalize":0,"review":0},"mask":{"keep":2,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0},"pseudonymize":{"keep":2,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0},"generalize":{"keep":6,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0},"review":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0}},"effective":{"keep":{"keep":11,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":1},"drop":{"keep":2,"drop":1,"mask":3,"pseudonymize":0,"generalize":0,"review":10},"mask":{"keep":0,"drop":0,"mask":1,"pseudonymize":0,"generalize":0,"review":1},"pseudonymize":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":2},"generalize":{"keep":1,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":5},"review":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0}}},"probability_metrics":{"singleton_n":38,"multiacceptable_n":5,"multiacceptable_coverage":0.11627906976744186,"nll":2.31958562845562,"nll_zero_mass_n":0,"brier":1.0623622185594461,"acceptable_set_n":43,"acceptable_set_nll":2.2930518972875116,"acceptable_set_zero_mass_n":0},"calibration_all_labels":{"equal_width":{"n":43,"bin_count":10,"ece":0.5328763314457827,"signed_mean_gap":0.49643025633900667,"absolute_mean_gap":0.49643025633900667,"bins":[{"index":0,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.0,"upper":0.1,"upper_inclusive":false},{"index":1,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.1,"upper":0.2,"upper_inclusive":false},{"index":2,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.2,"upper":0.3,"upper_inclusive":false},{"index":3,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.3,"upper":0.4,"upper_inclusive":false},{"index":4,"count":1,"confidence":0.4946397840976715,"empirical_correctness":0.0,"signed_gap":0.4946397840976715,"absolute_gap":0.4946397840976715,"lower":0.4,"upper":0.5,"upper_inclusive":false},{"index":5,"count":4,"confidence":0.5541023463010788,"empirical_correctness":0.75,"signed_gap":-0.1958976536989212,"absolute_gap":0.1958976536989212,"lower":0.5,"upper":0.6,"upper_inclusive":false},{"index":6,"count":5,"confidence":0.670441210269928,"empirical_correctness":0.4,"signed_gap":0.27044121026992796,"absolute_gap":0.27044121026992796,"lower":0.6,"upper":0.7,"upper_inclusive":false},{"index":7,"count":10,"confidence":0.7550723850727081,"empirical_correctness":0.2,"signed_gap":0.5550723850727082,"absolute_gap":0.5550723850727082,"lower":0.7,"upper":0.8,"upper_inclusive":false},{"index":8,"count":9,"confidence":0.8477534519301521,"empirical_correctness":0.3333333333333333,"signed_gap":0.5144201185968187,"absolute_gap":0.5144201185968187,"lower":0.8,"upper":0.9,"upper_inclusive":false},{"index":9,"count":14,"confidence":0.935910063130515,"empirical_correctness":0.21428571428571427,"signed_gap":0.7216243488448006,"absolute_gap":0.7216243488448006,"lower":0.9,"upper":1.0,"upper_inclusive":true}]},"equal_mass":{"n":43,"bin_count":10,"ece":0.49643025633900667,"signed_mean_gap":0.49643025633900667,"absolute_mean_gap":0.49643025633900667,"bins":[{"index":0,"count":4,"confidence":0.5293536111712456,"empirical_correctness":0.5,"signed_gap":0.029353611171245575,"absolute_gap":0.029353611171245575,"min_confidence":0.4946397840976715,"max_confidence":0.5678314566612244},{"index":1,"count":4,"confidence":0.6407871246337891,"empirical_correctness":0.5,"signed_gap":0.14078712463378906,"absolute_gap":0.14078712463378906,"min_confidence":0.5936347246170044,"max_confidence":0.6586597561836243},{"index":2,"count":4,"confidence":0.6983876824378967,"empirical_correctness":0.25,"signed_gap":0.44838768243789673,"absolute_gap":0.44838768243789673,"min_confidence":0.6879045963287354,"max_confidence":0.7059509754180908},{"index":3,"count":5,"confidence":0.7541020035743713,"empirical_correctness":0.2,"signed_gap":0.5541020035743713,"absolute_gap":0.5541020035743713,"min_confidence":0.7392433285713196,"max_confidence":0.7822433114051819},{"index":4,"count":4,"confidence":0.7932212650775909,"empirical_correctness":0.25,"signed_gap":0.5432212650775909,"absolute_gap":0.5432212650775909,"min_confidence":0.785129725933075,"max_confidence":0.803529679775238},{"index":5,"count":4,"confidence":0.8270363211631775,"empirical_correctness":0.25,"signed_gap":0.5770363211631775,"absolute_gap":0.5770363211631775,"min_confidence":0.804184079170227,"max_confidence":0.8611366152763367},{"index":6,"count":5,"confidence":0.8848169207572937,"empirical_correctness":0.4,"signed_gap":0.48481692075729366,"absolute_gap":0.48481692075729366,"min_confidence":0.868104100227356,"max_confidence":0.9059785008430481},{"index":7,"count":4,"confidence":0.9253856986761093,"empirical_correctness":0.25,"signed_gap":0.6753856986761093,"absolute_gap":0.6753856986761093,"min_confidence":0.9143377542495728,"max_confidence":0.9337130784988403},{"index":8,"count":4,"confidence":0.9372110217809677,"empirical_correctness":0.25,"signed_gap":0.6872110217809677,"absolute_gap":0.6872110217809677,"min_confidence":0.9345223307609558,"max_confidence":0.941292941570282},{"index":9,"count":5,"confidence":0.9492751002311707,"empirical_correctness":0.2,"signed_gap":0.7492751002311706,"absolute_gap":0.7492751002311706,"min_confidence":0.9429146647453308,"max_confidence":0.9539339542388916}]}},"calibration_singleton_labels":{"equal_width":{"n":38,"bin_count":10,"ece":0.494036507449652,"signed_mean_gap":0.452794896144616,"absolute_mean_gap":0.452794896144616,"bins":[{"index":0,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.0,"upper":0.1,"upper_inclusive":false},{"index":1,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.1,"upper":0.2,"upper_inclusive":false},{"index":2,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.2,"upper":0.3,"upper_inclusive":false},{"index":3,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.3,"upper":0.4,"upper_inclusive":false},{"index":4,"count":1,"confidence":0.4946397840976715,"empirical_correctness":0.0,"signed_gap":0.4946397840976715,"absolute_gap":0.4946397840976715,"lower":0.4,"upper":0.5,"upper_inclusive":false},{"index":5,"count":4,"confidence":0.5541023463010788,"empirical_correctness":0.75,"signed_gap":-0.1958976536989212,"absolute_gap":0.1958976536989212,"lower":0.5,"upper":0.6,"upper_inclusive":false},{"index":6,"count":5,"confidence":0.670441210269928,"empirical_correctness":0.4,"signed_gap":0.27044121026992796,"absolute_gap":0.27044121026992796,"lower":0.6,"upper":0.7,"upper_inclusive":false},{"index":7,"count":7,"confidence":0.7506364669118609,"empirical_correctness":0.2857142857142857,"signed_gap":0.4649221811975752,"absolute_gap":0.4649221811975752,"lower":0.7,"upper":0.8,"upper_inclusive":false},{"index":8,"count":8,"confidence":0.8422588631510735,"empirical_correctness":0.375,"signed_gap":0.46725886315107346,"absolute_gap":0.46725886315107346,"lower":0.8,"upper":0.9,"upper_inclusive":false},{"index":9,"count":13,"confidence":0.9346480507117051,"empirical_correctness":0.23076923076923078,"signed_gap":0.7038788199424744,"absolute_gap":0.7038788199424744,"lower":0.9,"upper":1.0,"upper_inclusive":true}]},"equal_mass":{"n":38,"bin_count":10,"ece":0.48056751724920777,"signed_mean_gap":0.452794896144616,"absolute_mean_gap":0.452794896144616,"bins":[{"index":0,"count":3,"confidence":0.5165276626745859,"empirical_correctness":0.3333333333333333,"signed_gap":0.18319432934125263,"absolute_gap":0.18319432934125263,"min_confidence":0.4946397840976715,"max_confidence":0.549000084400177},{"index":1,"count":4,"confidence":0.6180800497531891,"empirical_correctness":0.75,"signed_gap":-0.1319199502468109,"absolute_gap":0.1319199502468109,"min_confidence":0.5678314566612244,"max_confidence":0.6561287045478821},{"index":2,"count":4,"confidence":0.6865648776292801,"empirical_correctness":0.25,"signed_gap":0.4365648776292801,"absolute_gap":0.4365648776292801,"min_confidence":0.6586597561836243,"max_confidence":0.7049074769020081},{"index":3,"count":4,"confidence":0.7421938627958298,"empirical_correctness":0.25,"signed_gap":0.4921938627958298,"absolute_gap":0.4921938627958298,"min_confidence":0.7059509754180908,"max_confidence":0.7822433114051819},{"index":4,"count":4,"confidence":0.797121524810791,"empirical_correctness":0.5,"signed_gap":0.297121524810791,"absolute_gap":0.297121524810791,"min_confidence":0.785129725933075,"max_confidence":0.804184079170227},{"index":5,"count":3,"confidence":0.8346537351608276,"empirical_correctness":0.0,"signed_gap":0.8346537351608276,"absolute_gap":0.8346537351608276,"min_confidence":0.8079391717910767,"max_confidence":0.8611366152763367},{"index":6,"count":4,"confidence":0.8830936104059219,"empirical_correctness":0.5,"signed_gap":0.38309361040592194,"absolute_gap":0.38309361040592194,"min_confidence":0.868104100227356,"max_confidence":0.9059785008430481},{"index":7,"count":4,"confidence":0.9253856986761093,"empirical_correctness":0.25,"signed_gap":0.6753856986761093,"absolute_gap":0.6753856986761093,"min_confidence":0.9143377542495728,"max_confidence":0.9337130784988403},{"index":8,"count":4,"confidence":0.9372110217809677,"empirical_correctness":0.25,"signed_gap":0.6872110217809677,"absolute_gap":0.6872110217809677,"min_confidence":0.9345223307609558,"max_confidence":0.941292941570282},{"index":9,"count":4,"confidence":0.9485148191452026,"empirical_correctness":0.25,"signed_gap":0.6985148191452026,"absolute_gap":0.6985148191452026,"min_confidence":0.9429146647453308,"max_confidence":0.9539339542388916}]}},"latency_ms":{"n":43,"median":37.72479994222522,"p95":40.91835995204747},"per_action":{"keep":{"n":12,"raw":{"n":12,"acceptable":12,"unsafe":0,"review":0,"automated_count":12,"automated_acceptable":12,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":1.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.7575059933447592,"upper":1.0,"confidence_level":0.95}},"effective":{"n":12,"acceptable":11,"unsafe":0,"review":1,"automated_count":11,"automated_acceptable":11,"automated_unsafe":0,"unnecessary_reviews":1,"accuracy":0.9166666666666666,"unsafe_rate":0.0,"review_rate":0.08333333333333333,"automated_coverage":0.9166666666666666,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.08333333333333333,"accuracy_wilson_95":{"lower":0.646120088858883,"upper":0.9851349055950829,"confidence_level":0.95}}},"drop":{"n":16,"raw":{"n":16,"acceptable":1,"unsafe":15,"review":0,"automated_count":16,"automated_acceptable":1,"automated_unsafe":15,"unnecessary_reviews":0,"accuracy":0.0625,"unsafe_rate":0.9375,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0625,"conditional_unsafe":0.9375,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.01111934476464252,"upper":0.28328737570298945,"confidence_level":0.95}},"effective":{"n":16,"acceptable":1,"unsafe":2,"review":10,"automated_count":6,"automated_acceptable":1,"automated_unsafe":2,"unnecessary_reviews":10,"accuracy":0.0625,"unsafe_rate":0.125,"review_rate":0.625,"automated_coverage":0.375,"conditional_correctness":0.16666666666666666,"conditional_unsafe":0.3333333333333333,"unnecessary_review_rate":0.625,"accuracy_wilson_95":{"lower":0.01111934476464252,"upper":0.28328737570298945,"confidence_level":0.95}}},"mask":{"n":2,"raw":{"n":2,"acceptable":0,"unsafe":2,"review":0,"automated_count":2,"automated_acceptable":0,"automated_unsafe":2,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":1.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":1.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.6576197724933469,"confidence_level":0.95}},"effective":{"n":2,"acceptable":1,"unsafe":0,"review":1,"automated_count":1,"automated_acceptable":1,"automated_unsafe":0,"unnecessary_reviews":1,"accuracy":0.5,"unsafe_rate":0.0,"review_rate":0.5,"automated_coverage":0.5,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.5,"accuracy_wilson_95":{"lower":0.09453120573423074,"upper":0.9054687942657693,"confidence_level":0.95}}},"pseudonymize":{"n":2,"raw":{"n":2,"acceptable":0,"unsafe":2,"review":0,"automated_count":2,"automated_acceptable":0,"automated_unsafe":2,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":1.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":1.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.6576197724933469,"confidence_level":0.95}},"effective":{"n":2,"acceptable":0,"unsafe":0,"review":2,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":2,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.6576197724933469,"confidence_level":0.95}}},"generalize":{"n":6,"raw":{"n":6,"acceptable":0,"unsafe":6,"review":0,"automated_count":6,"automated_acceptable":0,"automated_unsafe":6,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":1.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":1.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":2.7755575615628914e-17,"upper":0.3903342879021653,"confidence_level":0.95}},"effective":{"n":6,"acceptable":0,"unsafe":1,"review":5,"automated_count":1,"automated_acceptable":0,"automated_unsafe":1,"unnecessary_reviews":5,"accuracy":0.0,"unsafe_rate":0.16666666666666666,"review_rate":0.8333333333333334,"automated_coverage":0.16666666666666666,"conditional_correctness":0.0,"conditional_unsafe":1.0,"unnecessary_review_rate":0.8333333333333334,"accuracy_wilson_95":{"lower":2.7755575615628914e-17,"upper":0.3903342879021653,"confidence_level":0.95}}},"review":{"n":0,"raw":{"n":0,"acceptable":0,"unsafe":0,"review":0,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":null,"unsafe_rate":null,"review_rate":null,"automated_coverage":null,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":null},"effective":{"n":0,"acceptable":0,"unsafe":0,"review":0,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":null,"unsafe_rate":null,"review_rate":null,"automated_coverage":null,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":null}}}},"diagnostic43-compact_no_samples":{"n":43,"raw":{"n":43,"acceptable":12,"unsafe":31,"review":0,"automated_count":43,"automated_acceptable":12,"automated_unsafe":31,"unnecessary_reviews":0,"accuracy":0.27906976744186046,"unsafe_rate":0.7209302325581395,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.27906976744186046,"conditional_unsafe":0.7209302325581395,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.16746613786237488,"upper":0.42691028681878873,"confidence_level":0.95}},"effective":{"n":43,"acceptable":17,"unsafe":3,"review":24,"automated_count":19,"automated_acceptable":12,"automated_unsafe":3,"unnecessary_reviews":19,"accuracy":0.3953488372093023,"unsafe_rate":0.06976744186046512,"review_rate":0.5581395348837209,"automated_coverage":0.4418604651162791,"conditional_correctness":0.631578947368421,"conditional_unsafe":0.15789473684210525,"unnecessary_review_rate":0.4418604651162791,"accuracy_wilson_95":{"lower":0.2636531475829744,"upper":0.5442093693712609,"confidence_level":0.95}},"confusion":{"raw":{"keep":{"keep":12,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0},"drop":{"keep":16,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0},"mask":{"keep":2,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0},"pseudonymize":{"keep":2,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0},"generalize":{"keep":6,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0},"review":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0}},"effective":{"keep":{"keep":11,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":1},"drop":{"keep":2,"drop":0,"mask":4,"pseudonymize":0,"generalize":0,"review":10},"mask":{"keep":0,"drop":0,"mask":1,"pseudonymize":0,"generalize":0,"review":1},"pseudonymize":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":2},"generalize":{"keep":1,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":5},"review":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0}}},"probability_metrics":{"singleton_n":38,"multiacceptable_n":5,"multiacceptable_coverage":0.11627906976744186,"nll":3.471342749963834,"nll_zero_mass_n":0,"brier":1.2441729715675958,"acceptable_set_n":43,"acceptable_set_nll":3.310136932025687,"acceptable_set_zero_mass_n":0},"calibration_all_labels":{"equal_width":{"n":43,"bin_count":10,"ece":0.6111621135889098,"signed_mean_gap":0.6111621135889098,"absolute_mean_gap":0.6111621135889098,"bins":[{"index":0,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.0,"upper":0.1,"upper_inclusive":false},{"index":1,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.1,"upper":0.2,"upper_inclusive":false},{"index":2,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.2,"upper":0.3,"upper_inclusive":false},{"index":3,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.3,"upper":0.4,"upper_inclusive":false},{"index":4,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.4,"upper":0.5,"upper_inclusive":false},{"index":5,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.5,"upper":0.6,"upper_inclusive":false},{"index":6,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.6,"upper":0.7,"upper_inclusive":false},{"index":7,"count":7,"confidence":0.7554990478924343,"empirical_correctness":0.2857142857142857,"signed_gap":0.4697847621781486,"absolute_gap":0.4697847621781486,"lower":0.7,"upper":0.8,"upper_inclusive":false},{"index":8,"count":9,"confidence":0.8306761450237699,"empirical_correctness":0.4444444444444444,"signed_gap":0.38623170057932543,"absolute_gap":0.38623170057932543,"lower":0.8,"upper":0.9,"upper_inclusive":false},{"index":9,"count":27,"confidence":0.9450145275504501,"empirical_correctness":0.2222222222222222,"signed_gap":0.7227923053282279,"absolute_gap":0.7227923053282279,"lower":0.9,"upper":1.0,"upper_inclusive":true}]},"equal_mass":{"n":43,"bin_count":10,"ece":0.6111621135889098,"signed_mean_gap":0.6111621135889098,"absolute_mean_gap":0.6111621135889098,"bins":[{"index":0,"count":4,"confidence":0.7392294406890869,"empirical_correctness":0.0,"signed_gap":0.7392294406890869,"absolute_gap":0.7392294406890869,"min_confidence":0.725081205368042,"max_confidence":0.7486968040466309},{"index":1,"count":4,"confidence":0.7838705629110336,"empirical_correctness":0.75,"signed_gap":0.03387056291103363,"absolute_gap":0.03387056291103363,"min_confidence":0.7645917534828186,"max_confidence":0.8039066791534424},{"index":2,"count":4,"confidence":0.8152537643909454,"empirical_correctness":0.5,"signed_gap":0.31525376439094543,"absolute_gap":0.31525376439094543,"min_confidence":0.8046489953994751,"max_confidence":0.8255011439323425},{"index":3,"count":5,"confidence":0.8630273699760437,"empirical_correctness":0.2,"signed_gap":0.6630273699760436,"absolute_gap":0.6630273699760436,"min_confidence":0.8275113105773926,"max_confidence":0.9039732813835144},{"index":4,"count":4,"confidence":0.9158614873886108,"empirical_correctness":0.25,"signed_gap":0.6658614873886108,"absolute_gap":0.6658614873886108,"min_confidence":0.9078887701034546,"max_confidence":0.92015540599823},{"index":5,"count":4,"confidence":0.9258634448051453,"empirical_correctness":0.25,"signed_gap":0.6758634448051453,"absolute_gap":0.6758634448051453,"min_confidence":0.9210139513015747,"max_confidence":0.9313244819641113},{"index":6,"count":5,"confidence":0.9403690338134766,"empirical_correctness":0.2,"signed_gap":0.7403690338134765,"absolute_gap":0.7403690338134765,"min_confidence":0.9327298998832703,"max_confidence":0.9443495273590088},{"index":7,"count":4,"confidence":0.9562413692474365,"empirical_correctness":0.0,"signed_gap":0.9562413692474365,"absolute_gap":0.9562413692474365,"min_confidence":0.9528045058250427,"max_confidence":0.9619946479797363},{"index":8,"count":4,"confidence":0.9659786522388458,"empirical_correctness":0.25,"signed_gap":0.7159786522388458,"absolute_gap":0.7159786522388458,"min_confidence":0.9635804891586304,"max_confidence":0.9680505394935608},{"index":9,"count":5,"confidence":0.9707587957382202,"empirical_correctness":0.4,"signed_gap":0.5707587957382202,"absolute_gap":0.5707587957382202,"min_confidence":0.9691075086593628,"max_confidence":0.9720401167869568}]}},"calibration_singleton_labels":{"equal_width":{"n":38,"bin_count":10,"ece":0.5813987208040137,"signed_mean_gap":0.5813987208040137,"absolute_mean_gap":0.5813987208040137,"bins":[{"index":0,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.0,"upper":0.1,"upper_inclusive":false},{"index":1,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.1,"upper":0.2,"upper_inclusive":false},{"index":2,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.2,"upper":0.3,"upper_inclusive":false},{"index":3,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.3,"upper":0.4,"upper_inclusive":false},{"index":4,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.4,"upper":0.5,"upper_inclusive":false},{"index":5,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.5,"upper":0.6,"upper_inclusive":false},{"index":6,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.6,"upper":0.7,"upper_inclusive":false},{"index":7,"count":5,"confidence":0.7655357599258423,"empirical_correctness":0.4,"signed_gap":0.3655357599258423,"absolute_gap":0.3655357599258423,"lower":0.7,"upper":0.8,"upper_inclusive":false},{"index":8,"count":8,"confidence":0.8275613486766815,"empirical_correctness":0.5,"signed_gap":0.3275613486766815,"absolute_gap":0.3275613486766815,"lower":0.8,"upper":0.9,"upper_inclusive":false},{"index":9,"count":25,"confidence":0.9457992720603943,"empirical_correctness":0.24,"signed_gap":0.7057992720603943,"absolute_gap":0.7057992720603943,"lower":0.9,"upper":1.0,"upper_inclusive":true}]},"equal_mass":{"n":38,"bin_count":10,"ece":0.6247913790376562,"signed_mean_gap":0.5813987208040137,"absolute_mean_gap":0.5813987208040137,"bins":[{"index":0,"count":3,"confidence":0.7535649935404459,"empirical_correctness":0.0,"signed_gap":0.7535649935404459,"absolute_gap":0.7535649935404459,"min_confidence":0.7474064230918884,"max_confidence":0.7645917534828186},{"index":1,"count":4,"confidence":0.7938848733901978,"empirical_correctness":1.0,"signed_gap":-0.20611512660980225,"absolute_gap":0.20611512660980225,"min_confidence":0.7676367163658142,"max_confidence":0.8046489953994751},{"index":2,"count":4,"confidence":0.8209693431854248,"empirical_correctness":0.5,"signed_gap":0.3209693431854248,"absolute_gap":0.3209693431854248,"min_confidence":0.8138381838798523,"max_confidence":0.8275113105773926},{"index":3,"count":4,"confidence":0.8877982646226883,"empirical_correctness":0.0,"signed_gap":0.8877982646226883,"absolute_gap":0.8877982646226883,"min_confidence":0.8323222398757935,"max_confidence":0.9152465462684631},{"index":4,"count":4,"confidence":0.9214711338281631,"empirical_correctness":0.25,"signed_gap":0.6714711338281631,"absolute_gap":0.6714711338281631,"min_confidence":0.9201552271842957,"max_confidence":0.9245599508285522},{"index":5,"count":3,"confidence":0.9302032589912415,"empirical_correctness":0.3333333333333333,"signed_gap":0.5968699256579082,"absolute_gap":0.5968699256579082,"min_confidence":0.9265553951263428,"max_confidence":0.9327298998832703},{"index":6,"count":4,"confidence":0.9422788172960281,"empirical_correctness":0.25,"signed_gap":0.6922788172960281,"absolute_gap":0.6922788172960281,"min_confidence":0.937920093536377,"max_confidence":0.9443495273590088},{"index":7,"count":4,"confidence":0.9562413692474365,"empirical_correctness":0.0,"signed_gap":0.9562413692474365,"absolute_gap":0.9562413692474365,"min_confidence":0.9528045058250427,"max_confidence":0.9619946479797363},{"index":8,"count":4,"confidence":0.9666462391614914,"empirical_correctness":0.5,"signed_gap":0.4666462391614914,"absolute_gap":0.4666462391614914,"min_confidence":0.9635804891586304,"max_confidence":0.9691075086593628},{"index":9,"count":4,"confidence":0.9711716175079346,"empirical_correctness":0.25,"signed_gap":0.7211716175079346,"absolute_gap":0.7211716175079346,"min_confidence":0.9704800248146057,"max_confidence":0.9720401167869568}]}},"latency_ms":{"n":43,"median":39.39579986035824,"p95":42.08684011828154},"per_action":{"keep":{"n":12,"raw":{"n":12,"acceptable":12,"unsafe":0,"review":0,"automated_count":12,"automated_acceptable":12,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":1.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.7575059933447592,"upper":1.0,"confidence_level":0.95}},"effective":{"n":12,"acceptable":11,"unsafe":0,"review":1,"automated_count":11,"automated_acceptable":11,"automated_unsafe":0,"unnecessary_reviews":1,"accuracy":0.9166666666666666,"unsafe_rate":0.0,"review_rate":0.08333333333333333,"automated_coverage":0.9166666666666666,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.08333333333333333,"accuracy_wilson_95":{"lower":0.646120088858883,"upper":0.9851349055950829,"confidence_level":0.95}}},"drop":{"n":16,"raw":{"n":16,"acceptable":0,"unsafe":16,"review":0,"automated_count":16,"automated_acceptable":0,"automated_unsafe":16,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":1.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":1.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.1936076805344365,"confidence_level":0.95}},"effective":{"n":16,"acceptable":0,"unsafe":2,"review":10,"automated_count":6,"automated_acceptable":0,"automated_unsafe":2,"unnecessary_reviews":10,"accuracy":0.0,"unsafe_rate":0.125,"review_rate":0.625,"automated_coverage":0.375,"conditional_correctness":0.0,"conditional_unsafe":0.3333333333333333,"unnecessary_review_rate":0.625,"accuracy_wilson_95":{"lower":0.0,"upper":0.1936076805344365,"confidence_level":0.95}}},"mask":{"n":2,"raw":{"n":2,"acceptable":0,"unsafe":2,"review":0,"automated_count":2,"automated_acceptable":0,"automated_unsafe":2,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":1.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":1.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.6576197724933469,"confidence_level":0.95}},"effective":{"n":2,"acceptable":1,"unsafe":0,"review":1,"automated_count":1,"automated_acceptable":1,"automated_unsafe":0,"unnecessary_reviews":1,"accuracy":0.5,"unsafe_rate":0.0,"review_rate":0.5,"automated_coverage":0.5,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.5,"accuracy_wilson_95":{"lower":0.09453120573423074,"upper":0.9054687942657693,"confidence_level":0.95}}},"pseudonymize":{"n":2,"raw":{"n":2,"acceptable":0,"unsafe":2,"review":0,"automated_count":2,"automated_acceptable":0,"automated_unsafe":2,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":1.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":1.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.6576197724933469,"confidence_level":0.95}},"effective":{"n":2,"acceptable":0,"unsafe":0,"review":2,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":2,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.6576197724933469,"confidence_level":0.95}}},"generalize":{"n":6,"raw":{"n":6,"acceptable":0,"unsafe":6,"review":0,"automated_count":6,"automated_acceptable":0,"automated_unsafe":6,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":1.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":1.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":2.7755575615628914e-17,"upper":0.3903342879021653,"confidence_level":0.95}},"effective":{"n":6,"acceptable":0,"unsafe":1,"review":5,"automated_count":1,"automated_acceptable":0,"automated_unsafe":1,"unnecessary_reviews":5,"accuracy":0.0,"unsafe_rate":0.16666666666666666,"review_rate":0.8333333333333334,"automated_coverage":0.16666666666666666,"conditional_correctness":0.0,"conditional_unsafe":1.0,"unnecessary_review_rate":0.8333333333333334,"accuracy_wilson_95":{"lower":2.7755575615628914e-17,"upper":0.3903342879021653,"confidence_level":0.95}}},"review":{"n":0,"raw":{"n":0,"acceptable":0,"unsafe":0,"review":0,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":null,"unsafe_rate":null,"review_rate":null,"automated_coverage":null,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":null},"effective":{"n":0,"acceptable":0,"unsafe":0,"review":0,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":null,"unsafe_rate":null,"review_rate":null,"automated_coverage":null,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":null}}}},"challenge56-no_samples":{"n":56,"raw":{"n":56,"acceptable":17,"unsafe":39,"review":0,"automated_count":56,"automated_acceptable":17,"automated_unsafe":39,"unnecessary_reviews":0,"accuracy":0.30357142857142855,"unsafe_rate":0.6964285714285714,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.30357142857142855,"conditional_unsafe":0.6964285714285714,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.19900325085369447,"upper":0.43335865285724834,"confidence_level":0.95}},"effective":{"n":56,"acceptable":16,"unsafe":0,"review":42,"automated_count":14,"automated_acceptable":14,"automated_unsafe":0,"unnecessary_reviews":40,"accuracy":0.2857142857142857,"unsafe_rate":0.0,"review_rate":0.75,"automated_coverage":0.25,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.7142857142857143,"accuracy_wilson_95":{"lower":0.18418775143858906,"upper":0.4147525071551667,"confidence_level":0.95}},"confusion":{"raw":{"keep":{"keep":8,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0},"drop":{"keep":22,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0},"mask":{"keep":0,"drop":0,"mask":8,"pseudonymize":0,"generalize":0,"review":0},"pseudonymize":{"keep":8,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0},"generalize":{"keep":7,"drop":0,"mask":0,"pseudonymize":0,"generalize":1,"review":0},"review":{"keep":2,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0}},"effective":{"keep":{"keep":5,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":3},"drop":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":22},"mask":{"keep":0,"drop":0,"mask":8,"pseudonymize":0,"generalize":0,"review":0},"pseudonymize":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":8},"generalize":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":1,"review":7},"review":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":2}}},"probability_metrics":{"singleton_n":56,"multiacceptable_n":0,"multiacceptable_coverage":0.0,"nll":2.727936346591906,"nll_zero_mass_n":0,"brier":1.2281170714716967,"acceptable_set_n":56,"acceptable_set_nll":2.727936346591906,"acceptable_set_zero_mass_n":0},"calibration_all_labels":{"equal_width":{"n":56,"bin_count":10,"ece":0.5820127555302211,"signed_mean_gap":0.5708722855363573,"absolute_mean_gap":0.5708722855363573,"bins":[{"index":0,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.0,"upper":0.1,"upper_inclusive":false},{"index":1,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.1,"upper":0.2,"upper_inclusive":false},{"index":2,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.2,"upper":0.3,"upper_inclusive":false},{"index":3,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.3,"upper":0.4,"upper_inclusive":false},{"index":4,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.4,"upper":0.5,"upper_inclusive":false},{"index":5,"count":2,"confidence":0.54046630859375,"empirical_correctness":0.5,"signed_gap":0.04046630859375,"absolute_gap":0.04046630859375,"lower":0.5,"upper":0.6,"upper_inclusive":false},{"index":6,"count":4,"confidence":0.6720167100429535,"empirical_correctness":0.75,"signed_gap":-0.07798328995704651,"absolute_gap":0.07798328995704651,"lower":0.6,"upper":0.7,"upper_inclusive":false},{"index":7,"count":4,"confidence":0.7548037022352219,"empirical_correctness":0.5,"signed_gap":0.25480370223522186,"absolute_gap":0.25480370223522186,"lower":0.7,"upper":0.8,"upper_inclusive":false},{"index":8,"count":12,"confidence":0.8530784795681635,"empirical_correctness":0.25,"signed_gap":0.6030784795681635,"absolute_gap":0.6030784795681635,"lower":0.8,"upper":0.9,"upper_inclusive":false},{"index":9,"count":34,"confidence":0.9395203520269955,"empirical_correctness":0.23529411764705882,"signed_gap":0.7042262343799366,"absolute_gap":0.7042262343799366,"lower":0.9,"upper":1.0,"upper_inclusive":true}]},"equal_mass":{"n":56,"bin_count":10,"ece":0.5708722855363574,"signed_mean_gap":0.5708722855363574,"absolute_mean_gap":0.5708722855363574,"bins":[{"index":0,"count":5,"confidence":0.616436243057251,"empirical_correctness":0.6,"signed_gap":0.016436243057251,"absolute_gap":0.016436243057251,"min_confidence":0.5063178539276123,"max_confidence":0.681142270565033},{"index":1,"count":6,"confidence":0.7518860896428426,"empirical_correctness":0.5,"signed_gap":0.25188608964284265,"absolute_gap":0.25188608964284265,"min_confidence":0.6868182420730591,"max_confidence":0.8052834868431091},{"index":2,"count":5,"confidence":0.835486114025116,"empirical_correctness":0.4,"signed_gap":0.43548611402511594,"absolute_gap":0.43548611402511594,"min_confidence":0.805404007434845,"max_confidence":0.8613754510879517},{"index":3,"count":6,"confidence":0.8757046163082123,"empirical_correctness":0.16666666666666666,"signed_gap":0.7090379496415457,"absolute_gap":0.7090379496415457,"min_confidence":0.8622866868972778,"max_confidence":0.897314727306366},{"index":4,"count":6,"confidence":0.9040419658025106,"empirical_correctness":0.16666666666666666,"signed_gap":0.737375299135844,"absolute_gap":0.737375299135844,"min_confidence":0.9007101058959961,"max_confidence":0.9069519639015198},{"index":5,"count":5,"confidence":0.9179298996925354,"empirical_correctness":0.2,"signed_gap":0.7179298996925354,"absolute_gap":0.7179298996925354,"min_confidence":0.9071710705757141,"max_confidence":0.9291270971298218},{"index":6,"count":6,"confidence":0.9344071547190348,"empirical_correctness":0.0,"signed_gap":0.9344071547190348,"absolute_gap":0.9344071547190348,"min_confidence":0.9310175776481628,"max_confidence":0.9362584352493286},{"index":7,"count":5,"confidence":0.9451405525207519,"empirical_correctness":0.2,"signed_gap":0.745140552520752,"absolute_gap":0.745140552520752,"min_confidence":0.9415803551673889,"max_confidence":0.9506464600563049},{"index":8,"count":6,"confidence":0.9604697525501251,"empirical_correctness":0.3333333333333333,"signed_gap":0.6271364192167919,"absolute_gap":0.6271364192167919,"min_confidence":0.9559230208396912,"max_confidence":0.968097448348999},{"index":9,"count":6,"confidence":0.9724710782368978,"empirical_correctness":0.5,"signed_gap":0.4724710782368978,"absolute_gap":0.4724710782368978,"min_confidence":0.9689539074897766,"max_confidence":0.9777429103851318}]}},"calibration_singleton_labels":{"equal_width":{"n":56,"bin_count":10,"ece":0.5820127555302211,"signed_mean_gap":0.5708722855363573,"absolute_mean_gap":0.5708722855363573,"bins":[{"index":0,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.0,"upper":0.1,"upper_inclusive":false},{"index":1,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.1,"upper":0.2,"upper_inclusive":false},{"index":2,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.2,"upper":0.3,"upper_inclusive":false},{"index":3,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.3,"upper":0.4,"upper_inclusive":false},{"index":4,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.4,"upper":0.5,"upper_inclusive":false},{"index":5,"count":2,"confidence":0.54046630859375,"empirical_correctness":0.5,"signed_gap":0.04046630859375,"absolute_gap":0.04046630859375,"lower":0.5,"upper":0.6,"upper_inclusive":false},{"index":6,"count":4,"confidence":0.6720167100429535,"empirical_correctness":0.75,"signed_gap":-0.07798328995704651,"absolute_gap":0.07798328995704651,"lower":0.6,"upper":0.7,"upper_inclusive":false},{"index":7,"count":4,"confidence":0.7548037022352219,"empirical_correctness":0.5,"signed_gap":0.25480370223522186,"absolute_gap":0.25480370223522186,"lower":0.7,"upper":0.8,"upper_inclusive":false},{"index":8,"count":12,"confidence":0.8530784795681635,"empirical_correctness":0.25,"signed_gap":0.6030784795681635,"absolute_gap":0.6030784795681635,"lower":0.8,"upper":0.9,"upper_inclusive":false},{"index":9,"count":34,"confidence":0.9395203520269955,"empirical_correctness":0.23529411764705882,"signed_gap":0.7042262343799366,"absolute_gap":0.7042262343799366,"lower":0.9,"upper":1.0,"upper_inclusive":true}]},"equal_mass":{"n":56,"bin_count":10,"ece":0.5708722855363574,"signed_mean_gap":0.5708722855363574,"absolute_mean_gap":0.5708722855363574,"bins":[{"index":0,"count":5,"confidence":0.616436243057251,"empirical_correctness":0.6,"signed_gap":0.016436243057251,"absolute_gap":0.016436243057251,"min_confidence":0.5063178539276123,"max_confidence":0.681142270565033},{"index":1,"count":6,"confidence":0.7518860896428426,"empirical_correctness":0.5,"signed_gap":0.25188608964284265,"absolute_gap":0.25188608964284265,"min_confidence":0.6868182420730591,"max_confidence":0.8052834868431091},{"index":2,"count":5,"confidence":0.835486114025116,"empirical_correctness":0.4,"signed_gap":0.43548611402511594,"absolute_gap":0.43548611402511594,"min_confidence":0.805404007434845,"max_confidence":0.8613754510879517},{"index":3,"count":6,"confidence":0.8757046163082123,"empirical_correctness":0.16666666666666666,"signed_gap":0.7090379496415457,"absolute_gap":0.7090379496415457,"min_confidence":0.8622866868972778,"max_confidence":0.897314727306366},{"index":4,"count":6,"confidence":0.9040419658025106,"empirical_correctness":0.16666666666666666,"signed_gap":0.737375299135844,"absolute_gap":0.737375299135844,"min_confidence":0.9007101058959961,"max_confidence":0.9069519639015198},{"index":5,"count":5,"confidence":0.9179298996925354,"empirical_correctness":0.2,"signed_gap":0.7179298996925354,"absolute_gap":0.7179298996925354,"min_confidence":0.9071710705757141,"max_confidence":0.9291270971298218},{"index":6,"count":6,"confidence":0.9344071547190348,"empirical_correctness":0.0,"signed_gap":0.9344071547190348,"absolute_gap":0.9344071547190348,"min_confidence":0.9310175776481628,"max_confidence":0.9362584352493286},{"index":7,"count":5,"confidence":0.9451405525207519,"empirical_correctness":0.2,"signed_gap":0.745140552520752,"absolute_gap":0.745140552520752,"min_confidence":0.9415803551673889,"max_confidence":0.9506464600563049},{"index":8,"count":6,"confidence":0.9604697525501251,"empirical_correctness":0.3333333333333333,"signed_gap":0.6271364192167919,"absolute_gap":0.6271364192167919,"min_confidence":0.9559230208396912,"max_confidence":0.968097448348999},{"index":9,"count":6,"confidence":0.9724710782368978,"empirical_correctness":0.5,"signed_gap":0.4724710782368978,"absolute_gap":0.4724710782368978,"min_confidence":0.9689539074897766,"max_confidence":0.9777429103851318}]}},"latency_ms":{"n":56,"median":44.99590001069009,"p95":47.46007494395599},"per_action":{"keep":{"n":8,"raw":{"n":8,"acceptable":8,"unsafe":0,"review":0,"automated_count":8,"automated_acceptable":8,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":1.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.6755924351161198,"upper":1.0,"confidence_level":0.95}},"effective":{"n":8,"acceptable":5,"unsafe":0,"review":3,"automated_count":5,"automated_acceptable":5,"automated_unsafe":0,"unnecessary_reviews":3,"accuracy":0.625,"unsafe_rate":0.0,"review_rate":0.375,"automated_coverage":0.625,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.375,"accuracy_wilson_95":{"lower":0.30574239460262737,"upper":0.8631557141764026,"confidence_level":0.95}}},"drop":{"n":22,"raw":{"n":22,"acceptable":0,"unsafe":22,"review":0,"automated_count":22,"automated_acceptable":0,"automated_unsafe":22,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":1.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":1.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.14865487460862087,"confidence_level":0.95}},"effective":{"n":22,"acceptable":0,"unsafe":0,"review":22,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":22,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.14865487460862087,"confidence_level":0.95}}},"mask":{"n":8,"raw":{"n":8,"acceptable":8,"unsafe":0,"review":0,"automated_count":8,"automated_acceptable":8,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":1.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.6755924351161198,"upper":1.0,"confidence_level":0.95}},"effective":{"n":8,"acceptable":8,"unsafe":0,"review":0,"automated_count":8,"automated_acceptable":8,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":1.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.6755924351161198,"upper":1.0,"confidence_level":0.95}}},"pseudonymize":{"n":8,"raw":{"n":8,"acceptable":0,"unsafe":8,"review":0,"automated_count":8,"automated_acceptable":0,"automated_unsafe":8,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":1.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":1.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.32440756488388023,"confidence_level":0.95}},"effective":{"n":8,"acceptable":0,"unsafe":0,"review":8,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":8,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.32440756488388023,"confidence_level":0.95}}},"generalize":{"n":8,"raw":{"n":8,"acceptable":1,"unsafe":7,"review":0,"automated_count":8,"automated_acceptable":1,"automated_unsafe":7,"unnecessary_reviews":0,"accuracy":0.125,"unsafe_rate":0.875,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.125,"conditional_unsafe":0.875,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.02241749145005667,"upper":0.4708881822128535,"confidence_level":0.95}},"effective":{"n":8,"acceptable":1,"unsafe":0,"review":7,"automated_count":1,"automated_acceptable":1,"automated_unsafe":0,"unnecessary_reviews":7,"accuracy":0.125,"unsafe_rate":0.0,"review_rate":0.875,"automated_coverage":0.125,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.875,"accuracy_wilson_95":{"lower":0.02241749145005667,"upper":0.4708881822128535,"confidence_level":0.95}}},"review":{"n":2,"raw":{"n":2,"acceptable":0,"unsafe":2,"review":0,"automated_count":2,"automated_acceptable":0,"automated_unsafe":2,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":1.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":1.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.6576197724933469,"confidence_level":0.95}},"effective":{"n":2,"acceptable":2,"unsafe":0,"review":2,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":1.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.34238022750665303,"upper":1.0,"confidence_level":0.95}}}}},"challenge56-compact_no_samples":{"n":56,"raw":{"n":56,"acceptable":8,"unsafe":48,"review":0,"automated_count":56,"automated_acceptable":8,"automated_unsafe":48,"unnecessary_reviews":0,"accuracy":0.14285714285714285,"unsafe_rate":0.8571428571428571,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.14285714285714285,"conditional_unsafe":0.8571428571428571,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.07420788536920624,"upper":0.25735921228705333,"confidence_level":0.95}},"effective":{"n":56,"acceptable":7,"unsafe":0,"review":51,"automated_count":5,"automated_acceptable":5,"automated_unsafe":0,"unnecessary_reviews":49,"accuracy":0.125,"unsafe_rate":0.0,"review_rate":0.9107142857142857,"automated_coverage":0.08928571428571429,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.875,"accuracy_wilson_95":{"lower":0.06189068695717122,"upper":0.2362547655819014,"confidence_level":0.95}},"confusion":{"raw":{"keep":{"keep":8,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0},"drop":{"keep":22,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0},"mask":{"keep":8,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0},"pseudonymize":{"keep":8,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0},"generalize":{"keep":8,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0},"review":{"keep":2,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0}},"effective":{"keep":{"keep":5,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":3},"drop":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":22},"mask":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":8},"pseudonymize":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":8},"generalize":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":8},"review":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":2}}},"probability_metrics":{"singleton_n":56,"multiacceptable_n":0,"multiacceptable_coverage":0.0,"nll":4.072904014206089,"nll_zero_mass_n":0,"brier":1.5156677035349728,"acceptable_set_n":56,"acceptable_set_nll":4.072904014206089,"acceptable_set_zero_mass_n":0},"calibration_all_labels":{"equal_width":{"n":56,"bin_count":10,"ece":0.7637562954000064,"signed_mean_gap":0.7637562954000064,"absolute_mean_gap":0.7637562954000064,"bins":[{"index":0,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.0,"upper":0.1,"upper_inclusive":false},{"index":1,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.1,"upper":0.2,"upper_inclusive":false},{"index":2,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.2,"upper":0.3,"upper_inclusive":false},{"index":3,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.3,"upper":0.4,"upper_inclusive":false},{"index":4,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.4,"upper":0.5,"upper_inclusive":false},{"index":5,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.5,"upper":0.6,"upper_inclusive":false},{"index":6,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.6,"upper":0.7,"upper_inclusive":false},{"index":7,"count":2,"confidence":0.7447120547294617,"empirical_correctness":0.0,"signed_gap":0.7447120547294617,"absolute_gap":0.7447120547294617,"lower":0.7,"upper":0.8,"upper_inclusive":false},{"index":8,"count":13,"confidence":0.850731569987077,"empirical_correctness":0.0,"signed_gap":0.850731569987077,"absolute_gap":0.850731569987077,"lower":0.8,"upper":0.9,"upper_inclusive":false},{"index":9,"count":41,"confidence":0.9322297078807179,"empirical_correctness":0.1951219512195122,"signed_gap":0.7371077566612056,"absolute_gap":0.7371077566612056,"lower":0.9,"upper":1.0,"upper_inclusive":true}]},"equal_mass":{"n":56,"bin_count":10,"ece":0.7637562954000064,"signed_mean_gap":0.7637562954000064,"absolute_mean_gap":0.7637562954000064,"bins":[{"index":0,"count":5,"confidence":0.789150083065033,"empirical_correctness":0.0,"signed_gap":0.789150083065033,"absolute_gap":0.789150083065033,"min_confidence":0.744382381439209,"max_confidence":0.8233417272567749},{"index":1,"count":6,"confidence":0.8452804982662201,"empirical_correctness":0.0,"signed_gap":0.8452804982662201,"absolute_gap":0.8452804982662201,"min_confidence":0.83163982629776,"max_confidence":0.868457019329071},{"index":2,"count":5,"confidence":0.8867631673812866,"empirical_correctness":0.0,"signed_gap":0.8867631673812866,"absolute_gap":0.8867631673812866,"min_confidence":0.8721866011619568,"max_confidence":0.9023147225379944},{"index":3,"count":6,"confidence":0.9076615770657858,"empirical_correctness":0.0,"signed_gap":0.9076615770657858,"absolute_gap":0.9076615770657858,"min_confidence":0.9028782844543457,"max_confidence":0.9122232794761658},{"index":4,"count":6,"confidence":0.9158720473448435,"empirical_correctness":0.3333333333333333,"signed_gap":0.5825387140115101,"absolute_gap":0.5825387140115101,"min_confidence":0.9148119688034058,"max_confidence":0.9167957901954651},{"index":5,"count":5,"confidence":0.9244845747947693,"empirical_correctness":0.2,"signed_gap":0.7244845747947692,"absolute_gap":0.7244845747947692,"min_confidence":0.916918158531189,"max_confidence":0.9287477731704712},{"index":6,"count":6,"confidence":0.9328752557436625,"empirical_correctness":0.3333333333333333,"signed_gap":0.5995419224103291,"absolute_gap":0.5995419224103291,"min_confidence":0.9293072819709778,"max_confidence":0.9370254874229431},{"index":7,"count":5,"confidence":0.9402053833007813,"empirical_correctness":0.2,"signed_gap":0.7402053833007813,"absolute_gap":0.7402053833007813,"min_confidence":0.9379842281341553,"max_confidence":0.9426330924034119},{"index":8,"count":6,"confidence":0.9481121798356374,"empirical_correctness":0.3333333333333333,"signed_gap":0.6147788465023041,"absolute_gap":0.6147788465023041,"min_confidence":0.9429446458816528,"max_confidence":0.9534339904785156},{"index":9,"count":6,"confidence":0.9614211916923523,"empirical_correctness":0.0,"signed_gap":0.9614211916923523,"absolute_gap":0.9614211916923523,"min_confidence":0.9548918008804321,"max_confidence":0.9707057476043701}]}},"calibration_singleton_labels":{"equal_width":{"n":56,"bin_count":10,"ece":0.7637562954000064,"signed_mean_gap":0.7637562954000064,"absolute_mean_gap":0.7637562954000064,"bins":[{"index":0,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.0,"upper":0.1,"upper_inclusive":false},{"index":1,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.1,"upper":0.2,"upper_inclusive":false},{"index":2,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.2,"upper":0.3,"upper_inclusive":false},{"index":3,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.3,"upper":0.4,"upper_inclusive":false},{"index":4,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.4,"upper":0.5,"upper_inclusive":false},{"index":5,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.5,"upper":0.6,"upper_inclusive":false},{"index":6,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.6,"upper":0.7,"upper_inclusive":false},{"index":7,"count":2,"confidence":0.7447120547294617,"empirical_correctness":0.0,"signed_gap":0.7447120547294617,"absolute_gap":0.7447120547294617,"lower":0.7,"upper":0.8,"upper_inclusive":false},{"index":8,"count":13,"confidence":0.850731569987077,"empirical_correctness":0.0,"signed_gap":0.850731569987077,"absolute_gap":0.850731569987077,"lower":0.8,"upper":0.9,"upper_inclusive":false},{"index":9,"count":41,"confidence":0.9322297078807179,"empirical_correctness":0.1951219512195122,"signed_gap":0.7371077566612056,"absolute_gap":0.7371077566612056,"lower":0.9,"upper":1.0,"upper_inclusive":true}]},"equal_mass":{"n":56,"bin_count":10,"ece":0.7637562954000064,"signed_mean_gap":0.7637562954000064,"absolute_mean_gap":0.7637562954000064,"bins":[{"index":0,"count":5,"confidence":0.789150083065033,"empirical_correctness":0.0,"signed_gap":0.789150083065033,"absolute_gap":0.789150083065033,"min_confidence":0.744382381439209,"max_confidence":0.8233417272567749},{"index":1,"count":6,"confidence":0.8452804982662201,"empirical_correctness":0.0,"signed_gap":0.8452804982662201,"absolute_gap":0.8452804982662201,"min_confidence":0.83163982629776,"max_confidence":0.868457019329071},{"index":2,"count":5,"confidence":0.8867631673812866,"empirical_correctness":0.0,"signed_gap":0.8867631673812866,"absolute_gap":0.8867631673812866,"min_confidence":0.8721866011619568,"max_confidence":0.9023147225379944},{"index":3,"count":6,"confidence":0.9076615770657858,"empirical_correctness":0.0,"signed_gap":0.9076615770657858,"absolute_gap":0.9076615770657858,"min_confidence":0.9028782844543457,"max_confidence":0.9122232794761658},{"index":4,"count":6,"confidence":0.9158720473448435,"empirical_correctness":0.3333333333333333,"signed_gap":0.5825387140115101,"absolute_gap":0.5825387140115101,"min_confidence":0.9148119688034058,"max_confidence":0.9167957901954651},{"index":5,"count":5,"confidence":0.9244845747947693,"empirical_correctness":0.2,"signed_gap":0.7244845747947692,"absolute_gap":0.7244845747947692,"min_confidence":0.916918158531189,"max_confidence":0.9287477731704712},{"index":6,"count":6,"confidence":0.9328752557436625,"empirical_correctness":0.3333333333333333,"signed_gap":0.5995419224103291,"absolute_gap":0.5995419224103291,"min_confidence":0.9293072819709778,"max_confidence":0.9370254874229431},{"index":7,"count":5,"confidence":0.9402053833007813,"empirical_correctness":0.2,"signed_gap":0.7402053833007813,"absolute_gap":0.7402053833007813,"min_confidence":0.9379842281341553,"max_confidence":0.9426330924034119},{"index":8,"count":6,"confidence":0.9481121798356374,"empirical_correctness":0.3333333333333333,"signed_gap":0.6147788465023041,"absolute_gap":0.6147788465023041,"min_confidence":0.9429446458816528,"max_confidence":0.9534339904785156},{"index":9,"count":6,"confidence":0.9614211916923523,"empirical_correctness":0.0,"signed_gap":0.9614211916923523,"absolute_gap":0.9614211916923523,"min_confidence":0.9548918008804321,"max_confidence":0.9707057476043701}]}},"latency_ms":{"n":56,"median":43.34870004095137,"p95":49.448200094047934},"per_action":{"keep":{"n":8,"raw":{"n":8,"acceptable":8,"unsafe":0,"review":0,"automated_count":8,"automated_acceptable":8,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":1.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.6755924351161198,"upper":1.0,"confidence_level":0.95}},"effective":{"n":8,"acceptable":5,"unsafe":0,"review":3,"automated_count":5,"automated_acceptable":5,"automated_unsafe":0,"unnecessary_reviews":3,"accuracy":0.625,"unsafe_rate":0.0,"review_rate":0.375,"automated_coverage":0.625,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.375,"accuracy_wilson_95":{"lower":0.30574239460262737,"upper":0.8631557141764026,"confidence_level":0.95}}},"drop":{"n":22,"raw":{"n":22,"acceptable":0,"unsafe":22,"review":0,"automated_count":22,"automated_acceptable":0,"automated_unsafe":22,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":1.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":1.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.14865487460862087,"confidence_level":0.95}},"effective":{"n":22,"acceptable":0,"unsafe":0,"review":22,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":22,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.14865487460862087,"confidence_level":0.95}}},"mask":{"n":8,"raw":{"n":8,"acceptable":0,"unsafe":8,"review":0,"automated_count":8,"automated_acceptable":0,"automated_unsafe":8,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":1.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":1.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.32440756488388023,"confidence_level":0.95}},"effective":{"n":8,"acceptable":0,"unsafe":0,"review":8,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":8,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.32440756488388023,"confidence_level":0.95}}},"pseudonymize":{"n":8,"raw":{"n":8,"acceptable":0,"unsafe":8,"review":0,"automated_count":8,"automated_acceptable":0,"automated_unsafe":8,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":1.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":1.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.32440756488388023,"confidence_level":0.95}},"effective":{"n":8,"acceptable":0,"unsafe":0,"review":8,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":8,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.32440756488388023,"confidence_level":0.95}}},"generalize":{"n":8,"raw":{"n":8,"acceptable":0,"unsafe":8,"review":0,"automated_count":8,"automated_acceptable":0,"automated_unsafe":8,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":1.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":1.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.32440756488388023,"confidence_level":0.95}},"effective":{"n":8,"acceptable":0,"unsafe":0,"review":8,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":8,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.32440756488388023,"confidence_level":0.95}}},"review":{"n":2,"raw":{"n":2,"acceptable":0,"unsafe":2,"review":0,"automated_count":2,"automated_acceptable":0,"automated_unsafe":2,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":1.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":1.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.6576197724933469,"confidence_level":0.95}},"effective":{"n":2,"acceptable":2,"unsafe":0,"review":2,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":1.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.34238022750665303,"upper":1.0,"confidence_level":0.95}}}}}},"composite":{"n":198,"scope":"descriptive sum across four correlated panels","raw":{"n":198,"acceptable":50,"unsafe":148,"review":0,"automated_count":198,"automated_acceptable":50,"automated_unsafe":148,"unnecessary_reviews":0,"accuracy":0.25252525252525254,"unsafe_rate":0.7474747474747475,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.25252525252525254,"conditional_unsafe":0.7474747474747475,"unnecessary_review_rate":0.0},"effective":{"n":198,"acceptable":58,"unsafe":6,"review":141,"automated_count":57,"automated_acceptable":44,"automated_unsafe":6,"unnecessary_reviews":127,"accuracy":0.29292929292929293,"unsafe_rate":0.030303030303030304,"review_rate":0.7121212121212122,"automated_coverage":0.2878787878787879,"conditional_correctness":0.7719298245614035,"conditional_unsafe":0.10526315789473684,"unnecessary_review_rate":0.6414141414141414}}},{"id":"HuggingFaceTB--SmolLM3-3B","platform":"Local","model_id":"HuggingFaceTB/SmolLM3-3B","revision":"a07cc9a04f16550a088caea529712d1d335b0ac1","loading":"bfloat16 / none","scoring_kind":"Adapted label scores","settings":{"dtype":"bfloat16","quantization":null,"max_tokens":4096,"effective_max_tokens":4096,"temperature":1.0,"trust_remote_code":false,"local_files_only":true,"use_safetensors":true,"guard_threshold":0.45,"seed":0,"tf32":false},"scoring_metadata":{"backend":"causal","protocol":"hf-local-sweep-backends-v1","probability_source":"adapted_conditional_label_softmax_not_calibrated_sixway","latency_scope":"synchronous_forward_scoring_including_transfer_excluding_tokenization","precision":"bfloat16","equal_mass_convention":"rank bins with floor boundaries at floor(i*n/k), k=min(10,n//2); minimum two rows"},"load_ms":5407.250942022074,"peak_allocated_bytes":6337086464,"peak_reserved_bytes":6343884800,"panels":{"diagnostic43-no_samples":{"n":43,"raw":{"n":43,"acceptable":13,"unsafe":2,"review":0,"automated_count":43,"automated_acceptable":13,"automated_unsafe":2,"unnecessary_reviews":0,"accuracy":0.3023255813953488,"unsafe_rate":0.046511627906976744,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.3023255813953488,"conditional_unsafe":0.046511627906976744,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.1860200013584818,"upper":0.45105364177729607,"confidence_level":0.95}},"effective":{"n":43,"acceptable":12,"unsafe":2,"review":10,"automated_count":33,"automated_acceptable":11,"automated_unsafe":2,"unnecessary_reviews":9,"accuracy":0.27906976744186046,"unsafe_rate":0.046511627906976744,"review_rate":0.23255813953488372,"automated_coverage":0.7674418604651163,"conditional_correctness":0.3333333333333333,"conditional_unsafe":0.06060606060606061,"unnecessary_review_rate":0.20930232558139536,"accuracy_wilson_95":{"lower":0.16746613786237488,"upper":0.42691028681878873,"confidence_level":0.95}},"confusion":{"raw":{"keep":{"keep":0,"drop":0,"mask":10,"pseudonymize":0,"generalize":2,"review":0},"drop":{"keep":0,"drop":0,"mask":13,"pseudonymize":0,"generalize":3,"review":0},"mask":{"keep":0,"drop":0,"mask":2,"pseudonymize":0,"generalize":0,"review":0},"pseudonymize":{"keep":0,"drop":0,"mask":1,"pseudonymize":1,"generalize":0,"review":0},"generalize":{"keep":0,"drop":0,"mask":1,"pseudonymize":0,"generalize":5,"review":0},"review":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0}},"effective":{"keep":{"keep":0,"drop":0,"mask":8,"pseudonymize":0,"generalize":0,"review":4},"drop":{"keep":0,"drop":0,"mask":10,"pseudonymize":0,"generalize":2,"review":4},"mask":{"keep":0,"drop":0,"mask":2,"pseudonymize":0,"generalize":0,"review":0},"pseudonymize":{"keep":0,"drop":0,"mask":1,"pseudonymize":0,"generalize":0,"review":1},"generalize":{"keep":0,"drop":0,"mask":1,"pseudonymize":0,"generalize":5,"review":0},"review":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0}}},"probability_metrics":{"singleton_n":38,"multiacceptable_n":5,"multiacceptable_coverage":0.11627906976744186,"nll":2.9164675820135404,"nll_zero_mass_n":0,"brier":1.2055875560152698,"acceptable_set_n":43,"acceptable_set_nll":2.5999169839191967,"acceptable_set_zero_mass_n":0},"calibration_all_labels":{"equal_width":{"n":43,"bin_count":10,"ece":0.4062150820743206,"signed_mean_gap":0.4062150820743206,"absolute_mean_gap":0.4062150820743206,"bins":[{"index":0,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.0,"upper":0.1,"upper_inclusive":false},{"index":1,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.1,"upper":0.2,"upper_inclusive":false},{"index":2,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.2,"upper":0.3,"upper_inclusive":false},{"index":3,"count":4,"confidence":0.36154431104660034,"empirical_correctness":0.25,"signed_gap":0.11154431104660034,"absolute_gap":0.11154431104660034,"lower":0.3,"upper":0.4,"upper_inclusive":false},{"index":4,"count":8,"confidence":0.45990683510899544,"empirical_correctness":0.125,"signed_gap":0.33490683510899544,"absolute_gap":0.33490683510899544,"lower":0.4,"upper":0.5,"upper_inclusive":false},{"index":5,"count":5,"confidence":0.5504296541213989,"empirical_correctness":0.4,"signed_gap":0.15042965412139886,"absolute_gap":0.15042965412139886,"lower":0.5,"upper":0.6,"upper_inclusive":false},{"index":6,"count":2,"confidence":0.6841808557510376,"empirical_correctness":0.5,"signed_gap":0.1841808557510376,"absolute_gap":0.1841808557510376,"lower":0.6,"upper":0.7,"upper_inclusive":false},{"index":7,"count":7,"confidence":0.7519115124429975,"empirical_correctness":0.42857142857142855,"signed_gap":0.32334008387156893,"absolute_gap":0.32334008387156893,"lower":0.7,"upper":0.8,"upper_inclusive":false},{"index":8,"count":5,"confidence":0.8804956555366517,"empirical_correctness":0.4,"signed_gap":0.48049565553665163,"absolute_gap":0.48049565553665163,"lower":0.8,"upper":0.9,"upper_inclusive":false},{"index":9,"count":12,"confidence":0.9629539797703425,"empirical_correctness":0.25,"signed_gap":0.7129539797703425,"absolute_gap":0.7129539797703425,"lower":0.9,"upper":1.0,"upper_inclusive":true}]},"equal_mass":{"n":43,"bin_count":10,"ece":0.4062150820743206,"signed_mean_gap":0.4062150820743206,"absolute_mean_gap":0.4062150820743206,"bins":[{"index":0,"count":4,"confidence":0.36154431104660034,"empirical_correctness":0.25,"signed_gap":0.11154431104660034,"absolute_gap":0.11154431104660034,"min_confidence":0.32771819829940796,"max_confidence":0.3892543911933899},{"index":1,"count":4,"confidence":0.4323693960905075,"empirical_correctness":0.25,"signed_gap":0.1823693960905075,"absolute_gap":0.1823693960905075,"min_confidence":0.40759581327438354,"max_confidence":0.46479669213294983},{"index":2,"count":4,"confidence":0.48744427412748337,"empirical_correctness":0.0,"signed_gap":0.48744427412748337,"absolute_gap":0.48744427412748337,"min_confidence":0.4816037118434906,"max_confidence":0.4959159195423126},{"index":3,"count":5,"confidence":0.5504296541213989,"empirical_correctness":0.4,"signed_gap":0.15042965412139886,"absolute_gap":0.15042965412139886,"min_confidence":0.5120512247085571,"max_confidence":0.5954848527908325},{"index":4,"count":4,"confidence":0.7021305412054062,"empirical_correctness":0.5,"signed_gap":0.2021305412054062,"absolute_gap":0.2021305412054062,"min_confidence":0.6690998673439026,"max_confidence":0.7288995981216431},{"index":5,"count":4,"confidence":0.7560245543718338,"empirical_correctness":0.5,"signed_gap":0.2560245543718338,"absolute_gap":0.2560245543718338,"min_confidence":0.7349201440811157,"max_confidence":0.7882097959518433},{"index":6,"count":5,"confidence":0.8608198523521423,"empirical_correctness":0.2,"signed_gap":0.6608198523521422,"absolute_gap":0.6608198523521422,"min_confidence":0.7991219162940979,"max_confidence":0.8969732522964478},{"index":7,"count":4,"confidence":0.9212683439254761,"empirical_correctness":0.25,"signed_gap":0.6712683439254761,"absolute_gap":0.6712683439254761,"min_confidence":0.8975009322166443,"max_confidence":0.9394747018814087},{"index":8,"count":4,"confidence":0.9668108224868774,"empirical_correctness":0.5,"signed_gap":0.46681082248687744,"absolute_gap":0.46681082248687744,"min_confidence":0.9514902234077454,"max_confidence":0.9757322072982788},{"index":9,"count":5,"confidence":0.9801264047622681,"empirical_correctness":0.2,"signed_gap":0.780126404762268,"absolute_gap":0.780126404762268,"min_confidence":0.9766650199890137,"max_confidence":0.9898741245269775}]}},"calibration_singleton_labels":{"equal_width":{"n":38,"bin_count":10,"ece":0.49550130884898336,"signed_mean_gap":0.49550130884898336,"absolute_mean_gap":0.49550130884898336,"bins":[{"index":0,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.0,"upper":0.1,"upper_inclusive":false},{"index":1,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.1,"upper":0.2,"upper_inclusive":false},{"index":2,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.2,"upper":0.3,"upper_inclusive":false},{"index":3,"count":4,"confidence":0.36154431104660034,"empirical_correctness":0.25,"signed_gap":0.11154431104660034,"absolute_gap":0.11154431104660034,"lower":0.3,"upper":0.4,"upper_inclusive":false},{"index":4,"count":7,"confidence":0.46179744601249695,"empirical_correctness":0.0,"signed_gap":0.46179744601249695,"absolute_gap":0.46179744601249695,"lower":0.4,"upper":0.5,"upper_inclusive":false},{"index":5,"count":4,"confidence":0.5588907599449158,"empirical_correctness":0.25,"signed_gap":0.30889075994491577,"absolute_gap":0.30889075994491577,"lower":0.5,"upper":0.6,"upper_inclusive":false},{"index":6,"count":2,"confidence":0.6841808557510376,"empirical_correctness":0.5,"signed_gap":0.1841808557510376,"absolute_gap":0.1841808557510376,"lower":0.6,"upper":0.7,"upper_inclusive":false},{"index":7,"count":6,"confidence":0.7547434071699778,"empirical_correctness":0.3333333333333333,"signed_gap":0.42141007383664447,"absolute_gap":0.42141007383664447,"lower":0.7,"upper":0.8,"upper_inclusive":false},{"index":8,"count":5,"confidence":0.8804956555366517,"empirical_correctness":0.4,"signed_gap":0.48049565553665163,"absolute_gap":0.48049565553665163,"lower":0.8,"upper":0.9,"upper_inclusive":false},{"index":9,"count":10,"confidence":0.9615426898002625,"empirical_correctness":0.1,"signed_gap":0.8615426898002625,"absolute_gap":0.8615426898002625,"lower":0.9,"upper":1.0,"upper_inclusive":true}]},"equal_mass":{"n":38,"bin_count":10,"ece":0.49550130884898336,"signed_mean_gap":0.49550130884898336,"absolute_mean_gap":0.49550130884898336,"bins":[{"index":0,"count":3,"confidence":0.35230761766433716,"empirical_correctness":0.0,"signed_gap":0.35230761766433716,"absolute_gap":0.35230761766433716,"min_confidence":0.32771819829940796,"max_confidence":0.36808645725250244},{"index":1,"count":4,"confidence":0.41801485419273376,"empirical_correctness":0.25,"signed_gap":0.16801485419273376,"absolute_gap":0.16801485419273376,"min_confidence":0.3892543911933899,"max_confidence":0.46479669213294983},{"index":2,"count":4,"confidence":0.48744427412748337,"empirical_correctness":0.0,"signed_gap":0.48744427412748337,"absolute_gap":0.48744427412748337,"min_confidence":0.4816037118434906,"max_confidence":0.4959159195423126},{"index":3,"count":4,"confidence":0.5588907599449158,"empirical_correctness":0.25,"signed_gap":0.30889075994491577,"absolute_gap":0.30889075994491577,"min_confidence":0.5120512247085571,"max_confidence":0.5954848527908325},{"index":4,"count":4,"confidence":0.7021305412054062,"empirical_correctness":0.5,"signed_gap":0.2021305412054062,"absolute_gap":0.2021305412054062,"min_confidence":0.6690998673439026,"max_confidence":0.7288995981216431},{"index":5,"count":3,"confidence":0.7630593578020731,"empirical_correctness":0.3333333333333333,"signed_gap":0.4297260244687398,"absolute_gap":0.4297260244687398,"min_confidence":0.7385192513465881,"max_confidence":0.7882097959518433},{"index":6,"count":4,"confidence":0.851781502366066,"empirical_correctness":0.25,"signed_gap":0.601781502366066,"absolute_gap":0.601781502366066,"min_confidence":0.7991219162940979,"max_confidence":0.89307701587677},{"index":7,"count":4,"confidence":0.9106429815292358,"empirical_correctness":0.25,"signed_gap":0.6606429815292358,"absolute_gap":0.6606429815292358,"min_confidence":0.8969732522964478,"max_confidence":0.9248727560043335},{"index":8,"count":4,"confidence":0.9608405381441116,"empirical_correctness":0.0,"signed_gap":0.9608405381441116,"absolute_gap":0.9608405381441116,"min_confidence":0.9394747018814087,"max_confidence":0.9766650199890137},{"index":9,"count":4,"confidence":0.9809917509555817,"empirical_correctness":0.25,"signed_gap":0.7309917509555817,"absolute_gap":0.7309917509555817,"min_confidence":0.9774218797683716,"max_confidence":0.9898741245269775}]}},"latency_ms":{"n":43,"median":57.35986499348655,"p95":62.13665133691393},"per_action":{"keep":{"n":12,"raw":{"n":12,"acceptable":0,"unsafe":0,"review":0,"automated_count":12,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":2.7755575615628914e-17,"upper":0.24249400665524085,"confidence_level":0.95}},"effective":{"n":12,"acceptable":0,"unsafe":0,"review":4,"automated_count":8,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":4,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":0.3333333333333333,"automated_coverage":0.6666666666666666,"conditional_correctness":0.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.3333333333333333,"accuracy_wilson_95":{"lower":2.7755575615628914e-17,"upper":0.24249400665524085,"confidence_level":0.95}}},"drop":{"n":16,"raw":{"n":16,"acceptable":0,"unsafe":2,"review":0,"automated_count":16,"automated_acceptable":0,"automated_unsafe":2,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":0.125,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":0.125,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.1936076805344365,"confidence_level":0.95}},"effective":{"n":16,"acceptable":0,"unsafe":2,"review":4,"automated_count":12,"automated_acceptable":0,"automated_unsafe":2,"unnecessary_reviews":4,"accuracy":0.0,"unsafe_rate":0.125,"review_rate":0.25,"automated_coverage":0.75,"conditional_correctness":0.0,"conditional_unsafe":0.16666666666666666,"unnecessary_review_rate":0.25,"accuracy_wilson_95":{"lower":0.0,"upper":0.1936076805344365,"confidence_level":0.95}}},"mask":{"n":2,"raw":{"n":2,"acceptable":2,"unsafe":0,"review":0,"automated_count":2,"automated_acceptable":2,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":1.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.34238022750665303,"upper":1.0,"confidence_level":0.95}},"effective":{"n":2,"acceptable":2,"unsafe":0,"review":0,"automated_count":2,"automated_acceptable":2,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":1.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.34238022750665303,"upper":1.0,"confidence_level":0.95}}},"pseudonymize":{"n":2,"raw":{"n":2,"acceptable":1,"unsafe":0,"review":0,"automated_count":2,"automated_acceptable":1,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":0.5,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.5,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.09453120573423074,"upper":0.9054687942657693,"confidence_level":0.95}},"effective":{"n":2,"acceptable":0,"unsafe":0,"review":1,"automated_count":1,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":1,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":0.5,"automated_coverage":0.5,"conditional_correctness":0.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.5,"accuracy_wilson_95":{"lower":0.0,"upper":0.6576197724933469,"confidence_level":0.95}}},"generalize":{"n":6,"raw":{"n":6,"acceptable":5,"unsafe":0,"review":0,"automated_count":6,"automated_acceptable":5,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":0.8333333333333334,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.8333333333333334,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.43649717781352987,"upper":0.9699466302516935,"confidence_level":0.95}},"effective":{"n":6,"acceptable":5,"unsafe":0,"review":0,"automated_count":6,"automated_acceptable":5,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":0.8333333333333334,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.8333333333333334,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.43649717781352987,"upper":0.9699466302516935,"confidence_level":0.95}}},"review":{"n":0,"raw":{"n":0,"acceptable":0,"unsafe":0,"review":0,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":null,"unsafe_rate":null,"review_rate":null,"automated_coverage":null,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":null},"effective":{"n":0,"acceptable":0,"unsafe":0,"review":0,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":null,"unsafe_rate":null,"review_rate":null,"automated_coverage":null,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":null}}}},"diagnostic43-compact_no_samples":{"n":43,"raw":{"n":43,"acceptable":12,"unsafe":9,"review":0,"automated_count":43,"automated_acceptable":12,"automated_unsafe":9,"unnecessary_reviews":0,"accuracy":0.27906976744186046,"unsafe_rate":0.20930232558139536,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.27906976744186046,"conditional_unsafe":0.20930232558139536,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.16746613786237488,"upper":0.42691028681878873,"confidence_level":0.95}},"effective":{"n":43,"acceptable":13,"unsafe":7,"review":15,"automated_count":28,"automated_acceptable":11,"automated_unsafe":7,"unnecessary_reviews":13,"accuracy":0.3023255813953488,"unsafe_rate":0.16279069767441862,"review_rate":0.3488372093023256,"automated_coverage":0.6511627906976745,"conditional_correctness":0.39285714285714285,"conditional_unsafe":0.25,"unnecessary_review_rate":0.3023255813953488,"accuracy_wilson_95":{"lower":0.1860200013584818,"upper":0.45105364177729607,"confidence_level":0.95}},"confusion":{"raw":{"keep":{"keep":0,"drop":0,"mask":6,"pseudonymize":3,"generalize":3,"review":0},"drop":{"keep":0,"drop":0,"mask":4,"pseudonymize":5,"generalize":7,"review":0},"mask":{"keep":0,"drop":0,"mask":2,"pseudonymize":0,"generalize":0,"review":0},"pseudonymize":{"keep":0,"drop":0,"mask":0,"pseudonymize":2,"generalize":0,"review":0},"generalize":{"keep":0,"drop":0,"mask":0,"pseudonymize":2,"generalize":4,"review":0},"review":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0}},"effective":{"keep":{"keep":0,"drop":0,"mask":6,"pseudonymize":0,"generalize":0,"review":6},"drop":{"keep":0,"drop":0,"mask":4,"pseudonymize":4,"generalize":3,"review":5},"mask":{"keep":0,"drop":0,"mask":2,"pseudonymize":0,"generalize":0,"review":0},"pseudonymize":{"keep":0,"drop":0,"mask":0,"pseudonymize":2,"generalize":0,"review":0},"generalize":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":4,"review":2},"review":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0}}},"probability_metrics":{"singleton_n":38,"multiacceptable_n":5,"multiacceptable_coverage":0.11627906976744186,"nll":2.7101495805392575,"nll_zero_mass_n":0,"brier":1.0827565897189575,"acceptable_set_n":43,"acceptable_set_nll":2.4301190226454876,"acceptable_set_zero_mass_n":0},"calibration_all_labels":{"equal_width":{"n":43,"bin_count":10,"ece":0.33812197557715484,"signed_mean_gap":0.3319642502208089,"absolute_mean_gap":0.3319642502208089,"bins":[{"index":0,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.0,"upper":0.1,"upper_inclusive":false},{"index":1,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.1,"upper":0.2,"upper_inclusive":false},{"index":2,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.2,"upper":0.3,"upper_inclusive":false},{"index":3,"count":6,"confidence":0.33324502408504486,"empirical_correctness":0.16666666666666666,"signed_gap":0.1665783574183782,"absolute_gap":0.1665783574183782,"lower":0.3,"upper":0.4,"upper_inclusive":false},{"index":4,"count":7,"confidence":0.4657420814037323,"empirical_correctness":0.14285714285714285,"signed_gap":0.32288493854658945,"absolute_gap":0.32288493854658945,"lower":0.4,"upper":0.5,"upper_inclusive":false},{"index":5,"count":9,"confidence":0.5279390017191569,"empirical_correctness":0.1111111111111111,"signed_gap":0.41682789060804576,"absolute_gap":0.41682789060804576,"lower":0.5,"upper":0.6,"upper_inclusive":false},{"index":6,"count":6,"confidence":0.6472391784191132,"empirical_correctness":0.16666666666666666,"signed_gap":0.48057251175244653,"absolute_gap":0.48057251175244653,"lower":0.6,"upper":0.7,"upper_inclusive":false},{"index":7,"count":5,"confidence":0.728345787525177,"empirical_correctness":0.6,"signed_gap":0.128345787525177,"absolute_gap":0.128345787525177,"lower":0.7,"upper":0.8,"upper_inclusive":false},{"index":8,"count":8,"confidence":0.858821764588356,"empirical_correctness":0.375,"signed_gap":0.483821764588356,"absolute_gap":0.483821764588356,"lower":0.8,"upper":0.9,"upper_inclusive":false},{"index":9,"count":2,"confidence":0.933804452419281,"empirical_correctness":1.0,"signed_gap":-0.066195547580719,"absolute_gap":0.066195547580719,"lower":0.9,"upper":1.0,"upper_inclusive":true}]},"equal_mass":{"n":43,"bin_count":10,"ece":0.3319642502208089,"signed_mean_gap":0.3319642502208089,"absolute_mean_gap":0.3319642502208089,"bins":[{"index":0,"count":4,"confidence":0.32020990550518036,"empirical_correctness":0.25,"signed_gap":0.07020990550518036,"absolute_gap":0.07020990550518036,"min_confidence":0.30120936036109924,"max_confidence":0.35252171754837036},{"index":1,"count":4,"confidence":0.4009118676185608,"empirical_correctness":0.0,"signed_gap":0.4009118676185608,"absolute_gap":0.4009118676185608,"min_confidence":0.3539571166038513,"max_confidence":0.45731985569000244},{"index":2,"count":4,"confidence":0.4720168933272362,"empirical_correctness":0.25,"signed_gap":0.22201689332723618,"absolute_gap":0.22201689332723618,"min_confidence":0.46242445707321167,"max_confidence":0.4773615598678589},{"index":3,"count":5,"confidence":0.5062398374080658,"empirical_correctness":0.2,"signed_gap":0.30623983740806576,"absolute_gap":0.30623983740806576,"min_confidence":0.48711004853248596,"max_confidence":0.5285133123397827},{"index":4,"count":4,"confidence":0.540435403585434,"empirical_correctness":0.0,"signed_gap":0.540435403585434,"absolute_gap":0.540435403585434,"min_confidence":0.5349960327148438,"max_confidence":0.5427147746086121},{"index":5,"count":4,"confidence":0.6056452244520187,"empirical_correctness":0.25,"signed_gap":0.35564522445201874,"absolute_gap":0.35564522445201874,"min_confidence":0.5456202626228333,"max_confidence":0.6385371685028076},{"index":6,"count":5,"confidence":0.6848050355911255,"empirical_correctness":0.2,"signed_gap":0.4848050355911255,"absolute_gap":0.4848050355911255,"min_confidence":0.6593800783157349,"max_confidence":0.7103449106216431},{"index":7,"count":4,"confidence":0.7585023194551468,"empirical_correctness":0.75,"signed_gap":0.00850231945514679,"absolute_gap":0.00850231945514679,"min_confidence":0.7130677700042725,"max_confidence":0.8098310828208923},{"index":8,"count":4,"confidence":0.8552103042602539,"empirical_correctness":0.5,"signed_gap":0.3552103042602539,"absolute_gap":0.3552103042602539,"min_confidence":0.8244866728782654,"max_confidence":0.8731106519699097},{"index":9,"count":5,"confidence":0.9015021443367004,"empirical_correctness":0.4,"signed_gap":0.5015021443367004,"absolute_gap":0.5015021443367004,"min_confidence":0.8732123374938965,"max_confidence":0.9411871433258057}]}},"calibration_singleton_labels":{"equal_width":{"n":38,"bin_count":10,"ece":0.40296402808867005,"signed_mean_gap":0.3998686145795019,"absolute_mean_gap":0.3998686145795019,"bins":[{"index":0,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.0,"upper":0.1,"upper_inclusive":false},{"index":1,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.1,"upper":0.2,"upper_inclusive":false},{"index":2,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.2,"upper":0.3,"upper_inclusive":false},{"index":3,"count":4,"confidence":0.3364347666501999,"empirical_correctness":0.0,"signed_gap":0.3364347666501999,"absolute_gap":0.3364347666501999,"lower":0.3,"upper":0.4,"upper_inclusive":false},{"index":4,"count":7,"confidence":0.4657420814037323,"empirical_correctness":0.14285714285714285,"signed_gap":0.32288493854658945,"absolute_gap":0.32288493854658945,"lower":0.4,"upper":0.5,"upper_inclusive":false},{"index":5,"count":9,"confidence":0.5279390017191569,"empirical_correctness":0.1111111111111111,"signed_gap":0.41682789060804576,"absolute_gap":0.41682789060804576,"lower":0.5,"upper":0.6,"upper_inclusive":false},{"index":6,"count":5,"confidence":0.6489795804023742,"empirical_correctness":0.0,"signed_gap":0.6489795804023742,"absolute_gap":0.6489795804023742,"lower":0.6,"upper":0.7,"upper_inclusive":false},{"index":7,"count":5,"confidence":0.728345787525177,"empirical_correctness":0.6,"signed_gap":0.128345787525177,"absolute_gap":0.128345787525177,"lower":0.7,"upper":0.8,"upper_inclusive":false},{"index":8,"count":7,"confidence":0.8585441027368818,"empirical_correctness":0.2857142857142857,"signed_gap":0.572829817022596,"absolute_gap":0.572829817022596,"lower":0.8,"upper":0.9,"upper_inclusive":false},{"index":9,"count":1,"confidence":0.9411871433258057,"empirical_correctness":1.0,"signed_gap":-0.058812856674194336,"absolute_gap":0.058812856674194336,"lower":0.9,"upper":1.0,"upper_inclusive":true}]},"equal_mass":{"n":38,"bin_count":10,"ece":0.3998686145795019,"signed_mean_gap":0.3998686145795019,"absolute_mean_gap":0.3998686145795019,"bins":[{"index":0,"count":3,"confidence":0.32702188690503436,"empirical_correctness":0.0,"signed_gap":0.32702188690503436,"absolute_gap":0.32702188690503436,"min_confidence":0.3052430748939514,"max_confidence":0.3539571166038513},{"index":1,"count":4,"confidence":0.4280287027359009,"empirical_correctness":0.25,"signed_gap":0.17802870273590088,"absolute_gap":0.17802870273590088,"min_confidence":0.3646734058856964,"max_confidence":0.46242445707321167},{"index":2,"count":4,"confidence":0.47818829119205475,"empirical_correctness":0.0,"signed_gap":0.47818829119205475,"absolute_gap":0.47818829119205475,"min_confidence":0.4714592397212982,"max_confidence":0.48711004853248596},{"index":3,"count":4,"confidence":0.5110222846269608,"empirical_correctness":0.25,"signed_gap":0.26102228462696075,"absolute_gap":0.26102228462696075,"min_confidence":0.5035869479179382,"max_confidence":0.5285133123397827},{"index":4,"count":4,"confidence":0.540435403585434,"empirical_correctness":0.0,"signed_gap":0.540435403585434,"absolute_gap":0.540435403585434,"min_confidence":0.5349960327148438,"max_confidence":0.5427147746086121},{"index":5,"count":3,"confidence":0.5946812431017557,"empirical_correctness":0.0,"signed_gap":0.5946812431017557,"absolute_gap":0.5946812431017557,"min_confidence":0.5456202626228333,"max_confidence":0.633108913898468},{"index":6,"count":4,"confidence":0.6784200668334961,"empirical_correctness":0.25,"signed_gap":0.4284200668334961,"absolute_gap":0.4284200668334961,"min_confidence":0.6593800783157349,"max_confidence":0.7072058320045471},{"index":7,"count":4,"confidence":0.7336307764053345,"empirical_correctness":0.5,"signed_gap":0.23363077640533447,"absolute_gap":0.23363077640533447,"min_confidence":0.7103449106216431,"max_confidence":0.7622186541557312},{"index":8,"count":4,"confidence":0.8424767255783081,"empirical_correctness":0.5,"signed_gap":0.3424767255783081,"absolute_gap":0.3424767255783081,"min_confidence":0.8098310828208923,"max_confidence":0.8731106519699097},{"index":9,"count":4,"confidence":0.8952722400426865,"empirical_correctness":0.25,"signed_gap":0.6452722400426865,"absolute_gap":0.6452722400426865,"min_confidence":0.8732123374938965,"max_confidence":0.9411871433258057}]}},"latency_ms":{"n":43,"median":59.48060000082478,"p95":67.08913546754047},"per_action":{"keep":{"n":12,"raw":{"n":12,"acceptable":0,"unsafe":0,"review":0,"automated_count":12,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":2.7755575615628914e-17,"upper":0.24249400665524085,"confidence_level":0.95}},"effective":{"n":12,"acceptable":0,"unsafe":0,"review":6,"automated_count":6,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":6,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":0.5,"automated_coverage":0.5,"conditional_correctness":0.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.5,"accuracy_wilson_95":{"lower":2.7755575615628914e-17,"upper":0.24249400665524085,"confidence_level":0.95}}},"drop":{"n":16,"raw":{"n":16,"acceptable":0,"unsafe":8,"review":0,"automated_count":16,"automated_acceptable":0,"automated_unsafe":8,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":0.5,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":0.5,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.1936076805344365,"confidence_level":0.95}},"effective":{"n":16,"acceptable":0,"unsafe":7,"review":5,"automated_count":11,"automated_acceptable":0,"automated_unsafe":7,"unnecessary_reviews":5,"accuracy":0.0,"unsafe_rate":0.4375,"review_rate":0.3125,"automated_coverage":0.6875,"conditional_correctness":0.0,"conditional_unsafe":0.6363636363636364,"unnecessary_review_rate":0.3125,"accuracy_wilson_95":{"lower":0.0,"upper":0.1936076805344365,"confidence_level":0.95}}},"mask":{"n":2,"raw":{"n":2,"acceptable":2,"unsafe":0,"review":0,"automated_count":2,"automated_acceptable":2,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":1.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.34238022750665303,"upper":1.0,"confidence_level":0.95}},"effective":{"n":2,"acceptable":2,"unsafe":0,"review":0,"automated_count":2,"automated_acceptable":2,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":1.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.34238022750665303,"upper":1.0,"confidence_level":0.95}}},"pseudonymize":{"n":2,"raw":{"n":2,"acceptable":2,"unsafe":0,"review":0,"automated_count":2,"automated_acceptable":2,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":1.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.34238022750665303,"upper":1.0,"confidence_level":0.95}},"effective":{"n":2,"acceptable":2,"unsafe":0,"review":0,"automated_count":2,"automated_acceptable":2,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":1.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.34238022750665303,"upper":1.0,"confidence_level":0.95}}},"generalize":{"n":6,"raw":{"n":6,"acceptable":4,"unsafe":0,"review":0,"automated_count":6,"automated_acceptable":4,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":0.6666666666666666,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.6666666666666666,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.299993315138392,"upper":0.9032285888942195,"confidence_level":0.95}},"effective":{"n":6,"acceptable":4,"unsafe":0,"review":2,"automated_count":4,"automated_acceptable":4,"automated_unsafe":0,"unnecessary_reviews":2,"accuracy":0.6666666666666666,"unsafe_rate":0.0,"review_rate":0.3333333333333333,"automated_coverage":0.6666666666666666,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.3333333333333333,"accuracy_wilson_95":{"lower":0.299993315138392,"upper":0.9032285888942195,"confidence_level":0.95}}},"review":{"n":0,"raw":{"n":0,"acceptable":0,"unsafe":0,"review":0,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":null,"unsafe_rate":null,"review_rate":null,"automated_coverage":null,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":null},"effective":{"n":0,"acceptable":0,"unsafe":0,"review":0,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":null,"unsafe_rate":null,"review_rate":null,"automated_coverage":null,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":null}}}},"challenge56-no_samples":{"n":56,"raw":{"n":56,"acceptable":22,"unsafe":1,"review":0,"automated_count":56,"automated_acceptable":22,"automated_unsafe":1,"unnecessary_reviews":0,"accuracy":0.39285714285714285,"unsafe_rate":0.017857142857142856,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.39285714285714285,"conditional_unsafe":0.017857142857142856,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.2758041573011051,"upper":0.5236659719957729,"confidence_level":0.95}},"effective":{"n":56,"acceptable":15,"unsafe":0,"review":13,"automated_count":43,"automated_acceptable":15,"automated_unsafe":0,"unnecessary_reviews":13,"accuracy":0.26785714285714285,"unsafe_rate":0.0,"review_rate":0.23214285714285715,"automated_coverage":0.7678571428571429,"conditional_correctness":0.3488372093023256,"conditional_unsafe":0.0,"unnecessary_review_rate":0.23214285714285715,"accuracy_wilson_95":{"lower":0.16957305418501728,"upper":0.39594555929155156,"confidence_level":0.95}},"confusion":{"raw":{"keep":{"keep":0,"drop":0,"mask":6,"pseudonymize":2,"generalize":0,"review":0},"drop":{"keep":0,"drop":0,"mask":21,"pseudonymize":1,"generalize":0,"review":0},"mask":{"keep":0,"drop":0,"mask":8,"pseudonymize":0,"generalize":0,"review":0},"pseudonymize":{"keep":0,"drop":0,"mask":0,"pseudonymize":8,"generalize":0,"review":0},"generalize":{"keep":0,"drop":0,"mask":2,"pseudonymize":0,"generalize":6,"review":0},"review":{"keep":0,"drop":0,"mask":2,"pseudonymize":0,"generalize":0,"review":0}},"effective":{"keep":{"keep":0,"drop":0,"mask":4,"pseudonymize":0,"generalize":0,"review":4},"drop":{"keep":0,"drop":0,"mask":20,"pseudonymize":0,"generalize":0,"review":2},"mask":{"keep":0,"drop":0,"mask":8,"pseudonymize":0,"generalize":0,"review":0},"pseudonymize":{"keep":0,"drop":0,"mask":0,"pseudonymize":1,"generalize":0,"review":7},"generalize":{"keep":0,"drop":0,"mask":2,"pseudonymize":0,"generalize":6,"review":0},"review":{"keep":0,"drop":0,"mask":2,"pseudonymize":0,"generalize":0,"review":0}}},"probability_metrics":{"singleton_n":56,"multiacceptable_n":0,"multiacceptable_coverage":0.0,"nll":2.0637909603653273,"nll_zero_mass_n":0,"brier":0.9008504420934652,"acceptable_set_n":56,"acceptable_set_nll":2.0637909603653273,"acceptable_set_zero_mass_n":0},"calibration_all_labels":{"equal_width":{"n":56,"bin_count":10,"ece":0.3269246369600296,"signed_mean_gap":0.32160867750644684,"absolute_mean_gap":0.32160867750644684,"bins":[{"index":0,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.0,"upper":0.1,"upper_inclusive":false},{"index":1,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.1,"upper":0.2,"upper_inclusive":false},{"index":2,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.2,"upper":0.3,"upper_inclusive":false},{"index":3,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.3,"upper":0.4,"upper_inclusive":false},{"index":4,"count":7,"confidence":0.4583681992122105,"empirical_correctness":0.0,"signed_gap":0.4583681992122105,"absolute_gap":0.4583681992122105,"lower":0.4,"upper":0.5,"upper_inclusive":false},{"index":5,"count":7,"confidence":0.5630665932382856,"empirical_correctness":0.14285714285714285,"signed_gap":0.4202094503811427,"absolute_gap":0.4202094503811427,"lower":0.5,"upper":0.6,"upper_inclusive":false},{"index":6,"count":14,"confidence":0.64940693123,"empirical_correctness":0.35714285714285715,"signed_gap":0.2922640740871429,"absolute_gap":0.2922640740871429,"lower":0.6,"upper":0.7,"upper_inclusive":false},{"index":7,"count":13,"confidence":0.7643188788340642,"empirical_correctness":0.38461538461538464,"signed_gap":0.3797034942186796,"absolute_gap":0.3797034942186796,"lower":0.7,"upper":0.8,"upper_inclusive":false},{"index":8,"count":6,"confidence":0.830174465974172,"empirical_correctness":0.3333333333333333,"signed_gap":0.4968411326408387,"absolute_gap":0.4968411326408387,"lower":0.8,"upper":0.9,"upper_inclusive":false},{"index":9,"count":9,"confidence":0.9834614594777426,"empirical_correctness":1.0,"signed_gap":-0.01653854052225745,"absolute_gap":0.01653854052225745,"lower":0.9,"upper":1.0,"upper_inclusive":true}]},"equal_mass":{"n":56,"bin_count":10,"ece":0.32380312255450655,"signed_mean_gap":0.32160867750644684,"absolute_mean_gap":0.32160867750644684,"bins":[{"index":0,"count":5,"confidence":0.45039613246917726,"empirical_correctness":0.0,"signed_gap":0.45039613246917726,"absolute_gap":0.45039613246917726,"min_confidence":0.4400913715362549,"max_confidence":0.460683137178421},{"index":1,"count":6,"confidence":0.5216878652572632,"empirical_correctness":0.0,"signed_gap":0.5216878652572632,"absolute_gap":0.5216878652572632,"min_confidence":0.4624859392642975,"max_confidence":0.570781409740448},{"index":2,"count":5,"confidence":0.5961017370223999,"empirical_correctness":0.6,"signed_gap":-0.0038982629776000977,"absolute_gap":0.0038982629776000977,"min_confidence":0.585955023765564,"max_confidence":0.6104851365089417},{"index":3,"count":6,"confidence":0.6410484115282694,"empirical_correctness":0.3333333333333333,"signed_gap":0.3077150781949361,"absolute_gap":0.3077150781949361,"min_confidence":0.6228468418121338,"max_confidence":0.6661087870597839},{"index":4,"count":6,"confidence":0.6721389293670654,"empirical_correctness":0.16666666666666666,"signed_gap":0.5054722627003988,"absolute_gap":0.5054722627003988,"min_confidence":0.6664859652519226,"max_confidence":0.6855331063270569},{"index":5,"count":5,"confidence":0.7422430634498596,"empirical_correctness":0.4,"signed_gap":0.34224306344985955,"absolute_gap":0.34224306344985955,"min_confidence":0.7176532745361328,"max_confidence":0.7565383911132812},{"index":6,"count":6,"confidence":0.7727554738521576,"empirical_correctness":0.3333333333333333,"signed_gap":0.4394221405188243,"absolute_gap":0.4394221405188243,"min_confidence":0.7683181166648865,"max_confidence":0.783697783946991},{"index":7,"count":5,"confidence":0.800442099571228,"empirical_correctness":0.6,"signed_gap":0.20044209957122805,"absolute_gap":0.20044209957122805,"min_confidence":0.7917866110801697,"max_confidence":0.8069685697555542},{"index":8,"count":6,"confidence":0.9100566407044729,"empirical_correctness":0.5,"signed_gap":0.4100566407044729,"absolute_gap":0.4100566407044729,"min_confidence":0.8307182788848877,"max_confidence":0.9878361821174622},{"index":9,"count":6,"confidence":0.9930078089237213,"empirical_correctness":1.0,"signed_gap":-0.0069921910762786865,"absolute_gap":0.0069921910762786865,"min_confidence":0.9881452322006226,"max_confidence":0.9958106279373169}]}},"calibration_singleton_labels":{"equal_width":{"n":56,"bin_count":10,"ece":0.3269246369600296,"signed_mean_gap":0.32160867750644684,"absolute_mean_gap":0.32160867750644684,"bins":[{"index":0,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.0,"upper":0.1,"upper_inclusive":false},{"index":1,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.1,"upper":0.2,"upper_inclusive":false},{"index":2,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.2,"upper":0.3,"upper_inclusive":false},{"index":3,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.3,"upper":0.4,"upper_inclusive":false},{"index":4,"count":7,"confidence":0.4583681992122105,"empirical_correctness":0.0,"signed_gap":0.4583681992122105,"absolute_gap":0.4583681992122105,"lower":0.4,"upper":0.5,"upper_inclusive":false},{"index":5,"count":7,"confidence":0.5630665932382856,"empirical_correctness":0.14285714285714285,"signed_gap":0.4202094503811427,"absolute_gap":0.4202094503811427,"lower":0.5,"upper":0.6,"upper_inclusive":false},{"index":6,"count":14,"confidence":0.64940693123,"empirical_correctness":0.35714285714285715,"signed_gap":0.2922640740871429,"absolute_gap":0.2922640740871429,"lower":0.6,"upper":0.7,"upper_inclusive":false},{"index":7,"count":13,"confidence":0.7643188788340642,"empirical_correctness":0.38461538461538464,"signed_gap":0.3797034942186796,"absolute_gap":0.3797034942186796,"lower":0.7,"upper":0.8,"upper_inclusive":false},{"index":8,"count":6,"confidence":0.830174465974172,"empirical_correctness":0.3333333333333333,"signed_gap":0.4968411326408387,"absolute_gap":0.4968411326408387,"lower":0.8,"upper":0.9,"upper_inclusive":false},{"index":9,"count":9,"confidence":0.9834614594777426,"empirical_correctness":1.0,"signed_gap":-0.01653854052225745,"absolute_gap":0.01653854052225745,"lower":0.9,"upper":1.0,"upper_inclusive":true}]},"equal_mass":{"n":56,"bin_count":10,"ece":0.32380312255450655,"signed_mean_gap":0.32160867750644684,"absolute_mean_gap":0.32160867750644684,"bins":[{"index":0,"count":5,"confidence":0.45039613246917726,"empirical_correctness":0.0,"signed_gap":0.45039613246917726,"absolute_gap":0.45039613246917726,"min_confidence":0.4400913715362549,"max_confidence":0.460683137178421},{"index":1,"count":6,"confidence":0.5216878652572632,"empirical_correctness":0.0,"signed_gap":0.5216878652572632,"absolute_gap":0.5216878652572632,"min_confidence":0.4624859392642975,"max_confidence":0.570781409740448},{"index":2,"count":5,"confidence":0.5961017370223999,"empirical_correctness":0.6,"signed_gap":-0.0038982629776000977,"absolute_gap":0.0038982629776000977,"min_confidence":0.585955023765564,"max_confidence":0.6104851365089417},{"index":3,"count":6,"confidence":0.6410484115282694,"empirical_correctness":0.3333333333333333,"signed_gap":0.3077150781949361,"absolute_gap":0.3077150781949361,"min_confidence":0.6228468418121338,"max_confidence":0.6661087870597839},{"index":4,"count":6,"confidence":0.6721389293670654,"empirical_correctness":0.16666666666666666,"signed_gap":0.5054722627003988,"absolute_gap":0.5054722627003988,"min_confidence":0.6664859652519226,"max_confidence":0.6855331063270569},{"index":5,"count":5,"confidence":0.7422430634498596,"empirical_correctness":0.4,"signed_gap":0.34224306344985955,"absolute_gap":0.34224306344985955,"min_confidence":0.7176532745361328,"max_confidence":0.7565383911132812},{"index":6,"count":6,"confidence":0.7727554738521576,"empirical_correctness":0.3333333333333333,"signed_gap":0.4394221405188243,"absolute_gap":0.4394221405188243,"min_confidence":0.7683181166648865,"max_confidence":0.783697783946991},{"index":7,"count":5,"confidence":0.800442099571228,"empirical_correctness":0.6,"signed_gap":0.20044209957122805,"absolute_gap":0.20044209957122805,"min_confidence":0.7917866110801697,"max_confidence":0.8069685697555542},{"index":8,"count":6,"confidence":0.9100566407044729,"empirical_correctness":0.5,"signed_gap":0.4100566407044729,"absolute_gap":0.4100566407044729,"min_confidence":0.8307182788848877,"max_confidence":0.9878361821174622},{"index":9,"count":6,"confidence":0.9930078089237213,"empirical_correctness":1.0,"signed_gap":-0.0069921910762786865,"absolute_gap":0.0069921910762786865,"min_confidence":0.9881452322006226,"max_confidence":0.9958106279373169}]}},"latency_ms":{"n":56,"median":46.50145448977128,"p95":49.608894769335166},"per_action":{"keep":{"n":8,"raw":{"n":8,"acceptable":0,"unsafe":0,"review":0,"automated_count":8,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.32440756488388023,"confidence_level":0.95}},"effective":{"n":8,"acceptable":0,"unsafe":0,"review":4,"automated_count":4,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":4,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":0.5,"automated_coverage":0.5,"conditional_correctness":0.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.5,"accuracy_wilson_95":{"lower":0.0,"upper":0.32440756488388023,"confidence_level":0.95}}},"drop":{"n":22,"raw":{"n":22,"acceptable":0,"unsafe":1,"review":0,"automated_count":22,"automated_acceptable":0,"automated_unsafe":1,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":0.045454545454545456,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":0.045454545454545456,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.14865487460862087,"confidence_level":0.95}},"effective":{"n":22,"acceptable":0,"unsafe":0,"review":2,"automated_count":20,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":2,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":0.09090909090909091,"automated_coverage":0.9090909090909091,"conditional_correctness":0.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.09090909090909091,"accuracy_wilson_95":{"lower":0.0,"upper":0.14865487460862087,"confidence_level":0.95}}},"mask":{"n":8,"raw":{"n":8,"acceptable":8,"unsafe":0,"review":0,"automated_count":8,"automated_acceptable":8,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":1.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.6755924351161198,"upper":1.0,"confidence_level":0.95}},"effective":{"n":8,"acceptable":8,"unsafe":0,"review":0,"automated_count":8,"automated_acceptable":8,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":1.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.6755924351161198,"upper":1.0,"confidence_level":0.95}}},"pseudonymize":{"n":8,"raw":{"n":8,"acceptable":8,"unsafe":0,"review":0,"automated_count":8,"automated_acceptable":8,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":1.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.6755924351161198,"upper":1.0,"confidence_level":0.95}},"effective":{"n":8,"acceptable":1,"unsafe":0,"review":7,"automated_count":1,"automated_acceptable":1,"automated_unsafe":0,"unnecessary_reviews":7,"accuracy":0.125,"unsafe_rate":0.0,"review_rate":0.875,"automated_coverage":0.125,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.875,"accuracy_wilson_95":{"lower":0.02241749145005667,"upper":0.4708881822128535,"confidence_level":0.95}}},"generalize":{"n":8,"raw":{"n":8,"acceptable":6,"unsafe":0,"review":0,"automated_count":8,"automated_acceptable":6,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":0.75,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.75,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.40927543031016883,"upper":0.9285207872478909,"confidence_level":0.95}},"effective":{"n":8,"acceptable":6,"unsafe":0,"review":0,"automated_count":8,"automated_acceptable":6,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":0.75,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.75,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.40927543031016883,"upper":0.9285207872478909,"confidence_level":0.95}}},"review":{"n":2,"raw":{"n":2,"acceptable":0,"unsafe":0,"review":0,"automated_count":2,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.6576197724933469,"confidence_level":0.95}},"effective":{"n":2,"acceptable":0,"unsafe":0,"review":0,"automated_count":2,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.6576197724933469,"confidence_level":0.95}}}}},"challenge56-compact_no_samples":{"n":56,"raw":{"n":56,"acceptable":21,"unsafe":24,"review":0,"automated_count":56,"automated_acceptable":21,"automated_unsafe":24,"unnecessary_reviews":0,"accuracy":0.375,"unsafe_rate":0.42857142857142855,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.375,"conditional_unsafe":0.42857142857142855,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.260102143264709,"upper":0.5059463409149819,"confidence_level":0.95}},"effective":{"n":56,"acceptable":15,"unsafe":0,"review":43,"automated_count":13,"automated_acceptable":13,"automated_unsafe":0,"unnecessary_reviews":41,"accuracy":0.26785714285714285,"unsafe_rate":0.0,"review_rate":0.7678571428571429,"automated_coverage":0.23214285714285715,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.7321428571428571,"accuracy_wilson_95":{"lower":0.16957305418501728,"upper":0.39594555929155156,"confidence_level":0.95}},"confusion":{"raw":{"keep":{"keep":0,"drop":0,"mask":0,"pseudonymize":8,"generalize":0,"review":0},"drop":{"keep":0,"drop":0,"mask":0,"pseudonymize":22,"generalize":0,"review":0},"mask":{"keep":0,"drop":0,"mask":8,"pseudonymize":0,"generalize":0,"review":0},"pseudonymize":{"keep":0,"drop":0,"mask":0,"pseudonymize":8,"generalize":0,"review":0},"generalize":{"keep":0,"drop":0,"mask":0,"pseudonymize":3,"generalize":5,"review":0},"review":{"keep":0,"drop":0,"mask":0,"pseudonymize":2,"generalize":0,"review":0}},"effective":{"keep":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":8},"drop":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":22},"mask":{"keep":0,"drop":0,"mask":8,"pseudonymize":0,"generalize":0,"review":0},"pseudonymize":{"keep":0,"drop":0,"mask":0,"pseudonymize":1,"generalize":0,"review":7},"generalize":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":4,"review":4},"review":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":2}}},"probability_metrics":{"singleton_n":56,"multiacceptable_n":0,"multiacceptable_coverage":0.0,"nll":2.182418004575718,"nll_zero_mass_n":0,"brier":0.9431450498832147,"acceptable_set_n":56,"acceptable_set_nll":2.182418004575718,"acceptable_set_zero_mass_n":0},"calibration_all_labels":{"equal_width":{"n":56,"bin_count":10,"ece":0.3815640104668481,"signed_mean_gap":0.3589792294161661,"absolute_mean_gap":0.3589792294161661,"bins":[{"index":0,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.0,"upper":0.1,"upper_inclusive":false},{"index":1,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.1,"upper":0.2,"upper_inclusive":false},{"index":2,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.2,"upper":0.3,"upper_inclusive":false},{"index":3,"count":2,"confidence":0.372562512755394,"empirical_correctness":0.5,"signed_gap":-0.12743748724460602,"absolute_gap":0.12743748724460602,"lower":0.3,"upper":0.4,"upper_inclusive":false},{"index":4,"count":2,"confidence":0.48184196650981903,"empirical_correctness":0.5,"signed_gap":-0.01815803349018097,"absolute_gap":0.01815803349018097,"lower":0.4,"upper":0.5,"upper_inclusive":false},{"index":5,"count":11,"confidence":0.5601161176508124,"empirical_correctness":0.2727272727272727,"signed_gap":0.28738884492353967,"absolute_gap":0.28738884492353967,"lower":0.5,"upper":0.6,"upper_inclusive":false},{"index":6,"count":8,"confidence":0.6427516341209412,"empirical_correctness":0.0,"signed_gap":0.6427516341209412,"absolute_gap":0.6427516341209412,"lower":0.6,"upper":0.7,"upper_inclusive":false},{"index":7,"count":11,"confidence":0.7608503103256226,"empirical_correctness":0.0,"signed_gap":0.7608503103256226,"absolute_gap":0.7608503103256226,"lower":0.7,"upper":0.8,"upper_inclusive":false},{"index":8,"count":13,"confidence":0.8509666873858526,"empirical_correctness":0.5384615384615384,"signed_gap":0.31250514892431414,"absolute_gap":0.31250514892431414,"lower":0.8,"upper":0.9,"upper_inclusive":false},{"index":9,"count":9,"confidence":0.9620907968944974,"empirical_correctness":1.0,"signed_gap":-0.03790920310550261,"absolute_gap":0.03790920310550261,"lower":0.9,"upper":1.0,"upper_inclusive":true}]},"equal_mass":{"n":56,"bin_count":10,"ece":0.3642487589802061,"signed_mean_gap":0.35897922941616606,"absolute_mean_gap":0.35897922941616606,"bins":[{"index":0,"count":5,"confidence":0.444320809841156,"empirical_correctness":0.4,"signed_gap":0.044320809841155995,"absolute_gap":0.044320809841155995,"min_confidence":0.35990720987319946,"max_confidence":0.512795090675354},{"index":1,"count":6,"confidence":0.5517785052458445,"empirical_correctness":0.16666666666666666,"signed_gap":0.38511183857917786,"absolute_gap":0.38511183857917786,"min_confidence":0.5162875056266785,"max_confidence":0.5791288018226624},{"index":2,"count":5,"confidence":0.589643931388855,"empirical_correctness":0.4,"signed_gap":0.18964393138885494,"absolute_gap":0.18964393138885494,"min_confidence":0.5791589617729187,"max_confidence":0.6104084849357605},{"index":3,"count":6,"confidence":0.6398697992165884,"empirical_correctness":0.0,"signed_gap":0.6398697992165884,"absolute_gap":0.6398697992165884,"min_confidence":0.623999834060669,"max_confidence":0.6595912575721741},{"index":4,"count":6,"confidence":0.7287127176920573,"empirical_correctness":0.0,"signed_gap":0.7287127176920573,"absolute_gap":0.7287127176920573,"min_confidence":0.6923857927322388,"max_confidence":0.748214602470398},{"index":5,"count":5,"confidence":0.7782256364822387,"empirical_correctness":0.0,"signed_gap":0.7782256364822387,"absolute_gap":0.7782256364822387,"min_confidence":0.7623188495635986,"max_confidence":0.7879564762115479},{"index":6,"count":6,"confidence":0.8127665917078654,"empirical_correctness":0.3333333333333333,"signed_gap":0.47943325837453205,"absolute_gap":0.47943325837453205,"min_confidence":0.7983347177505493,"max_confidence":0.835014820098877},{"index":7,"count":5,"confidence":0.8599792003631592,"empirical_correctness":0.6,"signed_gap":0.25997920036315925,"absolute_gap":0.25997920036315925,"min_confidence":0.836931586265564,"max_confidence":0.8778775334358215},{"index":8,"count":6,"confidence":0.9151283502578735,"empirical_correctness":0.8333333333333334,"signed_gap":0.08179501692454016,"absolute_gap":0.08179501692454016,"min_confidence":0.8903693556785583,"max_confidence":0.9547262787818909},{"index":9,"count":6,"confidence":0.9754088620344797,"empirical_correctness":1.0,"signed_gap":-0.02459113796552026,"absolute_gap":0.02459113796552026,"min_confidence":0.9635643362998962,"max_confidence":0.9856675863265991}]}},"calibration_singleton_labels":{"equal_width":{"n":56,"bin_count":10,"ece":0.3815640104668481,"signed_mean_gap":0.3589792294161661,"absolute_mean_gap":0.3589792294161661,"bins":[{"index":0,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.0,"upper":0.1,"upper_inclusive":false},{"index":1,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.1,"upper":0.2,"upper_inclusive":false},{"index":2,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.2,"upper":0.3,"upper_inclusive":false},{"index":3,"count":2,"confidence":0.372562512755394,"empirical_correctness":0.5,"signed_gap":-0.12743748724460602,"absolute_gap":0.12743748724460602,"lower":0.3,"upper":0.4,"upper_inclusive":false},{"index":4,"count":2,"confidence":0.48184196650981903,"empirical_correctness":0.5,"signed_gap":-0.01815803349018097,"absolute_gap":0.01815803349018097,"lower":0.4,"upper":0.5,"upper_inclusive":false},{"index":5,"count":11,"confidence":0.5601161176508124,"empirical_correctness":0.2727272727272727,"signed_gap":0.28738884492353967,"absolute_gap":0.28738884492353967,"lower":0.5,"upper":0.6,"upper_inclusive":false},{"index":6,"count":8,"confidence":0.6427516341209412,"empirical_correctness":0.0,"signed_gap":0.6427516341209412,"absolute_gap":0.6427516341209412,"lower":0.6,"upper":0.7,"upper_inclusive":false},{"index":7,"count":11,"confidence":0.7608503103256226,"empirical_correctness":0.0,"signed_gap":0.7608503103256226,"absolute_gap":0.7608503103256226,"lower":0.7,"upper":0.8,"upper_inclusive":false},{"index":8,"count":13,"confidence":0.8509666873858526,"empirical_correctness":0.5384615384615384,"signed_gap":0.31250514892431414,"absolute_gap":0.31250514892431414,"lower":0.8,"upper":0.9,"upper_inclusive":false},{"index":9,"count":9,"confidence":0.9620907968944974,"empirical_correctness":1.0,"signed_gap":-0.03790920310550261,"absolute_gap":0.03790920310550261,"lower":0.9,"upper":1.0,"upper_inclusive":true}]},"equal_mass":{"n":56,"bin_count":10,"ece":0.3642487589802061,"signed_mean_gap":0.35897922941616606,"absolute_mean_gap":0.35897922941616606,"bins":[{"index":0,"count":5,"confidence":0.444320809841156,"empirical_correctness":0.4,"signed_gap":0.044320809841155995,"absolute_gap":0.044320809841155995,"min_confidence":0.35990720987319946,"max_confidence":0.512795090675354},{"index":1,"count":6,"confidence":0.5517785052458445,"empirical_correctness":0.16666666666666666,"signed_gap":0.38511183857917786,"absolute_gap":0.38511183857917786,"min_confidence":0.5162875056266785,"max_confidence":0.5791288018226624},{"index":2,"count":5,"confidence":0.589643931388855,"empirical_correctness":0.4,"signed_gap":0.18964393138885494,"absolute_gap":0.18964393138885494,"min_confidence":0.5791589617729187,"max_confidence":0.6104084849357605},{"index":3,"count":6,"confidence":0.6398697992165884,"empirical_correctness":0.0,"signed_gap":0.6398697992165884,"absolute_gap":0.6398697992165884,"min_confidence":0.623999834060669,"max_confidence":0.6595912575721741},{"index":4,"count":6,"confidence":0.7287127176920573,"empirical_correctness":0.0,"signed_gap":0.7287127176920573,"absolute_gap":0.7287127176920573,"min_confidence":0.6923857927322388,"max_confidence":0.748214602470398},{"index":5,"count":5,"confidence":0.7782256364822387,"empirical_correctness":0.0,"signed_gap":0.7782256364822387,"absolute_gap":0.7782256364822387,"min_confidence":0.7623188495635986,"max_confidence":0.7879564762115479},{"index":6,"count":6,"confidence":0.8127665917078654,"empirical_correctness":0.3333333333333333,"signed_gap":0.47943325837453205,"absolute_gap":0.47943325837453205,"min_confidence":0.7983347177505493,"max_confidence":0.835014820098877},{"index":7,"count":5,"confidence":0.8599792003631592,"empirical_correctness":0.6,"signed_gap":0.25997920036315925,"absolute_gap":0.25997920036315925,"min_confidence":0.836931586265564,"max_confidence":0.8778775334358215},{"index":8,"count":6,"confidence":0.9151283502578735,"empirical_correctness":0.8333333333333334,"signed_gap":0.08179501692454016,"absolute_gap":0.08179501692454016,"min_confidence":0.8903693556785583,"max_confidence":0.9547262787818909},{"index":9,"count":6,"confidence":0.9754088620344797,"empirical_correctness":1.0,"signed_gap":-0.02459113796552026,"absolute_gap":0.02459113796552026,"min_confidence":0.9635643362998962,"max_confidence":0.9856675863265991}]}},"latency_ms":{"n":56,"median":56.74745450960472,"p95":60.65583998861257},"per_action":{"keep":{"n":8,"raw":{"n":8,"acceptable":0,"unsafe":0,"review":0,"automated_count":8,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.32440756488388023,"confidence_level":0.95}},"effective":{"n":8,"acceptable":0,"unsafe":0,"review":8,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":8,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.32440756488388023,"confidence_level":0.95}}},"drop":{"n":22,"raw":{"n":22,"acceptable":0,"unsafe":22,"review":0,"automated_count":22,"automated_acceptable":0,"automated_unsafe":22,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":1.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":1.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.14865487460862087,"confidence_level":0.95}},"effective":{"n":22,"acceptable":0,"unsafe":0,"review":22,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":22,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.14865487460862087,"confidence_level":0.95}}},"mask":{"n":8,"raw":{"n":8,"acceptable":8,"unsafe":0,"review":0,"automated_count":8,"automated_acceptable":8,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":1.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.6755924351161198,"upper":1.0,"confidence_level":0.95}},"effective":{"n":8,"acceptable":8,"unsafe":0,"review":0,"automated_count":8,"automated_acceptable":8,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":1.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.6755924351161198,"upper":1.0,"confidence_level":0.95}}},"pseudonymize":{"n":8,"raw":{"n":8,"acceptable":8,"unsafe":0,"review":0,"automated_count":8,"automated_acceptable":8,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":1.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.6755924351161198,"upper":1.0,"confidence_level":0.95}},"effective":{"n":8,"acceptable":1,"unsafe":0,"review":7,"automated_count":1,"automated_acceptable":1,"automated_unsafe":0,"unnecessary_reviews":7,"accuracy":0.125,"unsafe_rate":0.0,"review_rate":0.875,"automated_coverage":0.125,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.875,"accuracy_wilson_95":{"lower":0.02241749145005667,"upper":0.4708881822128535,"confidence_level":0.95}}},"generalize":{"n":8,"raw":{"n":8,"acceptable":5,"unsafe":0,"review":0,"automated_count":8,"automated_acceptable":5,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":0.625,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.625,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.30574239460262737,"upper":0.8631557141764026,"confidence_level":0.95}},"effective":{"n":8,"acceptable":4,"unsafe":0,"review":4,"automated_count":4,"automated_acceptable":4,"automated_unsafe":0,"unnecessary_reviews":4,"accuracy":0.5,"unsafe_rate":0.0,"review_rate":0.5,"automated_coverage":0.5,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.5,"accuracy_wilson_95":{"lower":0.21521606221387757,"upper":0.7847839377861224,"confidence_level":0.95}}},"review":{"n":2,"raw":{"n":2,"acceptable":0,"unsafe":2,"review":0,"automated_count":2,"automated_acceptable":0,"automated_unsafe":2,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":1.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":1.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.6576197724933469,"confidence_level":0.95}},"effective":{"n":2,"acceptable":2,"unsafe":0,"review":2,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":1.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.34238022750665303,"upper":1.0,"confidence_level":0.95}}}}}},"composite":{"n":198,"scope":"descriptive sum across four correlated panels","raw":{"n":198,"acceptable":68,"unsafe":36,"review":0,"automated_count":198,"automated_acceptable":68,"automated_unsafe":36,"unnecessary_reviews":0,"accuracy":0.3434343434343434,"unsafe_rate":0.18181818181818182,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.3434343434343434,"conditional_unsafe":0.18181818181818182,"unnecessary_review_rate":0.0},"effective":{"n":198,"acceptable":55,"unsafe":9,"review":81,"automated_count":117,"automated_acceptable":50,"automated_unsafe":9,"unnecessary_reviews":76,"accuracy":0.2777777777777778,"unsafe_rate":0.045454545454545456,"review_rate":0.4090909090909091,"automated_coverage":0.5909090909090909,"conditional_correctness":0.42735042735042733,"conditional_unsafe":0.07692307692307693,"unnecessary_review_rate":0.3838383838383838}}},{"id":"Horizon-Labs--multilingual-zeroshot-base","platform":"Local","model_id":"Horizon-Labs/multilingual-zeroshot-base","revision":"e944825ed3708cc63a32ca2992f9f594701dc447","loading":"float32 / none","scoring_kind":"Adapted NLI","settings":{"dtype":"float32","quantization":null,"max_tokens":4096,"effective_max_tokens":4096,"temperature":1.0,"scoring_batch_size":1,"trust_remote_code":false,"local_files_only":true,"use_safetensors":true,"guard_threshold":0.45,"seed":0,"tf32":false},"scoring_metadata":{"backend":"nli","protocol":"hf-local-sweep-backends-v1","probability_source":"adapted_entailment_softmax_not_calibrated_sixway","latency_scope":"synchronous_forward_scoring_including_transfer_excluding_tokenization","precision":"float32","equal_mass_convention":"rank bins with floor boundaries at floor(i*n/k), k=min(10,n//2); minimum two rows"},"load_ms":2671.7145889997482,"peak_allocated_bytes":1295690240,"peak_reserved_bytes":1312817152,"panels":{"diagnostic43-no_samples":{"n":43,"raw":{"n":43,"acceptable":7,"unsafe":7,"review":0,"automated_count":43,"automated_acceptable":7,"automated_unsafe":7,"unnecessary_reviews":0,"accuracy":0.16279069767441862,"unsafe_rate":0.16279069767441862,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.16279069767441862,"conditional_unsafe":0.16279069767441862,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.08116627347062008,"upper":0.29972405893747167,"confidence_level":0.95}},"effective":{"n":43,"acceptable":5,"unsafe":0,"review":43,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":38,"accuracy":0.11627906976744186,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":0.8837209302325582,"accuracy_wilson_95":{"lower":0.05070427739804166,"upper":0.24479161810082128,"confidence_level":0.95}},"confusion":{"raw":{"keep":{"keep":0,"drop":0,"mask":8,"pseudonymize":4,"generalize":0,"review":0},"drop":{"keep":0,"drop":0,"mask":11,"pseudonymize":5,"generalize":0,"review":0},"mask":{"keep":0,"drop":0,"mask":1,"pseudonymize":1,"generalize":0,"review":0},"pseudonymize":{"keep":0,"drop":0,"mask":0,"pseudonymize":2,"generalize":0,"review":0},"generalize":{"keep":0,"drop":0,"mask":5,"pseudonymize":1,"generalize":0,"review":0},"review":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0}},"effective":{"keep":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":12},"drop":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":16},"mask":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":2},"pseudonymize":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":2},"generalize":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":6},"review":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0}}},"probability_metrics":{"singleton_n":38,"multiacceptable_n":5,"multiacceptable_coverage":0.11627906976744186,"nll":1.832739126824429,"nll_zero_mass_n":0,"brier":0.8463745343099209,"acceptable_set_n":43,"acceptable_set_nll":1.7008828793692075,"acceptable_set_zero_mass_n":0},"calibration_all_labels":{"equal_width":{"n":43,"bin_count":10,"ece":0.01727881819702856,"signed_mean_gap":0.01727881819702856,"absolute_mean_gap":0.01727881819702856,"bins":[{"index":0,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.0,"upper":0.1,"upper_inclusive":false},{"index":1,"count":43,"confidence":0.18006951587144718,"empirical_correctness":0.16279069767441862,"signed_gap":0.01727881819702856,"absolute_gap":0.01727881819702856,"lower":0.1,"upper":0.2,"upper_inclusive":false},{"index":2,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.2,"upper":0.3,"upper_inclusive":false},{"index":3,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.3,"upper":0.4,"upper_inclusive":false},{"index":4,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.4,"upper":0.5,"upper_inclusive":false},{"index":5,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.5,"upper":0.6,"upper_inclusive":false},{"index":6,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.6,"upper":0.7,"upper_inclusive":false},{"index":7,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.7,"upper":0.8,"upper_inclusive":false},{"index":8,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.8,"upper":0.9,"upper_inclusive":false},{"index":9,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.9,"upper":1.0,"upper_inclusive":true}]},"equal_mass":{"n":43,"bin_count":10,"ece":0.15901475590328837,"signed_mean_gap":0.01727881819702858,"absolute_mean_gap":0.01727881819702858,"bins":[{"index":0,"count":4,"confidence":0.17544376105070114,"empirical_correctness":0.5,"signed_gap":-0.32455623894929886,"absolute_gap":0.32455623894929886,"min_confidence":0.1749778836965561,"max_confidence":0.17601549625396729},{"index":1,"count":4,"confidence":0.17664838209748268,"empirical_correctness":0.0,"signed_gap":0.17664838209748268,"absolute_gap":0.17664838209748268,"min_confidence":0.17623992264270782,"max_confidence":0.17714069783687592},{"index":2,"count":4,"confidence":0.17759623378515244,"empirical_correctness":0.0,"signed_gap":0.17759623378515244,"absolute_gap":0.17759623378515244,"min_confidence":0.17745646834373474,"max_confidence":0.17781339585781097},{"index":3,"count":5,"confidence":0.1783041089773178,"empirical_correctness":0.2,"signed_gap":-0.021695891022682212,"absolute_gap":0.021695891022682212,"min_confidence":0.1779155433177948,"max_confidence":0.17912036180496216},{"index":4,"count":4,"confidence":0.17960474640130997,"empirical_correctness":0.25,"signed_gap":-0.07039525359869003,"absolute_gap":0.07039525359869003,"min_confidence":0.17913879454135895,"max_confidence":0.18021345138549805},{"index":5,"count":4,"confidence":0.1808462142944336,"empirical_correctness":0.0,"signed_gap":0.1808462142944336,"absolute_gap":0.1808462142944336,"min_confidence":0.18033885955810547,"max_confidence":0.18114440143108368},{"index":6,"count":5,"confidence":0.18168839514255525,"empirical_correctness":0.2,"signed_gap":-0.018311604857444763,"absolute_gap":0.018311604857444763,"min_confidence":0.18134117126464844,"max_confidence":0.18202939629554749},{"index":7,"count":4,"confidence":0.18237366899847984,"empirical_correctness":0.0,"signed_gap":0.18237366899847984,"absolute_gap":0.18237366899847984,"min_confidence":0.18205834925174713,"max_confidence":0.18295714259147644},{"index":8,"count":4,"confidence":0.1831301972270012,"empirical_correctness":0.5,"signed_gap":-0.3168698027729988,"absolute_gap":0.3168698027729988,"min_confidence":0.18297065794467926,"max_confidence":0.18334658443927765},{"index":9,"count":5,"confidence":0.18409076929092408,"empirical_correctness":0.0,"signed_gap":0.18409076929092408,"absolute_gap":0.18409076929092408,"min_confidence":0.1835213452577591,"max_confidence":0.1848512589931488}]}},"calibration_singleton_labels":{"equal_width":{"n":38,"bin_count":10,"ece":0.10107306116505674,"signed_mean_gap":0.10107306116505674,"absolute_mean_gap":0.10107306116505674,"bins":[{"index":0,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.0,"upper":0.1,"upper_inclusive":false},{"index":1,"count":38,"confidence":0.18002042958610937,"empirical_correctness":0.07894736842105263,"signed_gap":0.10107306116505674,"absolute_gap":0.10107306116505674,"lower":0.1,"upper":0.2,"upper_inclusive":false},{"index":2,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.2,"upper":0.3,"upper_inclusive":false},{"index":3,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.3,"upper":0.4,"upper_inclusive":false},{"index":4,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.4,"upper":0.5,"upper_inclusive":false},{"index":5,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.5,"upper":0.6,"upper_inclusive":false},{"index":6,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.6,"upper":0.7,"upper_inclusive":false},{"index":7,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.7,"upper":0.8,"upper_inclusive":false},{"index":8,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.8,"upper":0.9,"upper_inclusive":false},{"index":9,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.9,"upper":1.0,"upper_inclusive":true}]},"equal_mass":{"n":38,"bin_count":10,"ece":0.15519301671730845,"signed_mean_gap":0.10107306116505672,"absolute_mean_gap":0.10107306116505672,"bins":[{"index":0,"count":3,"confidence":0.17548736929893494,"empirical_correctness":0.3333333333333333,"signed_gap":-0.15784596403439838,"absolute_gap":0.15784596403439838,"min_confidence":0.1749778836965561,"max_confidence":0.17601549625396729},{"index":1,"count":4,"confidence":0.17664838209748268,"empirical_correctness":0.0,"signed_gap":0.17664838209748268,"absolute_gap":0.17664838209748268,"min_confidence":0.17623992264270782,"max_confidence":0.17714069783687592},{"index":2,"count":4,"confidence":0.17759623378515244,"empirical_correctness":0.0,"signed_gap":0.17759623378515244,"absolute_gap":0.17759623378515244,"min_confidence":0.17745646834373474,"max_confidence":0.17781339585781097},{"index":3,"count":4,"confidence":0.17810004577040672,"empirical_correctness":0.25,"signed_gap":-0.07189995422959328,"absolute_gap":0.07189995422959328,"min_confidence":0.1779155433177948,"max_confidence":0.17824284732341766},{"index":4,"count":4,"confidence":0.179331474006176,"empirical_correctness":0.0,"signed_gap":0.179331474006176,"absolute_gap":0.179331474006176,"min_confidence":0.17912036180496216,"max_confidence":0.1798333376646042},{"index":5,"count":3,"confidence":0.1807468185822169,"empirical_correctness":0.0,"signed_gap":0.1807468185822169,"absolute_gap":0.1807468185822169,"min_confidence":0.18033885955810547,"max_confidence":0.18096747994422913},{"index":6,"count":4,"confidence":0.18152087181806564,"empirical_correctness":0.0,"signed_gap":0.18152087181806564,"absolute_gap":0.18152087181806564,"min_confidence":0.18114440143108368,"max_confidence":0.1818953901529312},{"index":7,"count":4,"confidence":0.18237366899847984,"empirical_correctness":0.0,"signed_gap":0.18237366899847984,"absolute_gap":0.18237366899847984,"min_confidence":0.18205834925174713,"max_confidence":0.18295714259147644},{"index":8,"count":4,"confidence":0.18321463838219643,"empirical_correctness":0.25,"signed_gap":-0.06678536161780357,"absolute_gap":0.06678536161780357,"min_confidence":0.18297065794467926,"max_confidence":0.1835213452577591},{"index":9,"count":4,"confidence":0.18423312529921532,"empirical_correctness":0.0,"signed_gap":0.18423312529921532,"absolute_gap":0.18423312529921532,"min_confidence":0.18363960087299347,"max_confidence":0.1848512589931488}]}},"latency_ms":{"n":43,"median":38.55846502119675,"p95":39.13074713200331},"per_action":{"keep":{"n":12,"raw":{"n":12,"acceptable":0,"unsafe":0,"review":0,"automated_count":12,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":2.7755575615628914e-17,"upper":0.24249400665524085,"confidence_level":0.95}},"effective":{"n":12,"acceptable":0,"unsafe":0,"review":12,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":12,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":2.7755575615628914e-17,"upper":0.24249400665524085,"confidence_level":0.95}}},"drop":{"n":16,"raw":{"n":16,"acceptable":0,"unsafe":5,"review":0,"automated_count":16,"automated_acceptable":0,"automated_unsafe":5,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":0.3125,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":0.3125,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.1936076805344365,"confidence_level":0.95}},"effective":{"n":16,"acceptable":0,"unsafe":0,"review":16,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":16,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.1936076805344365,"confidence_level":0.95}}},"mask":{"n":2,"raw":{"n":2,"acceptable":1,"unsafe":1,"review":0,"automated_count":2,"automated_acceptable":1,"automated_unsafe":1,"unnecessary_reviews":0,"accuracy":0.5,"unsafe_rate":0.5,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.5,"conditional_unsafe":0.5,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.09453120573423074,"upper":0.9054687942657693,"confidence_level":0.95}},"effective":{"n":2,"acceptable":0,"unsafe":0,"review":2,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":2,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.6576197724933469,"confidence_level":0.95}}},"pseudonymize":{"n":2,"raw":{"n":2,"acceptable":2,"unsafe":0,"review":0,"automated_count":2,"automated_acceptable":2,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":1.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.34238022750665303,"upper":1.0,"confidence_level":0.95}},"effective":{"n":2,"acceptable":0,"unsafe":0,"review":2,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":2,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.6576197724933469,"confidence_level":0.95}}},"generalize":{"n":6,"raw":{"n":6,"acceptable":0,"unsafe":0,"review":0,"automated_count":6,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":2.7755575615628914e-17,"upper":0.3903342879021653,"confidence_level":0.95}},"effective":{"n":6,"acceptable":0,"unsafe":0,"review":6,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":6,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":2.7755575615628914e-17,"upper":0.3903342879021653,"confidence_level":0.95}}},"review":{"n":0,"raw":{"n":0,"acceptable":0,"unsafe":0,"review":0,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":null,"unsafe_rate":null,"review_rate":null,"automated_coverage":null,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":null},"effective":{"n":0,"acceptable":0,"unsafe":0,"review":0,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":null,"unsafe_rate":null,"review_rate":null,"automated_coverage":null,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":null}}}},"diagnostic43-compact_no_samples":{"n":43,"raw":{"n":43,"acceptable":10,"unsafe":8,"review":8,"automated_count":35,"automated_acceptable":10,"automated_unsafe":8,"unnecessary_reviews":8,"accuracy":0.23255813953488372,"unsafe_rate":0.18604651162790697,"review_rate":0.18604651162790697,"automated_coverage":0.813953488372093,"conditional_correctness":0.2857142857142857,"conditional_unsafe":0.22857142857142856,"unnecessary_review_rate":0.18604651162790697,"accuracy_wilson_95":{"lower":0.1315367847938448,"upper":0.37744520297809003,"confidence_level":0.95}},"effective":{"n":43,"acceptable":5,"unsafe":0,"review":43,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":38,"accuracy":0.11627906976744186,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":0.8837209302325582,"accuracy_wilson_95":{"lower":0.05070427739804166,"upper":0.24479161810082128,"confidence_level":0.95}},"confusion":{"raw":{"keep":{"keep":4,"drop":0,"mask":6,"pseudonymize":0,"generalize":0,"review":2},"drop":{"keep":4,"drop":0,"mask":9,"pseudonymize":0,"generalize":0,"review":3},"mask":{"keep":0,"drop":0,"mask":1,"pseudonymize":0,"generalize":0,"review":1},"pseudonymize":{"keep":1,"drop":0,"mask":1,"pseudonymize":0,"generalize":0,"review":0},"generalize":{"keep":3,"drop":0,"mask":1,"pseudonymize":0,"generalize":0,"review":2},"review":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0}},"effective":{"keep":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":12},"drop":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":16},"mask":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":2},"pseudonymize":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":2},"generalize":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":6},"review":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0}}},"probability_metrics":{"singleton_n":38,"multiacceptable_n":5,"multiacceptable_coverage":0.11627906976744186,"nll":1.8160709317100359,"nll_zero_mass_n":0,"brier":0.8412632308591901,"acceptable_set_n":43,"acceptable_set_nll":1.6844693032227722,"acceptable_set_zero_mass_n":0},"calibration_all_labels":{"equal_width":{"n":43,"bin_count":10,"ece":0.055333706182102815,"signed_mean_gap":-0.055333706182102815,"absolute_mean_gap":0.055333706182102815,"bins":[{"index":0,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.0,"upper":0.1,"upper_inclusive":false},{"index":1,"count":43,"confidence":0.1772244333527809,"empirical_correctness":0.23255813953488372,"signed_gap":-0.055333706182102815,"absolute_gap":0.055333706182102815,"lower":0.1,"upper":0.2,"upper_inclusive":false},{"index":2,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.2,"upper":0.3,"upper_inclusive":false},{"index":3,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.3,"upper":0.4,"upper_inclusive":false},{"index":4,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.4,"upper":0.5,"upper_inclusive":false},{"index":5,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.5,"upper":0.6,"upper_inclusive":false},{"index":6,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.6,"upper":0.7,"upper_inclusive":false},{"index":7,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.7,"upper":0.8,"upper_inclusive":false},{"index":8,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.8,"upper":0.9,"upper_inclusive":false},{"index":9,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.9,"upper":1.0,"upper_inclusive":true}]},"equal_mass":{"n":43,"bin_count":10,"ece":0.2283615715974985,"signed_mean_gap":-0.05533370618210282,"absolute_mean_gap":0.05533370618210282,"bins":[{"index":0,"count":4,"confidence":0.1740451641380787,"empirical_correctness":0.0,"signed_gap":0.1740451641380787,"absolute_gap":0.1740451641380787,"min_confidence":0.17371287941932678,"max_confidence":0.17433366179466248},{"index":1,"count":4,"confidence":0.17500081285834312,"empirical_correctness":0.5,"signed_gap":-0.3249991871416569,"absolute_gap":0.3249991871416569,"min_confidence":0.17440931499004364,"max_confidence":0.17540031671524048},{"index":2,"count":4,"confidence":0.17572017759084702,"empirical_correctness":0.5,"signed_gap":-0.324279822409153,"absolute_gap":0.324279822409153,"min_confidence":0.17541678249835968,"max_confidence":0.17590555548667908},{"index":3,"count":5,"confidence":0.17661804854869842,"empirical_correctness":0.0,"signed_gap":0.17661804854869842,"absolute_gap":0.17661804854869842,"min_confidence":0.17634037137031555,"max_confidence":0.1770007312297821},{"index":4,"count":4,"confidence":0.1771305613219738,"empirical_correctness":0.0,"signed_gap":0.1771305613219738,"absolute_gap":0.1771305613219738,"min_confidence":0.17701154947280884,"max_confidence":0.17722097039222717},{"index":5,"count":4,"confidence":0.17736006900668144,"empirical_correctness":0.25,"signed_gap":-0.07263993099331856,"absolute_gap":0.07263993099331856,"min_confidence":0.1772719770669937,"max_confidence":0.17753912508487701},{"index":6,"count":5,"confidence":0.17789800465106964,"empirical_correctness":0.4,"signed_gap":-0.22210199534893038,"absolute_gap":0.22210199534893038,"min_confidence":0.17757822573184967,"max_confidence":0.17823736369609833},{"index":7,"count":4,"confidence":0.17881463840603828,"empirical_correctness":0.0,"signed_gap":0.17881463840603828,"absolute_gap":0.17881463840603828,"min_confidence":0.17847539484500885,"max_confidence":0.17897607386112213},{"index":8,"count":4,"confidence":0.17926185205578804,"empirical_correctness":0.0,"signed_gap":0.17926185205578804,"absolute_gap":0.17926185205578804,"min_confidence":0.1790132224559784,"max_confidence":0.1796330213546753},{"index":9,"count":5,"confidence":0.17974745333194733,"empirical_correctness":0.6,"signed_gap":-0.4202525466680527,"absolute_gap":0.4202525466680527,"min_confidence":0.17963416874408722,"max_confidence":0.1798560619354248}]}},"calibration_singleton_labels":{"equal_width":{"n":38,"bin_count":10,"ece":0.04545436956380544,"signed_mean_gap":0.04545436956380544,"absolute_mean_gap":0.04545436956380544,"bins":[{"index":0,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.0,"upper":0.1,"upper_inclusive":false},{"index":1,"count":38,"confidence":0.1770333169322265,"empirical_correctness":0.13157894736842105,"signed_gap":0.04545436956380544,"absolute_gap":0.04545436956380544,"lower":0.1,"upper":0.2,"upper_inclusive":false},{"index":2,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.2,"upper":0.3,"upper_inclusive":false},{"index":3,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.3,"upper":0.4,"upper_inclusive":false},{"index":4,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.4,"upper":0.5,"upper_inclusive":false},{"index":5,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.5,"upper":0.6,"upper_inclusive":false},{"index":6,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.6,"upper":0.7,"upper_inclusive":false},{"index":7,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.7,"upper":0.8,"upper_inclusive":false},{"index":8,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.8,"upper":0.9,"upper_inclusive":false},{"index":9,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.9,"upper":1.0,"upper_inclusive":true}]},"equal_mass":{"n":38,"bin_count":10,"ece":0.15991968465478798,"signed_mean_gap":0.04545436956380543,"absolute_mean_gap":0.04545436956380543,"bins":[{"index":0,"count":3,"confidence":0.17394899825255075,"empirical_correctness":0.0,"signed_gap":0.17394899825255075,"absolute_gap":0.17394899825255075,"min_confidence":0.17371287941932678,"max_confidence":0.1741020679473877},{"index":1,"count":4,"confidence":0.17473414912819862,"empirical_correctness":0.5,"signed_gap":-0.3252658508718014,"absolute_gap":0.3252658508718014,"min_confidence":0.17433366179466248,"max_confidence":0.1753099262714386},{"index":2,"count":4,"confidence":0.17559386789798737,"empirical_correctness":0.25,"signed_gap":-0.07440613210201263,"absolute_gap":0.07440613210201263,"min_confidence":0.17540031671524048,"max_confidence":0.17582885921001434},{"index":3,"count":4,"confidence":0.1762525625526905,"empirical_correctness":0.25,"signed_gap":-0.0737474374473095,"absolute_gap":0.0737474374473095,"min_confidence":0.17590555548667908,"max_confidence":0.17641861736774445},{"index":4,"count":4,"confidence":0.17702656611800194,"empirical_correctness":0.0,"signed_gap":0.17702656611800194,"absolute_gap":0.17702656611800194,"min_confidence":0.17698481678962708,"max_confidence":0.17710916697978973},{"index":5,"count":3,"confidence":0.17722450196743011,"empirical_correctness":0.0,"signed_gap":0.17722450196743011,"absolute_gap":0.17722450196743011,"min_confidence":0.17718055844306946,"max_confidence":0.1772719770669937},{"index":6,"count":4,"confidence":0.17745732888579369,"empirical_correctness":0.0,"signed_gap":0.17745732888579369,"absolute_gap":0.17745732888579369,"min_confidence":0.17731404304504395,"max_confidence":0.17762191593647003},{"index":7,"count":4,"confidence":0.1785651221871376,"empirical_correctness":0.0,"signed_gap":0.1785651221871376,"absolute_gap":0.1785651221871376,"min_confidence":0.1779780089855194,"max_confidence":0.17891713976860046},{"index":8,"count":4,"confidence":0.17909761518239975,"empirical_correctness":0.0,"signed_gap":0.17909761518239975,"absolute_gap":0.17909761518239975,"min_confidence":0.17897607386112213,"max_confidence":0.17921724915504456},{"index":9,"count":4,"confidence":0.17970917373895645,"empirical_correctness":0.25,"signed_gap":-0.07029082626104355,"absolute_gap":0.07029082626104355,"min_confidence":0.1796330213546753,"max_confidence":0.1798560619354248}]}},"latency_ms":{"n":43,"median":47.775466984603554,"p95":48.47126210224815},"per_action":{"keep":{"n":12,"raw":{"n":12,"acceptable":4,"unsafe":0,"review":2,"automated_count":10,"automated_acceptable":4,"automated_unsafe":0,"unnecessary_reviews":2,"accuracy":0.3333333333333333,"unsafe_rate":0.0,"review_rate":0.16666666666666666,"automated_coverage":0.8333333333333334,"conditional_correctness":0.4,"conditional_unsafe":0.0,"unnecessary_review_rate":0.16666666666666666,"accuracy_wilson_95":{"lower":0.13812009109121315,"upper":0.6093779111272004,"confidence_level":0.95}},"effective":{"n":12,"acceptable":0,"unsafe":0,"review":12,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":12,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":2.7755575615628914e-17,"upper":0.24249400665524085,"confidence_level":0.95}}},"drop":{"n":16,"raw":{"n":16,"acceptable":0,"unsafe":4,"review":3,"automated_count":13,"automated_acceptable":0,"automated_unsafe":4,"unnecessary_reviews":3,"accuracy":0.0,"unsafe_rate":0.25,"review_rate":0.1875,"automated_coverage":0.8125,"conditional_correctness":0.0,"conditional_unsafe":0.3076923076923077,"unnecessary_review_rate":0.1875,"accuracy_wilson_95":{"lower":0.0,"upper":0.1936076805344365,"confidence_level":0.95}},"effective":{"n":16,"acceptable":0,"unsafe":0,"review":16,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":16,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.1936076805344365,"confidence_level":0.95}}},"mask":{"n":2,"raw":{"n":2,"acceptable":1,"unsafe":0,"review":1,"automated_count":1,"automated_acceptable":1,"automated_unsafe":0,"unnecessary_reviews":1,"accuracy":0.5,"unsafe_rate":0.0,"review_rate":0.5,"automated_coverage":0.5,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.5,"accuracy_wilson_95":{"lower":0.09453120573423074,"upper":0.9054687942657693,"confidence_level":0.95}},"effective":{"n":2,"acceptable":0,"unsafe":0,"review":2,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":2,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.6576197724933469,"confidence_level":0.95}}},"pseudonymize":{"n":2,"raw":{"n":2,"acceptable":0,"unsafe":1,"review":0,"automated_count":2,"automated_acceptable":0,"automated_unsafe":1,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":0.5,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":0.5,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.6576197724933469,"confidence_level":0.95}},"effective":{"n":2,"acceptable":0,"unsafe":0,"review":2,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":2,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.6576197724933469,"confidence_level":0.95}}},"generalize":{"n":6,"raw":{"n":6,"acceptable":0,"unsafe":3,"review":2,"automated_count":4,"automated_acceptable":0,"automated_unsafe":3,"unnecessary_reviews":2,"accuracy":0.0,"unsafe_rate":0.5,"review_rate":0.3333333333333333,"automated_coverage":0.6666666666666666,"conditional_correctness":0.0,"conditional_unsafe":0.75,"unnecessary_review_rate":0.3333333333333333,"accuracy_wilson_95":{"lower":2.7755575615628914e-17,"upper":0.3903342879021653,"confidence_level":0.95}},"effective":{"n":6,"acceptable":0,"unsafe":0,"review":6,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":6,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":2.7755575615628914e-17,"upper":0.3903342879021653,"confidence_level":0.95}}},"review":{"n":0,"raw":{"n":0,"acceptable":0,"unsafe":0,"review":0,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":null,"unsafe_rate":null,"review_rate":null,"automated_coverage":null,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":null},"effective":{"n":0,"acceptable":0,"unsafe":0,"review":0,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":null,"unsafe_rate":null,"review_rate":null,"automated_coverage":null,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":null}}}},"challenge56-no_samples":{"n":56,"raw":{"n":56,"acceptable":8,"unsafe":10,"review":6,"automated_count":50,"automated_acceptable":7,"automated_unsafe":10,"unnecessary_reviews":5,"accuracy":0.14285714285714285,"unsafe_rate":0.17857142857142858,"review_rate":0.10714285714285714,"automated_coverage":0.8928571428571429,"conditional_correctness":0.14,"conditional_unsafe":0.2,"unnecessary_review_rate":0.08928571428571429,"accuracy_wilson_95":{"lower":0.07420788536920624,"upper":0.25735921228705333,"confidence_level":0.95}},"effective":{"n":56,"acceptable":2,"unsafe":0,"review":56,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":54,"accuracy":0.03571428571428571,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":0.9642857142857143,"accuracy_wilson_95":{"lower":0.00984942514823639,"upper":0.12118780180490107,"confidence_level":0.95}},"confusion":{"raw":{"keep":{"keep":1,"drop":0,"mask":3,"pseudonymize":4,"generalize":0,"review":0},"drop":{"keep":2,"drop":0,"mask":11,"pseudonymize":5,"generalize":0,"review":4},"mask":{"keep":1,"drop":0,"mask":6,"pseudonymize":1,"generalize":0,"review":0},"pseudonymize":{"keep":0,"drop":1,"mask":7,"pseudonymize":0,"generalize":0,"review":0},"generalize":{"keep":1,"drop":1,"mask":1,"pseudonymize":4,"generalize":0,"review":1},"review":{"keep":0,"drop":0,"mask":1,"pseudonymize":0,"generalize":0,"review":1}},"effective":{"keep":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":8},"drop":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":22},"mask":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":8},"pseudonymize":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":8},"generalize":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":8},"review":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":2}}},"probability_metrics":{"singleton_n":56,"multiacceptable_n":0,"multiacceptable_coverage":0.0,"nll":1.8204582847100679,"nll_zero_mass_n":0,"brier":0.8429094619618397,"acceptable_set_n":56,"acceptable_set_nll":1.8204582847100679,"acceptable_set_zero_mass_n":0},"calibration_all_labels":{"equal_width":{"n":56,"bin_count":10,"ece":0.03765446825751236,"signed_mean_gap":0.03765446825751236,"absolute_mean_gap":0.03765446825751236,"bins":[{"index":0,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.0,"upper":0.1,"upper_inclusive":false},{"index":1,"count":56,"confidence":0.18051161111465522,"empirical_correctness":0.14285714285714285,"signed_gap":0.03765446825751237,"absolute_gap":0.03765446825751237,"lower":0.1,"upper":0.2,"upper_inclusive":false},{"index":2,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.2,"upper":0.3,"upper_inclusive":false},{"index":3,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.3,"upper":0.4,"upper_inclusive":false},{"index":4,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.4,"upper":0.5,"upper_inclusive":false},{"index":5,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.5,"upper":0.6,"upper_inclusive":false},{"index":6,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.6,"upper":0.7,"upper_inclusive":false},{"index":7,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.7,"upper":0.8,"upper_inclusive":false},{"index":8,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.8,"upper":0.9,"upper_inclusive":false},{"index":9,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.9,"upper":1.0,"upper_inclusive":true}]},"equal_mass":{"n":56,"bin_count":10,"ece":0.05263843307537693,"signed_mean_gap":0.03765446825751236,"absolute_mean_gap":0.03765446825751236,"bins":[{"index":0,"count":5,"confidence":0.1732092022895813,"empirical_correctness":0.2,"signed_gap":-0.0267907977104187,"absolute_gap":0.0267907977104187,"min_confidence":0.1694241315126419,"max_confidence":0.17521749436855316},{"index":1,"count":6,"confidence":0.17700876792271933,"empirical_correctness":0.0,"signed_gap":0.17700876792271933,"absolute_gap":0.17700876792271933,"min_confidence":0.17540864646434784,"max_confidence":0.17822405695915222},{"index":2,"count":5,"confidence":0.17863291800022124,"empirical_correctness":0.2,"signed_gap":-0.02136708199977877,"absolute_gap":0.02136708199977877,"min_confidence":0.17833513021469116,"max_confidence":0.17886711657047272},{"index":3,"count":6,"confidence":0.1793878898024559,"empirical_correctness":0.0,"signed_gap":0.1793878898024559,"absolute_gap":0.1793878898024559,"min_confidence":0.1791246235370636,"max_confidence":0.17968642711639404},{"index":4,"count":6,"confidence":0.18010944624741873,"empirical_correctness":0.16666666666666666,"signed_gap":0.013442779580752073,"absolute_gap":0.013442779580752073,"min_confidence":0.17969565093517303,"max_confidence":0.1805221289396286},{"index":5,"count":5,"confidence":0.18109035789966582,"empirical_correctness":0.2,"signed_gap":-0.01890964210033419,"absolute_gap":0.01890964210033419,"min_confidence":0.18088452517986298,"max_confidence":0.18141159415245056},{"index":6,"count":6,"confidence":0.18180020650227866,"empirical_correctness":0.16666666666666666,"signed_gap":0.015133539835611998,"absolute_gap":0.015133539835611998,"min_confidence":0.1814768761396408,"max_confidence":0.18246880173683167},{"index":7,"count":5,"confidence":0.1831573188304901,"empirical_correctness":0.2,"signed_gap":-0.016842681169509904,"absolute_gap":0.016842681169509904,"min_confidence":0.18265199661254883,"max_confidence":0.18347285687923431},{"index":8,"count":6,"confidence":0.18425112962722778,"empirical_correctness":0.16666666666666666,"signed_gap":0.017584462960561126,"absolute_gap":0.017584462960561126,"min_confidence":0.1835179477930069,"max_confidence":0.18463310599327087},{"index":9,"count":6,"confidence":0.18547609945138296,"empirical_correctness":0.16666666666666666,"signed_gap":0.018809432784716307,"absolute_gap":0.018809432784716307,"min_confidence":0.18468065559864044,"max_confidence":0.1863856464624405}]}},"calibration_singleton_labels":{"equal_width":{"n":56,"bin_count":10,"ece":0.03765446825751236,"signed_mean_gap":0.03765446825751236,"absolute_mean_gap":0.03765446825751236,"bins":[{"index":0,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.0,"upper":0.1,"upper_inclusive":false},{"index":1,"count":56,"confidence":0.18051161111465522,"empirical_correctness":0.14285714285714285,"signed_gap":0.03765446825751237,"absolute_gap":0.03765446825751237,"lower":0.1,"upper":0.2,"upper_inclusive":false},{"index":2,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.2,"upper":0.3,"upper_inclusive":false},{"index":3,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.3,"upper":0.4,"upper_inclusive":false},{"index":4,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.4,"upper":0.5,"upper_inclusive":false},{"index":5,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.5,"upper":0.6,"upper_inclusive":false},{"index":6,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.6,"upper":0.7,"upper_inclusive":false},{"index":7,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.7,"upper":0.8,"upper_inclusive":false},{"index":8,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.8,"upper":0.9,"upper_inclusive":false},{"index":9,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.9,"upper":1.0,"upper_inclusive":true}]},"equal_mass":{"n":56,"bin_count":10,"ece":0.05263843307537693,"signed_mean_gap":0.03765446825751236,"absolute_mean_gap":0.03765446825751236,"bins":[{"index":0,"count":5,"confidence":0.1732092022895813,"empirical_correctness":0.2,"signed_gap":-0.0267907977104187,"absolute_gap":0.0267907977104187,"min_confidence":0.1694241315126419,"max_confidence":0.17521749436855316},{"index":1,"count":6,"confidence":0.17700876792271933,"empirical_correctness":0.0,"signed_gap":0.17700876792271933,"absolute_gap":0.17700876792271933,"min_confidence":0.17540864646434784,"max_confidence":0.17822405695915222},{"index":2,"count":5,"confidence":0.17863291800022124,"empirical_correctness":0.2,"signed_gap":-0.02136708199977877,"absolute_gap":0.02136708199977877,"min_confidence":0.17833513021469116,"max_confidence":0.17886711657047272},{"index":3,"count":6,"confidence":0.1793878898024559,"empirical_correctness":0.0,"signed_gap":0.1793878898024559,"absolute_gap":0.1793878898024559,"min_confidence":0.1791246235370636,"max_confidence":0.17968642711639404},{"index":4,"count":6,"confidence":0.18010944624741873,"empirical_correctness":0.16666666666666666,"signed_gap":0.013442779580752073,"absolute_gap":0.013442779580752073,"min_confidence":0.17969565093517303,"max_confidence":0.1805221289396286},{"index":5,"count":5,"confidence":0.18109035789966582,"empirical_correctness":0.2,"signed_gap":-0.01890964210033419,"absolute_gap":0.01890964210033419,"min_confidence":0.18088452517986298,"max_confidence":0.18141159415245056},{"index":6,"count":6,"confidence":0.18180020650227866,"empirical_correctness":0.16666666666666666,"signed_gap":0.015133539835611998,"absolute_gap":0.015133539835611998,"min_confidence":0.1814768761396408,"max_confidence":0.18246880173683167},{"index":7,"count":5,"confidence":0.1831573188304901,"empirical_correctness":0.2,"signed_gap":-0.016842681169509904,"absolute_gap":0.016842681169509904,"min_confidence":0.18265199661254883,"max_confidence":0.18347285687923431},{"index":8,"count":6,"confidence":0.18425112962722778,"empirical_correctness":0.16666666666666666,"signed_gap":0.017584462960561126,"absolute_gap":0.017584462960561126,"min_confidence":0.1835179477930069,"max_confidence":0.18463310599327087},{"index":9,"count":6,"confidence":0.18547609945138296,"empirical_correctness":0.16666666666666666,"signed_gap":0.018809432784716307,"absolute_gap":0.018809432784716307,"min_confidence":0.18468065559864044,"max_confidence":0.1863856464624405}]}},"latency_ms":{"n":56,"median":35.96910249325447,"p95":37.06366950063966},"per_action":{"keep":{"n":8,"raw":{"n":8,"acceptable":1,"unsafe":0,"review":0,"automated_count":8,"automated_acceptable":1,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":0.125,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.125,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.02241749145005667,"upper":0.4708881822128535,"confidence_level":0.95}},"effective":{"n":8,"acceptable":0,"unsafe":0,"review":8,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":8,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.32440756488388023,"confidence_level":0.95}}},"drop":{"n":22,"raw":{"n":22,"acceptable":0,"unsafe":7,"review":4,"automated_count":18,"automated_acceptable":0,"automated_unsafe":7,"unnecessary_reviews":4,"accuracy":0.0,"unsafe_rate":0.3181818181818182,"review_rate":0.18181818181818182,"automated_coverage":0.8181818181818182,"conditional_correctness":0.0,"conditional_unsafe":0.3888888888888889,"unnecessary_review_rate":0.18181818181818182,"accuracy_wilson_95":{"lower":0.0,"upper":0.14865487460862087,"confidence_level":0.95}},"effective":{"n":22,"acceptable":0,"unsafe":0,"review":22,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":22,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.14865487460862087,"confidence_level":0.95}}},"mask":{"n":8,"raw":{"n":8,"acceptable":6,"unsafe":2,"review":0,"automated_count":8,"automated_acceptable":6,"automated_unsafe":2,"unnecessary_reviews":0,"accuracy":0.75,"unsafe_rate":0.25,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.75,"conditional_unsafe":0.25,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.40927543031016883,"upper":0.9285207872478909,"confidence_level":0.95}},"effective":{"n":8,"acceptable":0,"unsafe":0,"review":8,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":8,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.32440756488388023,"confidence_level":0.95}}},"pseudonymize":{"n":8,"raw":{"n":8,"acceptable":0,"unsafe":0,"review":0,"automated_count":8,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.32440756488388023,"confidence_level":0.95}},"effective":{"n":8,"acceptable":0,"unsafe":0,"review":8,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":8,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.32440756488388023,"confidence_level":0.95}}},"generalize":{"n":8,"raw":{"n":8,"acceptable":0,"unsafe":1,"review":1,"automated_count":7,"automated_acceptable":0,"automated_unsafe":1,"unnecessary_reviews":1,"accuracy":0.0,"unsafe_rate":0.125,"review_rate":0.125,"automated_coverage":0.875,"conditional_correctness":0.0,"conditional_unsafe":0.14285714285714285,"unnecessary_review_rate":0.125,"accuracy_wilson_95":{"lower":0.0,"upper":0.32440756488388023,"confidence_level":0.95}},"effective":{"n":8,"acceptable":0,"unsafe":0,"review":8,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":8,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.32440756488388023,"confidence_level":0.95}}},"review":{"n":2,"raw":{"n":2,"acceptable":1,"unsafe":0,"review":1,"automated_count":1,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":0.5,"unsafe_rate":0.0,"review_rate":0.5,"automated_coverage":0.5,"conditional_correctness":0.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.09453120573423074,"upper":0.9054687942657693,"confidence_level":0.95}},"effective":{"n":2,"acceptable":2,"unsafe":0,"review":2,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":1.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.34238022750665303,"upper":1.0,"confidence_level":0.95}}}}},"challenge56-compact_no_samples":{"n":56,"raw":{"n":56,"acceptable":13,"unsafe":7,"review":9,"automated_count":47,"automated_acceptable":13,"automated_unsafe":7,"unnecessary_reviews":9,"accuracy":0.23214285714285715,"unsafe_rate":0.125,"review_rate":0.16071428571428573,"automated_coverage":0.8392857142857143,"conditional_correctness":0.2765957446808511,"conditional_unsafe":0.14893617021276595,"unnecessary_review_rate":0.16071428571428573,"accuracy_wilson_95":{"lower":0.1409937910920862,"upper":0.35768153215010856,"confidence_level":0.95}},"effective":{"n":56,"acceptable":2,"unsafe":0,"review":56,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":54,"accuracy":0.03571428571428571,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":0.9642857142857143,"accuracy_wilson_95":{"lower":0.00984942514823639,"upper":0.12118780180490107,"confidence_level":0.95}},"confusion":{"raw":{"keep":{"keep":7,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":1},"drop":{"keep":3,"drop":0,"mask":13,"pseudonymize":0,"generalize":0,"review":6},"mask":{"keep":1,"drop":0,"mask":6,"pseudonymize":0,"generalize":0,"review":1},"pseudonymize":{"keep":0,"drop":0,"mask":8,"pseudonymize":0,"generalize":0,"review":0},"generalize":{"keep":2,"drop":0,"mask":5,"pseudonymize":0,"generalize":0,"review":1},"review":{"keep":1,"drop":0,"mask":1,"pseudonymize":0,"generalize":0,"review":0}},"effective":{"keep":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":8},"drop":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":22},"mask":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":8},"pseudonymize":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":8},"generalize":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":8},"review":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":2}}},"probability_metrics":{"singleton_n":56,"multiacceptable_n":0,"multiacceptable_coverage":0.0,"nll":1.8162642621768943,"nll_zero_mass_n":0,"brier":0.841248551099647,"acceptable_set_n":56,"acceptable_set_nll":1.8162642621768943,"acceptable_set_zero_mass_n":0},"calibration_all_labels":{"equal_width":{"n":56,"bin_count":10,"ece":0.05531395838728973,"signed_mean_gap":-0.05531395838728973,"absolute_mean_gap":0.05531395838728973,"bins":[{"index":0,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.0,"upper":0.1,"upper_inclusive":false},{"index":1,"count":56,"confidence":0.17682889875556743,"empirical_correctness":0.23214285714285715,"signed_gap":-0.055313958387289724,"absolute_gap":0.055313958387289724,"lower":0.1,"upper":0.2,"upper_inclusive":false},{"index":2,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.2,"upper":0.3,"upper_inclusive":false},{"index":3,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.3,"upper":0.4,"upper_inclusive":false},{"index":4,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.4,"upper":0.5,"upper_inclusive":false},{"index":5,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.5,"upper":0.6,"upper_inclusive":false},{"index":6,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.6,"upper":0.7,"upper_inclusive":false},{"index":7,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.7,"upper":0.8,"upper_inclusive":false},{"index":8,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.8,"upper":0.9,"upper_inclusive":false},{"index":9,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.9,"upper":1.0,"upper_inclusive":true}]},"equal_mass":{"n":56,"bin_count":10,"ece":0.12491144372948579,"signed_mean_gap":-0.05531395838728972,"absolute_mean_gap":0.05531395838728972,"bins":[{"index":0,"count":5,"confidence":0.17303573191165925,"empirical_correctness":0.0,"signed_gap":0.17303573191165925,"absolute_gap":0.17303573191165925,"min_confidence":0.17236846685409546,"max_confidence":0.17393344640731812},{"index":1,"count":6,"confidence":0.17485657831033072,"empirical_correctness":0.3333333333333333,"signed_gap":-0.1584767550230026,"absolute_gap":0.1584767550230026,"min_confidence":0.17407715320587158,"max_confidence":0.17527616024017334},{"index":2,"count":5,"confidence":0.17548657953739166,"empirical_correctness":0.2,"signed_gap":-0.02451342046260835,"absolute_gap":0.02451342046260835,"min_confidence":0.17534196376800537,"max_confidence":0.17558105289936066},{"index":3,"count":6,"confidence":0.17595024158557257,"empirical_correctness":0.16666666666666666,"signed_gap":0.009283574918905912,"absolute_gap":0.009283574918905912,"min_confidence":0.17588649690151215,"max_confidence":0.17606478929519653},{"index":4,"count":6,"confidence":0.17658082892497381,"empirical_correctness":0.16666666666666666,"signed_gap":0.009914162258307158,"absolute_gap":0.009914162258307158,"min_confidence":0.17630897462368011,"max_confidence":0.1768224984407425},{"index":5,"count":5,"confidence":0.17714363634586333,"empirical_correctness":0.0,"signed_gap":0.17714363634586333,"absolute_gap":0.17714363634586333,"min_confidence":0.17700105905532837,"max_confidence":0.17723779380321503},{"index":6,"count":6,"confidence":0.17755543688933054,"empirical_correctness":0.6666666666666666,"signed_gap":-0.4891112297773361,"absolute_gap":0.4891112297773361,"min_confidence":0.17729686200618744,"max_confidence":0.1778416782617569},{"index":7,"count":5,"confidence":0.17807018160820007,"empirical_correctness":0.2,"signed_gap":-0.021929818391799943,"absolute_gap":0.021929818391799943,"min_confidence":0.1778755933046341,"max_confidence":0.17832474410533905},{"index":8,"count":6,"confidence":0.1785721406340599,"empirical_correctness":0.3333333333333333,"signed_gap":-0.1547611926992734,"absolute_gap":0.1547611926992734,"min_confidence":0.17834828794002533,"max_confidence":0.17881090939044952},{"index":9,"count":6,"confidence":0.18044105420509973,"empirical_correctness":0.16666666666666666,"signed_gap":0.013774387538433075,"absolute_gap":0.013774387538433075,"min_confidence":0.1794872283935547,"max_confidence":0.18187037110328674}]}},"calibration_singleton_labels":{"equal_width":{"n":56,"bin_count":10,"ece":0.05531395838728973,"signed_mean_gap":-0.05531395838728973,"absolute_mean_gap":0.05531395838728973,"bins":[{"index":0,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.0,"upper":0.1,"upper_inclusive":false},{"index":1,"count":56,"confidence":0.17682889875556743,"empirical_correctness":0.23214285714285715,"signed_gap":-0.055313958387289724,"absolute_gap":0.055313958387289724,"lower":0.1,"upper":0.2,"upper_inclusive":false},{"index":2,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.2,"upper":0.3,"upper_inclusive":false},{"index":3,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.3,"upper":0.4,"upper_inclusive":false},{"index":4,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.4,"upper":0.5,"upper_inclusive":false},{"index":5,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.5,"upper":0.6,"upper_inclusive":false},{"index":6,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.6,"upper":0.7,"upper_inclusive":false},{"index":7,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.7,"upper":0.8,"upper_inclusive":false},{"index":8,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.8,"upper":0.9,"upper_inclusive":false},{"index":9,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.9,"upper":1.0,"upper_inclusive":true}]},"equal_mass":{"n":56,"bin_count":10,"ece":0.12491144372948579,"signed_mean_gap":-0.05531395838728972,"absolute_mean_gap":0.05531395838728972,"bins":[{"index":0,"count":5,"confidence":0.17303573191165925,"empirical_correctness":0.0,"signed_gap":0.17303573191165925,"absolute_gap":0.17303573191165925,"min_confidence":0.17236846685409546,"max_confidence":0.17393344640731812},{"index":1,"count":6,"confidence":0.17485657831033072,"empirical_correctness":0.3333333333333333,"signed_gap":-0.1584767550230026,"absolute_gap":0.1584767550230026,"min_confidence":0.17407715320587158,"max_confidence":0.17527616024017334},{"index":2,"count":5,"confidence":0.17548657953739166,"empirical_correctness":0.2,"signed_gap":-0.02451342046260835,"absolute_gap":0.02451342046260835,"min_confidence":0.17534196376800537,"max_confidence":0.17558105289936066},{"index":3,"count":6,"confidence":0.17595024158557257,"empirical_correctness":0.16666666666666666,"signed_gap":0.009283574918905912,"absolute_gap":0.009283574918905912,"min_confidence":0.17588649690151215,"max_confidence":0.17606478929519653},{"index":4,"count":6,"confidence":0.17658082892497381,"empirical_correctness":0.16666666666666666,"signed_gap":0.009914162258307158,"absolute_gap":0.009914162258307158,"min_confidence":0.17630897462368011,"max_confidence":0.1768224984407425},{"index":5,"count":5,"confidence":0.17714363634586333,"empirical_correctness":0.0,"signed_gap":0.17714363634586333,"absolute_gap":0.17714363634586333,"min_confidence":0.17700105905532837,"max_confidence":0.17723779380321503},{"index":6,"count":6,"confidence":0.17755543688933054,"empirical_correctness":0.6666666666666666,"signed_gap":-0.4891112297773361,"absolute_gap":0.4891112297773361,"min_confidence":0.17729686200618744,"max_confidence":0.1778416782617569},{"index":7,"count":5,"confidence":0.17807018160820007,"empirical_correctness":0.2,"signed_gap":-0.021929818391799943,"absolute_gap":0.021929818391799943,"min_confidence":0.1778755933046341,"max_confidence":0.17832474410533905},{"index":8,"count":6,"confidence":0.1785721406340599,"empirical_correctness":0.3333333333333333,"signed_gap":-0.1547611926992734,"absolute_gap":0.1547611926992734,"min_confidence":0.17834828794002533,"max_confidence":0.17881090939044952},{"index":9,"count":6,"confidence":0.18044105420509973,"empirical_correctness":0.16666666666666666,"signed_gap":0.013774387538433075,"absolute_gap":0.013774387538433075,"min_confidence":0.1794872283935547,"max_confidence":0.18187037110328674}]}},"latency_ms":{"n":56,"median":38.85249453014694,"p95":39.54620903823525},"per_action":{"keep":{"n":8,"raw":{"n":8,"acceptable":7,"unsafe":0,"review":1,"automated_count":7,"automated_acceptable":7,"automated_unsafe":0,"unnecessary_reviews":1,"accuracy":0.875,"unsafe_rate":0.0,"review_rate":0.125,"automated_coverage":0.875,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.125,"accuracy_wilson_95":{"lower":0.5291118177871464,"upper":0.9775825085499433,"confidence_level":0.95}},"effective":{"n":8,"acceptable":0,"unsafe":0,"review":8,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":8,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.32440756488388023,"confidence_level":0.95}}},"drop":{"n":22,"raw":{"n":22,"acceptable":0,"unsafe":3,"review":6,"automated_count":16,"automated_acceptable":0,"automated_unsafe":3,"unnecessary_reviews":6,"accuracy":0.0,"unsafe_rate":0.13636363636363635,"review_rate":0.2727272727272727,"automated_coverage":0.7272727272727273,"conditional_correctness":0.0,"conditional_unsafe":0.1875,"unnecessary_review_rate":0.2727272727272727,"accuracy_wilson_95":{"lower":0.0,"upper":0.14865487460862087,"confidence_level":0.95}},"effective":{"n":22,"acceptable":0,"unsafe":0,"review":22,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":22,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.14865487460862087,"confidence_level":0.95}}},"mask":{"n":8,"raw":{"n":8,"acceptable":6,"unsafe":1,"review":1,"automated_count":7,"automated_acceptable":6,"automated_unsafe":1,"unnecessary_reviews":1,"accuracy":0.75,"unsafe_rate":0.125,"review_rate":0.125,"automated_coverage":0.875,"conditional_correctness":0.8571428571428571,"conditional_unsafe":0.14285714285714285,"unnecessary_review_rate":0.125,"accuracy_wilson_95":{"lower":0.40927543031016883,"upper":0.9285207872478909,"confidence_level":0.95}},"effective":{"n":8,"acceptable":0,"unsafe":0,"review":8,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":8,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.32440756488388023,"confidence_level":0.95}}},"pseudonymize":{"n":8,"raw":{"n":8,"acceptable":0,"unsafe":0,"review":0,"automated_count":8,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.32440756488388023,"confidence_level":0.95}},"effective":{"n":8,"acceptable":0,"unsafe":0,"review":8,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":8,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.32440756488388023,"confidence_level":0.95}}},"generalize":{"n":8,"raw":{"n":8,"acceptable":0,"unsafe":2,"review":1,"automated_count":7,"automated_acceptable":0,"automated_unsafe":2,"unnecessary_reviews":1,"accuracy":0.0,"unsafe_rate":0.25,"review_rate":0.125,"automated_coverage":0.875,"conditional_correctness":0.0,"conditional_unsafe":0.2857142857142857,"unnecessary_review_rate":0.125,"accuracy_wilson_95":{"lower":0.0,"upper":0.32440756488388023,"confidence_level":0.95}},"effective":{"n":8,"acceptable":0,"unsafe":0,"review":8,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":8,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.32440756488388023,"confidence_level":0.95}}},"review":{"n":2,"raw":{"n":2,"acceptable":0,"unsafe":1,"review":0,"automated_count":2,"automated_acceptable":0,"automated_unsafe":1,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":0.5,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":0.5,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.6576197724933469,"confidence_level":0.95}},"effective":{"n":2,"acceptable":2,"unsafe":0,"review":2,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":1.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.34238022750665303,"upper":1.0,"confidence_level":0.95}}}}}},"composite":{"n":198,"scope":"descriptive sum across four correlated panels","raw":{"n":198,"acceptable":38,"unsafe":32,"review":23,"automated_count":175,"automated_acceptable":37,"automated_unsafe":32,"unnecessary_reviews":22,"accuracy":0.1919191919191919,"unsafe_rate":0.16161616161616163,"review_rate":0.11616161616161616,"automated_coverage":0.8838383838383839,"conditional_correctness":0.21142857142857144,"conditional_unsafe":0.18285714285714286,"unnecessary_review_rate":0.1111111111111111},"effective":{"n":198,"acceptable":14,"unsafe":0,"review":198,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":184,"accuracy":0.0707070707070707,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":0.9292929292929293}}},{"id":"Horizon-Labs--multilingual-zeroshot-large","platform":"Local","model_id":"Horizon-Labs/multilingual-zeroshot-large","revision":"f11cdaec81323057a66f0e0e78aaf49b3bffbc6e","loading":"float32 / none","scoring_kind":"Adapted NLI","settings":{"dtype":"float32","quantization":null,"max_tokens":4096,"effective_max_tokens":4096,"temperature":1.0,"scoring_batch_size":1,"trust_remote_code":false,"local_files_only":true,"use_safetensors":true,"guard_threshold":0.45,"seed":0,"tf32":false},"scoring_metadata":{"backend":"nli","protocol":"hf-local-sweep-backends-v1","probability_source":"adapted_entailment_softmax_not_calibrated_sixway","latency_scope":"synchronous_forward_scoring_including_transfer_excluding_tokenization","precision":"float32","equal_mass_convention":"rank bins with floor boundaries at floor(i*n/k), k=min(10,n//2); minimum two rows"},"load_ms":2347.633555997163,"peak_allocated_bytes":2324177920,"peak_reserved_bytes":2342518784,"panels":{"diagnostic43-no_samples":{"n":43,"raw":{"n":43,"acceptable":12,"unsafe":25,"review":0,"automated_count":43,"automated_acceptable":12,"automated_unsafe":25,"unnecessary_reviews":0,"accuracy":0.27906976744186046,"unsafe_rate":0.5813953488372093,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.27906976744186046,"conditional_unsafe":0.5813953488372093,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.16746613786237488,"upper":0.42691028681878873,"confidence_level":0.95}},"effective":{"n":43,"acceptable":6,"unsafe":0,"review":38,"automated_count":5,"automated_acceptable":1,"automated_unsafe":0,"unnecessary_reviews":33,"accuracy":0.13953488372093023,"unsafe_rate":0.0,"review_rate":0.8837209302325582,"automated_coverage":0.11627906976744186,"conditional_correctness":0.2,"conditional_unsafe":0.0,"unnecessary_review_rate":0.7674418604651163,"accuracy_wilson_95":{"lower":0.06555723427964878,"upper":0.2726358796738286,"confidence_level":0.95}},"confusion":{"raw":{"keep":{"keep":12,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0},"drop":{"keep":12,"drop":0,"mask":2,"pseudonymize":2,"generalize":0,"review":0},"mask":{"keep":2,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0},"pseudonymize":{"keep":0,"drop":0,"mask":2,"pseudonymize":0,"generalize":0,"review":0},"generalize":{"keep":4,"drop":0,"mask":2,"pseudonymize":0,"generalize":0,"review":0},"review":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0}},"effective":{"keep":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":12},"drop":{"keep":0,"drop":0,"mask":4,"pseudonymize":0,"generalize":0,"review":12},"mask":{"keep":0,"drop":0,"mask":1,"pseudonymize":0,"generalize":0,"review":1},"pseudonymize":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":2},"generalize":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":6},"review":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0}}},"probability_metrics":{"singleton_n":38,"multiacceptable_n":5,"multiacceptable_coverage":0.11627906976744186,"nll":1.7727706695733692,"nll_zero_mass_n":0,"brier":0.8253187312004535,"acceptable_set_n":43,"acceptable_set_nll":1.645705672725332,"acceptable_set_zero_mass_n":0},"calibration_all_labels":{"equal_width":{"n":43,"bin_count":10,"ece":0.0726993849804235,"signed_mean_gap":-0.0726993849804235,"absolute_mean_gap":0.0726993849804235,"bins":[{"index":0,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.0,"upper":0.1,"upper_inclusive":false},{"index":1,"count":22,"confidence":0.19288551333275708,"empirical_correctness":0.22727272727272727,"signed_gap":-0.034387213939970185,"absolute_gap":0.034387213939970185,"lower":0.1,"upper":0.2,"upper_inclusive":false},{"index":2,"count":21,"confidence":0.22049738821529208,"empirical_correctness":0.3333333333333333,"signed_gap":-0.11283594511804124,"absolute_gap":0.11283594511804124,"lower":0.2,"upper":0.3,"upper_inclusive":false},{"index":3,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.3,"upper":0.4,"upper_inclusive":false},{"index":4,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.4,"upper":0.5,"upper_inclusive":false},{"index":5,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.5,"upper":0.6,"upper_inclusive":false},{"index":6,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.6,"upper":0.7,"upper_inclusive":false},{"index":7,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.7,"upper":0.8,"upper_inclusive":false},{"index":8,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.8,"upper":0.9,"upper_inclusive":false},{"index":9,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.9,"upper":1.0,"upper_inclusive":true}]},"equal_mass":{"n":43,"bin_count":10,"ece":0.15453897971053457,"signed_mean_gap":-0.07269938498042351,"absolute_mean_gap":0.07269938498042351,"bins":[{"index":0,"count":4,"confidence":0.18829869106411934,"empirical_correctness":0.0,"signed_gap":0.18829869106411934,"absolute_gap":0.18829869106411934,"min_confidence":0.18784882128238678,"max_confidence":0.18898896872997284},{"index":1,"count":4,"confidence":0.18957415595650673,"empirical_correctness":0.25,"signed_gap":-0.06042584404349327,"absolute_gap":0.06042584404349327,"min_confidence":0.18899230659008026,"max_confidence":0.19063735008239746},{"index":2,"count":4,"confidence":0.19200028479099274,"empirical_correctness":0.25,"signed_gap":-0.05799971520900726,"absolute_gap":0.05799971520900726,"min_confidence":0.1910446584224701,"max_confidence":0.19286444783210754},{"index":3,"count":5,"confidence":0.19551002383232116,"empirical_correctness":0.2,"signed_gap":-0.00448997616767885,"absolute_gap":0.00448997616767885,"min_confidence":0.19416572153568268,"max_confidence":0.1960771381855011},{"index":4,"count":4,"confidence":0.19717298820614815,"empirical_correctness":0.5,"signed_gap":-0.30282701179385185,"absolute_gap":0.30282701179385185,"min_confidence":0.19613592326641083,"max_confidence":0.1977233588695526},{"index":5,"count":4,"confidence":0.20265549048781395,"empirical_correctness":0.0,"signed_gap":0.20265549048781395,"absolute_gap":0.20265549048781395,"min_confidence":0.19774669408798218,"max_confidence":0.20475715398788452},{"index":6,"count":5,"confidence":0.20935980081558228,"empirical_correctness":0.6,"signed_gap":-0.3906401991844177,"absolute_gap":0.3906401991844177,"min_confidence":0.2056279480457306,"max_confidence":0.21250075101852417},{"index":7,"count":4,"confidence":0.21541662141680717,"empirical_correctness":0.25,"signed_gap":-0.034583378583192825,"absolute_gap":0.034583378583192825,"min_confidence":0.21294169127941132,"max_confidence":0.2191661298274994},{"index":8,"count":4,"confidence":0.2283424586057663,"empirical_correctness":0.5,"signed_gap":-0.2716575413942337,"absolute_gap":0.2716575413942337,"min_confidence":0.22007358074188232,"max_confidence":0.2341099977493286},{"index":9,"count":5,"confidence":0.2391469120979309,"empirical_correctness":0.2,"signed_gap":0.03914691209793089,"absolute_gap":0.03914691209793089,"min_confidence":0.23547588288784027,"max_confidence":0.2411729246377945}]}},"calibration_singleton_labels":{"equal_width":{"n":38,"bin_count":10,"ece":0.10963377199674908,"signed_mean_gap":-0.10963377199674908,"absolute_mean_gap":0.10963377199674908,"bins":[{"index":0,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.0,"upper":0.1,"upper_inclusive":false},{"index":1,"count":19,"confidence":0.19258005132800654,"empirical_correctness":0.2631578947368421,"signed_gap":-0.07057784340883555,"absolute_gap":0.07057784340883555,"lower":0.1,"upper":0.2,"upper_inclusive":false},{"index":2,"count":19,"confidence":0.21973135204691635,"empirical_correctness":0.3684210526315789,"signed_gap":-0.14868970058466258,"absolute_gap":0.14868970058466258,"lower":0.2,"upper":0.3,"upper_inclusive":false},{"index":3,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.3,"upper":0.4,"upper_inclusive":false},{"index":4,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.4,"upper":0.5,"upper_inclusive":false},{"index":5,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.5,"upper":0.6,"upper_inclusive":false},{"index":6,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.6,"upper":0.7,"upper_inclusive":false},{"index":7,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.7,"upper":0.8,"upper_inclusive":false},{"index":8,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.8,"upper":0.9,"upper_inclusive":false},{"index":9,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.9,"upper":1.0,"upper_inclusive":true}]},"equal_mass":{"n":38,"bin_count":10,"ece":0.17158540691200055,"signed_mean_gap":-0.10963377199674908,"absolute_mean_gap":0.10963377199674908,"bins":[{"index":0,"count":3,"confidence":0.18806859850883484,"empirical_correctness":0.0,"signed_gap":0.18806859850883484,"absolute_gap":0.18806859850883484,"min_confidence":0.18784882128238678,"max_confidence":0.1882922500371933},{"index":1,"count":4,"confidence":0.18952574208378792,"empirical_correctness":0.25,"signed_gap":-0.06047425791621208,"absolute_gap":0.06047425791621208,"min_confidence":0.18898896872997284,"max_confidence":0.19063735008239746},{"index":2,"count":4,"confidence":0.19200028479099274,"empirical_correctness":0.25,"signed_gap":-0.05799971520900726,"absolute_gap":0.05799971520900726,"min_confidence":0.1910446584224701,"max_confidence":0.19286444783210754},{"index":3,"count":4,"confidence":0.19536824524402618,"empirical_correctness":0.25,"signed_gap":-0.054631754755973816,"absolute_gap":0.054631754755973816,"min_confidence":0.19416572153568268,"max_confidence":0.19602975249290466},{"index":4,"count":4,"confidence":0.19680952280759811,"empirical_correctness":0.5,"signed_gap":-0.3031904771924019,"absolute_gap":0.3031904771924019,"min_confidence":0.1960771381855011,"max_confidence":0.1977233588695526},{"index":5,"count":3,"confidence":0.20429175595442453,"empirical_correctness":0.0,"signed_gap":0.20429175595442453,"absolute_gap":0.20429175595442453,"min_confidence":0.20352256298065186,"max_confidence":0.20475715398788452},{"index":6,"count":4,"confidence":0.2085745632648468,"empirical_correctness":0.5,"signed_gap":-0.2914254367351532,"absolute_gap":0.2914254367351532,"min_confidence":0.2056279480457306,"max_confidence":0.21100904047489166},{"index":7,"count":4,"confidence":0.21375027671456337,"empirical_correctness":0.25,"signed_gap":-0.03624972328543663,"absolute_gap":0.03624972328543663,"min_confidence":0.21250075101852417,"max_confidence":0.21606436371803284},{"index":8,"count":4,"confidence":0.22811559587717056,"empirical_correctness":0.75,"signed_gap":-0.5218844041228294,"absolute_gap":0.5218844041228294,"min_confidence":0.2191661298274994,"max_confidence":0.2341099977493286},{"index":9,"count":4,"confidence":0.24006466940045357,"empirical_correctness":0.25,"signed_gap":-0.009935330599546432,"absolute_gap":0.009935330599546432,"min_confidence":0.23885492980480194,"max_confidence":0.2411729246377945}]}},"latency_ms":{"n":43,"median":101.08164796838537,"p95":102.49965802067891},"per_action":{"keep":{"n":12,"raw":{"n":12,"acceptable":12,"unsafe":0,"review":0,"automated_count":12,"automated_acceptable":12,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":1.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.7575059933447592,"upper":1.0,"confidence_level":0.95}},"effective":{"n":12,"acceptable":0,"unsafe":0,"review":12,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":12,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":2.7755575615628914e-17,"upper":0.24249400665524085,"confidence_level":0.95}}},"drop":{"n":16,"raw":{"n":16,"acceptable":0,"unsafe":14,"review":0,"automated_count":16,"automated_acceptable":0,"automated_unsafe":14,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":0.875,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":0.875,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.1936076805344365,"confidence_level":0.95}},"effective":{"n":16,"acceptable":0,"unsafe":0,"review":12,"automated_count":4,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":12,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":0.75,"automated_coverage":0.25,"conditional_correctness":0.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.75,"accuracy_wilson_95":{"lower":0.0,"upper":0.1936076805344365,"confidence_level":0.95}}},"mask":{"n":2,"raw":{"n":2,"acceptable":0,"unsafe":2,"review":0,"automated_count":2,"automated_acceptable":0,"automated_unsafe":2,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":1.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":1.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.6576197724933469,"confidence_level":0.95}},"effective":{"n":2,"acceptable":1,"unsafe":0,"review":1,"automated_count":1,"automated_acceptable":1,"automated_unsafe":0,"unnecessary_reviews":1,"accuracy":0.5,"unsafe_rate":0.0,"review_rate":0.5,"automated_coverage":0.5,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.5,"accuracy_wilson_95":{"lower":0.09453120573423074,"upper":0.9054687942657693,"confidence_level":0.95}}},"pseudonymize":{"n":2,"raw":{"n":2,"acceptable":0,"unsafe":0,"review":0,"automated_count":2,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.6576197724933469,"confidence_level":0.95}},"effective":{"n":2,"acceptable":0,"unsafe":0,"review":2,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":2,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.6576197724933469,"confidence_level":0.95}}},"generalize":{"n":6,"raw":{"n":6,"acceptable":0,"unsafe":4,"review":0,"automated_count":6,"automated_acceptable":0,"automated_unsafe":4,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":0.6666666666666666,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":0.6666666666666666,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":2.7755575615628914e-17,"upper":0.3903342879021653,"confidence_level":0.95}},"effective":{"n":6,"acceptable":0,"unsafe":0,"review":6,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":6,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":2.7755575615628914e-17,"upper":0.3903342879021653,"confidence_level":0.95}}},"review":{"n":0,"raw":{"n":0,"acceptable":0,"unsafe":0,"review":0,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":null,"unsafe_rate":null,"review_rate":null,"automated_coverage":null,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":null},"effective":{"n":0,"acceptable":0,"unsafe":0,"review":0,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":null,"unsafe_rate":null,"review_rate":null,"automated_coverage":null,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":null}}}},"diagnostic43-compact_no_samples":{"n":43,"raw":{"n":43,"acceptable":11,"unsafe":27,"review":0,"automated_count":43,"automated_acceptable":11,"automated_unsafe":27,"unnecessary_reviews":0,"accuracy":0.2558139534883721,"unsafe_rate":0.627906976744186,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.2558139534883721,"conditional_unsafe":0.627906976744186,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.14929517123808245,"upper":0.40238403498846675,"confidence_level":0.95}},"effective":{"n":43,"acceptable":6,"unsafe":0,"review":39,"automated_count":4,"automated_acceptable":1,"automated_unsafe":0,"unnecessary_reviews":34,"accuracy":0.13953488372093023,"unsafe_rate":0.0,"review_rate":0.9069767441860465,"automated_coverage":0.09302325581395349,"conditional_correctness":0.25,"conditional_unsafe":0.0,"unnecessary_review_rate":0.7906976744186046,"accuracy_wilson_95":{"lower":0.06555723427964878,"upper":0.2726358796738286,"confidence_level":0.95}},"confusion":{"raw":{"keep":{"keep":10,"drop":0,"mask":2,"pseudonymize":0,"generalize":0,"review":0},"drop":{"keep":12,"drop":0,"mask":3,"pseudonymize":1,"generalize":0,"review":0},"mask":{"keep":2,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0},"pseudonymize":{"keep":2,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0},"generalize":{"keep":6,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0},"review":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0}},"effective":{"keep":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":12},"drop":{"keep":0,"drop":0,"mask":3,"pseudonymize":0,"generalize":0,"review":13},"mask":{"keep":0,"drop":0,"mask":1,"pseudonymize":0,"generalize":0,"review":1},"pseudonymize":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":2},"generalize":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":6},"review":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0}}},"probability_metrics":{"singleton_n":38,"multiacceptable_n":5,"multiacceptable_coverage":0.11627906976744186,"nll":1.7853728044506791,"nll_zero_mass_n":0,"brier":0.8306359254962691,"acceptable_set_n":43,"acceptable_set_nll":1.6614522382692314,"acceptable_set_zero_mass_n":0},"calibration_all_labels":{"equal_width":{"n":43,"bin_count":10,"ece":0.04477873513864914,"signed_mean_gap":-0.04477873513864914,"absolute_mean_gap":0.04477873513864914,"bins":[{"index":0,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.0,"upper":0.1,"upper_inclusive":false},{"index":1,"count":5,"confidence":0.19512536525726318,"empirical_correctness":0.2,"signed_gap":-0.004874634742736833,"absolute_gap":0.004874634742736833,"lower":0.1,"upper":0.2,"upper_inclusive":false},{"index":2,"count":38,"confidence":0.213128620072415,"empirical_correctness":0.2631578947368421,"signed_gap":-0.05002927466442708,"absolute_gap":0.05002927466442708,"lower":0.2,"upper":0.3,"upper_inclusive":false},{"index":3,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.3,"upper":0.4,"upper_inclusive":false},{"index":4,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.4,"upper":0.5,"upper_inclusive":false},{"index":5,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.5,"upper":0.6,"upper_inclusive":false},{"index":6,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.6,"upper":0.7,"upper_inclusive":false},{"index":7,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.7,"upper":0.8,"upper_inclusive":false},{"index":8,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.8,"upper":0.9,"upper_inclusive":false},{"index":9,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.9,"upper":1.0,"upper_inclusive":true}]},"equal_mass":{"n":43,"bin_count":10,"ece":0.13210029657496963,"signed_mean_gap":-0.04477873513864916,"absolute_mean_gap":0.04477873513864916,"bins":[{"index":0,"count":4,"confidence":0.1941467858850956,"empirical_correctness":0.25,"signed_gap":-0.055853214114904404,"absolute_gap":0.055853214114904404,"min_confidence":0.18896915018558502,"max_confidence":0.19864286482334137},{"index":1,"count":4,"confidence":0.2002675086259842,"empirical_correctness":0.0,"signed_gap":0.2002675086259842,"absolute_gap":0.2002675086259842,"min_confidence":0.19903968274593353,"max_confidence":0.20126716792583466},{"index":2,"count":4,"confidence":0.20417383685708046,"empirical_correctness":0.5,"signed_gap":-0.29582616314291954,"absolute_gap":0.29582616314291954,"min_confidence":0.20360234379768372,"max_confidence":0.2046600729227066},{"index":3,"count":5,"confidence":0.2056768625974655,"empirical_correctness":0.2,"signed_gap":0.0056768625974654985,"absolute_gap":0.0056768625974654985,"min_confidence":0.20512473583221436,"max_confidence":0.20687460899353027},{"index":4,"count":4,"confidence":0.20760975033044815,"empirical_correctness":0.0,"signed_gap":0.20760975033044815,"absolute_gap":0.20760975033044815,"min_confidence":0.20698407292366028,"max_confidence":0.20823806524276733},{"index":5,"count":4,"confidence":0.20884136483073235,"empirical_correctness":0.25,"signed_gap":-0.041158635169267654,"absolute_gap":0.041158635169267654,"min_confidence":0.2086690217256546,"max_confidence":0.20913249254226685},{"index":6,"count":5,"confidence":0.21234120130538942,"empirical_correctness":0.2,"signed_gap":0.012341201305389404,"absolute_gap":0.012341201305389404,"min_confidence":0.21049822866916656,"max_confidence":0.21445204317569733},{"index":7,"count":4,"confidence":0.21901384368538857,"empirical_correctness":0.25,"signed_gap":-0.030986156314611435,"absolute_gap":0.030986156314611435,"min_confidence":0.21597695350646973,"max_confidence":0.22158706188201904},{"index":8,"count":4,"confidence":0.22309937328100204,"empirical_correctness":0.75,"signed_gap":-0.526900626718998,"absolute_gap":0.526900626718998,"min_confidence":0.22172974050045013,"max_confidence":0.22576700150966644},{"index":9,"count":5,"confidence":0.2311628431081772,"empirical_correctness":0.2,"signed_gap":0.031162843108177185,"absolute_gap":0.031162843108177185,"min_confidence":0.2289382517337799,"max_confidence":0.23552961647510529}]}},"calibration_singleton_labels":{"equal_width":{"n":38,"bin_count":10,"ece":0.05212992037597453,"signed_mean_gap":-0.05212992037597453,"absolute_mean_gap":0.05212992037597453,"bins":[{"index":0,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.0,"upper":0.1,"upper_inclusive":false},{"index":1,"count":5,"confidence":0.19512536525726318,"empirical_correctness":0.2,"signed_gap":-0.004874634742736833,"absolute_gap":0.004874634742736833,"lower":0.1,"upper":0.2,"upper_inclusive":false},{"index":2,"count":33,"confidence":0.2134374605886864,"empirical_correctness":0.2727272727272727,"signed_gap":-0.059289812138586306,"absolute_gap":0.059289812138586306,"lower":0.2,"upper":0.3,"upper_inclusive":false},{"index":3,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.3,"upper":0.4,"upper_inclusive":false},{"index":4,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.4,"upper":0.5,"upper_inclusive":false},{"index":5,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.5,"upper":0.6,"upper_inclusive":false},{"index":6,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.6,"upper":0.7,"upper_inclusive":false},{"index":7,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.7,"upper":0.8,"upper_inclusive":false},{"index":8,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.8,"upper":0.9,"upper_inclusive":false},{"index":9,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.9,"upper":1.0,"upper_inclusive":true}]},"equal_mass":{"n":38,"bin_count":10,"ece":0.186717857655726,"signed_mean_gap":-0.05212992037597455,"absolute_mean_gap":0.05212992037597455,"bins":[{"index":0,"count":3,"confidence":0.19264809290568033,"empirical_correctness":0.3333333333333333,"signed_gap":-0.14068524042765299,"absolute_gap":0.14068524042765299,"min_confidence":0.18896915018558502,"max_confidence":0.19739368557929993},{"index":1,"count":4,"confidence":0.19961143285036087,"empirical_correctness":0.0,"signed_gap":0.19961143285036087,"absolute_gap":0.19961143285036087,"min_confidence":0.19864286482334137,"max_confidence":0.2007497102022171},{"index":2,"count":4,"confidence":0.2033715769648552,"empirical_correctness":0.25,"signed_gap":-0.046628423035144806,"absolute_gap":0.046628423035144806,"min_confidence":0.20126716792583466,"max_confidence":0.2046600729227066},{"index":3,"count":4,"confidence":0.20578985661268234,"empirical_correctness":0.25,"signed_gap":-0.04421014338731766,"absolute_gap":0.04421014338731766,"min_confidence":0.20512473583221436,"max_confidence":0.20687460899353027},{"index":4,"count":4,"confidence":0.2079622782766819,"empirical_correctness":0.0,"signed_gap":0.2079622782766819,"absolute_gap":0.2079622782766819,"min_confidence":0.20698407292366028,"max_confidence":0.2086690217256546},{"index":5,"count":3,"confidence":0.20889881253242493,"empirical_correctness":0.3333333333333333,"signed_gap":-0.12443452080090839,"absolute_gap":0.12443452080090839,"min_confidence":0.2086840569972992,"max_confidence":0.20913249254226685},{"index":6,"count":4,"confidence":0.21181349083781242,"empirical_correctness":0.25,"signed_gap":-0.038186509162187576,"absolute_gap":0.038186509162187576,"min_confidence":0.21049822866916656,"max_confidence":0.2141648381948471},{"index":7,"count":4,"confidence":0.21843644976615906,"empirical_correctness":0.5,"signed_gap":-0.28156355023384094,"absolute_gap":0.28156355023384094,"min_confidence":0.21445204317569733,"max_confidence":0.22172974050045013},{"index":8,"count":4,"confidence":0.2249015010893345,"empirical_correctness":0.75,"signed_gap":-0.5250984989106655,"absolute_gap":0.5250984989106655,"min_confidence":0.2221742421388626,"max_confidence":0.2289382517337799},{"index":9,"count":4,"confidence":0.2317189909517765,"empirical_correctness":0.0,"signed_gap":0.2317189909517765,"absolute_gap":0.2317189909517765,"min_confidence":0.22965097427368164,"max_confidence":0.23552961647510529}]}},"latency_ms":{"n":43,"median":102.02593699796125,"p95":103.04023697972298},"per_action":{"keep":{"n":12,"raw":{"n":12,"acceptable":10,"unsafe":0,"review":0,"automated_count":12,"automated_acceptable":10,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":0.8333333333333334,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.8333333333333334,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.5519691377470266,"upper":0.9530348578161463,"confidence_level":0.95}},"effective":{"n":12,"acceptable":0,"unsafe":0,"review":12,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":12,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":2.7755575615628914e-17,"upper":0.24249400665524085,"confidence_level":0.95}}},"drop":{"n":16,"raw":{"n":16,"acceptable":0,"unsafe":13,"review":0,"automated_count":16,"automated_acceptable":0,"automated_unsafe":13,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":0.8125,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":0.8125,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.1936076805344365,"confidence_level":0.95}},"effective":{"n":16,"acceptable":0,"unsafe":0,"review":13,"automated_count":3,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":13,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":0.8125,"automated_coverage":0.1875,"conditional_correctness":0.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.8125,"accuracy_wilson_95":{"lower":0.0,"upper":0.1936076805344365,"confidence_level":0.95}}},"mask":{"n":2,"raw":{"n":2,"acceptable":0,"unsafe":2,"review":0,"automated_count":2,"automated_acceptable":0,"automated_unsafe":2,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":1.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":1.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.6576197724933469,"confidence_level":0.95}},"effective":{"n":2,"acceptable":1,"unsafe":0,"review":1,"automated_count":1,"automated_acceptable":1,"automated_unsafe":0,"unnecessary_reviews":1,"accuracy":0.5,"unsafe_rate":0.0,"review_rate":0.5,"automated_coverage":0.5,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.5,"accuracy_wilson_95":{"lower":0.09453120573423074,"upper":0.9054687942657693,"confidence_level":0.95}}},"pseudonymize":{"n":2,"raw":{"n":2,"acceptable":0,"unsafe":2,"review":0,"automated_count":2,"automated_acceptable":0,"automated_unsafe":2,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":1.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":1.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.6576197724933469,"confidence_level":0.95}},"effective":{"n":2,"acceptable":0,"unsafe":0,"review":2,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":2,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.6576197724933469,"confidence_level":0.95}}},"generalize":{"n":6,"raw":{"n":6,"acceptable":0,"unsafe":6,"review":0,"automated_count":6,"automated_acceptable":0,"automated_unsafe":6,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":1.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":1.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":2.7755575615628914e-17,"upper":0.3903342879021653,"confidence_level":0.95}},"effective":{"n":6,"acceptable":0,"unsafe":0,"review":6,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":6,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":2.7755575615628914e-17,"upper":0.3903342879021653,"confidence_level":0.95}}},"review":{"n":0,"raw":{"n":0,"acceptable":0,"unsafe":0,"review":0,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":null,"unsafe_rate":null,"review_rate":null,"automated_coverage":null,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":null},"effective":{"n":0,"acceptable":0,"unsafe":0,"review":0,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":null,"unsafe_rate":null,"review_rate":null,"automated_coverage":null,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":null}}}},"challenge56-no_samples":{"n":56,"raw":{"n":56,"acceptable":5,"unsafe":33,"review":1,"automated_count":55,"automated_acceptable":5,"automated_unsafe":33,"unnecessary_reviews":1,"accuracy":0.08928571428571429,"unsafe_rate":0.5892857142857143,"review_rate":0.017857142857142856,"automated_coverage":0.9821428571428571,"conditional_correctness":0.09090909090909091,"conditional_unsafe":0.6,"unnecessary_review_rate":0.017857142857142856,"accuracy_wilson_95":{"lower":0.03874214844958693,"upper":0.19256001385511162,"confidence_level":0.95}},"effective":{"n":56,"acceptable":2,"unsafe":0,"review":56,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":54,"accuracy":0.03571428571428571,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":0.9642857142857143,"accuracy_wilson_95":{"lower":0.00984942514823639,"upper":0.12118780180490107,"confidence_level":0.95}},"confusion":{"raw":{"keep":{"keep":5,"drop":0,"mask":1,"pseudonymize":2,"generalize":0,"review":0},"drop":{"keep":13,"drop":0,"mask":9,"pseudonymize":0,"generalize":0,"review":0},"mask":{"keep":6,"drop":0,"mask":0,"pseudonymize":2,"generalize":0,"review":0},"pseudonymize":{"keep":5,"drop":0,"mask":2,"pseudonymize":0,"generalize":0,"review":1},"generalize":{"keep":6,"drop":0,"mask":2,"pseudonymize":0,"generalize":0,"review":0},"review":{"keep":1,"drop":0,"mask":1,"pseudonymize":0,"generalize":0,"review":0}},"effective":{"keep":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":8},"drop":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":22},"mask":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":8},"pseudonymize":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":8},"generalize":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":8},"review":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":2}}},"probability_metrics":{"singleton_n":56,"multiacceptable_n":0,"multiacceptable_coverage":0.0,"nll":1.9004277114554218,"nll_zero_mass_n":0,"brier":0.8684787664705416,"acceptable_set_n":56,"acceptable_set_nll":1.9004277114554218,"acceptable_set_zero_mass_n":0},"calibration_all_labels":{"equal_width":{"n":56,"bin_count":10,"ece":0.12471720975424562,"signed_mean_gap":0.12471720975424562,"absolute_mean_gap":0.12471720975424562,"bins":[{"index":0,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.0,"upper":0.1,"upper_inclusive":false},{"index":1,"count":5,"confidence":0.19176367223262786,"empirical_correctness":0.0,"signed_gap":0.19176367223262786,"absolute_gap":0.19176367223262786,"lower":0.1,"upper":0.2,"upper_inclusive":false},{"index":2,"count":51,"confidence":0.2161832428446003,"empirical_correctness":0.09803921568627451,"signed_gap":0.1181440271583258,"absolute_gap":0.1181440271583258,"lower":0.2,"upper":0.3,"upper_inclusive":false},{"index":3,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.3,"upper":0.4,"upper_inclusive":false},{"index":4,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.4,"upper":0.5,"upper_inclusive":false},{"index":5,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.5,"upper":0.6,"upper_inclusive":false},{"index":6,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.6,"upper":0.7,"upper_inclusive":false},{"index":7,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.7,"upper":0.8,"upper_inclusive":false},{"index":8,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.8,"upper":0.9,"upper_inclusive":false},{"index":9,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.9,"upper":1.0,"upper_inclusive":true}]},"equal_mass":{"n":56,"bin_count":10,"ece":0.12471720975424562,"signed_mean_gap":0.12471720975424562,"absolute_mean_gap":0.12471720975424562,"bins":[{"index":0,"count":5,"confidence":0.19176367223262786,"empirical_correctness":0.0,"signed_gap":0.19176367223262786,"absolute_gap":0.19176367223262786,"min_confidence":0.18822772800922394,"max_confidence":0.1953076273202896},{"index":1,"count":6,"confidence":0.20198343694210052,"empirical_correctness":0.16666666666666666,"signed_gap":0.03531677027543387,"absolute_gap":0.03531677027543387,"min_confidence":0.2005750685930252,"max_confidence":0.20330668985843658},{"index":2,"count":5,"confidence":0.20427553057670594,"empirical_correctness":0.0,"signed_gap":0.20427553057670594,"absolute_gap":0.20427553057670594,"min_confidence":0.20370030403137207,"max_confidence":0.20491966605186462},{"index":3,"count":6,"confidence":0.2067397733529409,"empirical_correctness":0.0,"signed_gap":0.2067397733529409,"absolute_gap":0.2067397733529409,"min_confidence":0.2050374299287796,"max_confidence":0.2081526815891266},{"index":4,"count":6,"confidence":0.20977818965911865,"empirical_correctness":0.16666666666666666,"signed_gap":0.043111522992451995,"absolute_gap":0.043111522992451995,"min_confidence":0.2082863599061966,"max_confidence":0.21122631430625916},{"index":5,"count":5,"confidence":0.21292338371276856,"empirical_correctness":0.0,"signed_gap":0.21292338371276856,"absolute_gap":0.21292338371276856,"min_confidence":0.21210718154907227,"max_confidence":0.21392489969730377},{"index":6,"count":6,"confidence":0.2156489466627439,"empirical_correctness":0.0,"signed_gap":0.2156489466627439,"absolute_gap":0.2156489466627439,"min_confidence":0.2148890346288681,"max_confidence":0.21680636703968048},{"index":7,"count":5,"confidence":0.22259435951709747,"empirical_correctness":0.2,"signed_gap":0.022594359517097456,"absolute_gap":0.022594359517097456,"min_confidence":0.21724151074886322,"max_confidence":0.2263011932373047},{"index":8,"count":6,"confidence":0.22868009905020395,"empirical_correctness":0.16666666666666666,"signed_gap":0.06201343238353729,"absolute_gap":0.06201343238353729,"min_confidence":0.22745519876480103,"max_confidence":0.23031337559223175},{"index":9,"count":6,"confidence":0.24156605700651804,"empirical_correctness":0.16666666666666666,"signed_gap":0.07489939033985138,"absolute_gap":0.07489939033985138,"min_confidence":0.23148952424526215,"max_confidence":0.2598181962966919}]}},"calibration_singleton_labels":{"equal_width":{"n":56,"bin_count":10,"ece":0.12471720975424562,"signed_mean_gap":0.12471720975424562,"absolute_mean_gap":0.12471720975424562,"bins":[{"index":0,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.0,"upper":0.1,"upper_inclusive":false},{"index":1,"count":5,"confidence":0.19176367223262786,"empirical_correctness":0.0,"signed_gap":0.19176367223262786,"absolute_gap":0.19176367223262786,"lower":0.1,"upper":0.2,"upper_inclusive":false},{"index":2,"count":51,"confidence":0.2161832428446003,"empirical_correctness":0.09803921568627451,"signed_gap":0.1181440271583258,"absolute_gap":0.1181440271583258,"lower":0.2,"upper":0.3,"upper_inclusive":false},{"index":3,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.3,"upper":0.4,"upper_inclusive":false},{"index":4,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.4,"upper":0.5,"upper_inclusive":false},{"index":5,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.5,"upper":0.6,"upper_inclusive":false},{"index":6,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.6,"upper":0.7,"upper_inclusive":false},{"index":7,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.7,"upper":0.8,"upper_inclusive":false},{"index":8,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.8,"upper":0.9,"upper_inclusive":false},{"index":9,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.9,"upper":1.0,"upper_inclusive":true}]},"equal_mass":{"n":56,"bin_count":10,"ece":0.12471720975424562,"signed_mean_gap":0.12471720975424562,"absolute_mean_gap":0.12471720975424562,"bins":[{"index":0,"count":5,"confidence":0.19176367223262786,"empirical_correctness":0.0,"signed_gap":0.19176367223262786,"absolute_gap":0.19176367223262786,"min_confidence":0.18822772800922394,"max_confidence":0.1953076273202896},{"index":1,"count":6,"confidence":0.20198343694210052,"empirical_correctness":0.16666666666666666,"signed_gap":0.03531677027543387,"absolute_gap":0.03531677027543387,"min_confidence":0.2005750685930252,"max_confidence":0.20330668985843658},{"index":2,"count":5,"confidence":0.20427553057670594,"empirical_correctness":0.0,"signed_gap":0.20427553057670594,"absolute_gap":0.20427553057670594,"min_confidence":0.20370030403137207,"max_confidence":0.20491966605186462},{"index":3,"count":6,"confidence":0.2067397733529409,"empirical_correctness":0.0,"signed_gap":0.2067397733529409,"absolute_gap":0.2067397733529409,"min_confidence":0.2050374299287796,"max_confidence":0.2081526815891266},{"index":4,"count":6,"confidence":0.20977818965911865,"empirical_correctness":0.16666666666666666,"signed_gap":0.043111522992451995,"absolute_gap":0.043111522992451995,"min_confidence":0.2082863599061966,"max_confidence":0.21122631430625916},{"index":5,"count":5,"confidence":0.21292338371276856,"empirical_correctness":0.0,"signed_gap":0.21292338371276856,"absolute_gap":0.21292338371276856,"min_confidence":0.21210718154907227,"max_confidence":0.21392489969730377},{"index":6,"count":6,"confidence":0.2156489466627439,"empirical_correctness":0.0,"signed_gap":0.2156489466627439,"absolute_gap":0.2156489466627439,"min_confidence":0.2148890346288681,"max_confidence":0.21680636703968048},{"index":7,"count":5,"confidence":0.22259435951709747,"empirical_correctness":0.2,"signed_gap":0.022594359517097456,"absolute_gap":0.022594359517097456,"min_confidence":0.21724151074886322,"max_confidence":0.2263011932373047},{"index":8,"count":6,"confidence":0.22868009905020395,"empirical_correctness":0.16666666666666666,"signed_gap":0.06201343238353729,"absolute_gap":0.06201343238353729,"min_confidence":0.22745519876480103,"max_confidence":0.23031337559223175},{"index":9,"count":6,"confidence":0.24156605700651804,"empirical_correctness":0.16666666666666666,"signed_gap":0.07489939033985138,"absolute_gap":0.07489939033985138,"min_confidence":0.23148952424526215,"max_confidence":0.2598181962966919}]}},"latency_ms":{"n":56,"median":92.64647102099843,"p95":93.68843998527154},"per_action":{"keep":{"n":8,"raw":{"n":8,"acceptable":5,"unsafe":0,"review":0,"automated_count":8,"automated_acceptable":5,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":0.625,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.625,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.30574239460262737,"upper":0.8631557141764026,"confidence_level":0.95}},"effective":{"n":8,"acceptable":0,"unsafe":0,"review":8,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":8,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.32440756488388023,"confidence_level":0.95}}},"drop":{"n":22,"raw":{"n":22,"acceptable":0,"unsafe":13,"review":0,"automated_count":22,"automated_acceptable":0,"automated_unsafe":13,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":0.5909090909090909,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":0.5909090909090909,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.14865487460862087,"confidence_level":0.95}},"effective":{"n":22,"acceptable":0,"unsafe":0,"review":22,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":22,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.14865487460862087,"confidence_level":0.95}}},"mask":{"n":8,"raw":{"n":8,"acceptable":0,"unsafe":8,"review":0,"automated_count":8,"automated_acceptable":0,"automated_unsafe":8,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":1.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":1.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.32440756488388023,"confidence_level":0.95}},"effective":{"n":8,"acceptable":0,"unsafe":0,"review":8,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":8,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.32440756488388023,"confidence_level":0.95}}},"pseudonymize":{"n":8,"raw":{"n":8,"acceptable":0,"unsafe":5,"review":1,"automated_count":7,"automated_acceptable":0,"automated_unsafe":5,"unnecessary_reviews":1,"accuracy":0.0,"unsafe_rate":0.625,"review_rate":0.125,"automated_coverage":0.875,"conditional_correctness":0.0,"conditional_unsafe":0.7142857142857143,"unnecessary_review_rate":0.125,"accuracy_wilson_95":{"lower":0.0,"upper":0.32440756488388023,"confidence_level":0.95}},"effective":{"n":8,"acceptable":0,"unsafe":0,"review":8,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":8,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.32440756488388023,"confidence_level":0.95}}},"generalize":{"n":8,"raw":{"n":8,"acceptable":0,"unsafe":6,"review":0,"automated_count":8,"automated_acceptable":0,"automated_unsafe":6,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":0.75,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":0.75,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.32440756488388023,"confidence_level":0.95}},"effective":{"n":8,"acceptable":0,"unsafe":0,"review":8,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":8,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.32440756488388023,"confidence_level":0.95}}},"review":{"n":2,"raw":{"n":2,"acceptable":0,"unsafe":1,"review":0,"automated_count":2,"automated_acceptable":0,"automated_unsafe":1,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":0.5,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":0.5,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.6576197724933469,"confidence_level":0.95}},"effective":{"n":2,"acceptable":2,"unsafe":0,"review":2,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":1.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.34238022750665303,"upper":1.0,"confidence_level":0.95}}}}},"challenge56-compact_no_samples":{"n":56,"raw":{"n":56,"acceptable":3,"unsafe":26,"review":1,"automated_count":55,"automated_acceptable":3,"automated_unsafe":26,"unnecessary_reviews":1,"accuracy":0.05357142857142857,"unsafe_rate":0.4642857142857143,"review_rate":0.017857142857142856,"automated_coverage":0.9821428571428571,"conditional_correctness":0.05454545454545454,"conditional_unsafe":0.4727272727272727,"unnecessary_review_rate":0.017857142857142856,"accuracy_wilson_95":{"lower":0.018385781382109126,"upper":0.14607309068821533,"confidence_level":0.95}},"effective":{"n":56,"acceptable":2,"unsafe":0,"review":56,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":54,"accuracy":0.03571428571428571,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":0.9642857142857143,"accuracy_wilson_95":{"lower":0.00984942514823639,"upper":0.12118780180490107,"confidence_level":0.95}},"confusion":{"raw":{"keep":{"keep":2,"drop":0,"mask":6,"pseudonymize":0,"generalize":0,"review":0},"drop":{"keep":10,"drop":0,"mask":12,"pseudonymize":0,"generalize":0,"review":0},"mask":{"keep":6,"drop":0,"mask":1,"pseudonymize":0,"generalize":0,"review":1},"pseudonymize":{"keep":4,"drop":0,"mask":4,"pseudonymize":0,"generalize":0,"review":0},"generalize":{"keep":4,"drop":0,"mask":4,"pseudonymize":0,"generalize":0,"review":0},"review":{"keep":2,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0}},"effective":{"keep":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":8},"drop":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":22},"mask":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":8},"pseudonymize":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":8},"generalize":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":8},"review":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":2}}},"probability_metrics":{"singleton_n":56,"multiacceptable_n":0,"multiacceptable_coverage":0.0,"nll":1.8453122247329794,"nll_zero_mass_n":0,"brier":0.8520378990727119,"acceptable_set_n":56,"acceptable_set_nll":1.8453122247329794,"acceptable_set_zero_mass_n":0},"calibration_all_labels":{"equal_width":{"n":56,"bin_count":10,"ece":0.16214338902916228,"signed_mean_gap":0.16134619872484887,"absolute_mean_gap":0.16134619872484887,"bins":[{"index":0,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.0,"upper":0.1,"upper_inclusive":false},{"index":1,"count":5,"confidence":0.19553573429584503,"empirical_correctness":0.2,"signed_gap":-0.004464265704154979,"absolute_gap":0.004464265704154979,"lower":0.1,"upper":0.2,"upper_inclusive":false},{"index":2,"count":51,"confidence":0.21681781288455515,"empirical_correctness":0.0392156862745098,"signed_gap":0.17760212661004535,"absolute_gap":0.17760212661004535,"lower":0.2,"upper":0.3,"upper_inclusive":false},{"index":3,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.3,"upper":0.4,"upper_inclusive":false},{"index":4,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.4,"upper":0.5,"upper_inclusive":false},{"index":5,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.5,"upper":0.6,"upper_inclusive":false},{"index":6,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.6,"upper":0.7,"upper_inclusive":false},{"index":7,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.7,"upper":0.8,"upper_inclusive":false},{"index":8,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.8,"upper":0.9,"upper_inclusive":false},{"index":9,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.9,"upper":1.0,"upper_inclusive":true}]},"equal_mass":{"n":56,"bin_count":10,"ece":0.16214338902916228,"signed_mean_gap":0.16134619872484887,"absolute_mean_gap":0.16134619872484887,"bins":[{"index":0,"count":5,"confidence":0.19553573429584503,"empirical_correctness":0.2,"signed_gap":-0.004464265704154979,"absolute_gap":0.004464265704154979,"min_confidence":0.1904963254928589,"max_confidence":0.19951370358467102},{"index":1,"count":6,"confidence":0.20317169278860092,"empirical_correctness":0.0,"signed_gap":0.20317169278860092,"absolute_gap":0.20317169278860092,"min_confidence":0.2018505334854126,"max_confidence":0.2037724256515503},{"index":2,"count":5,"confidence":0.2052905172109604,"empirical_correctness":0.0,"signed_gap":0.2052905172109604,"absolute_gap":0.2052905172109604,"min_confidence":0.2037837654352188,"max_confidence":0.20696993172168732},{"index":3,"count":6,"confidence":0.2083556925257047,"empirical_correctness":0.0,"signed_gap":0.2083556925257047,"absolute_gap":0.2083556925257047,"min_confidence":0.20727050304412842,"max_confidence":0.20989613234996796},{"index":4,"count":6,"confidence":0.2128220001856486,"empirical_correctness":0.16666666666666666,"signed_gap":0.046155333518981934,"absolute_gap":0.046155333518981934,"min_confidence":0.21068306267261505,"max_confidence":0.21477286517620087},{"index":5,"count":5,"confidence":0.21643634140491486,"empirical_correctness":0.0,"signed_gap":0.21643634140491486,"absolute_gap":0.21643634140491486,"min_confidence":0.21499064564704895,"max_confidence":0.2176080197095871},{"index":6,"count":6,"confidence":0.21863829096158346,"empirical_correctness":0.0,"signed_gap":0.21863829096158346,"absolute_gap":0.21863829096158346,"min_confidence":0.2179182767868042,"max_confidence":0.22045476734638214},{"index":7,"count":5,"confidence":0.22215285301208496,"empirical_correctness":0.0,"signed_gap":0.22215285301208496,"absolute_gap":0.22215285301208496,"min_confidence":0.2205636203289032,"max_confidence":0.22451229393482208},{"index":8,"count":6,"confidence":0.2282173087199529,"empirical_correctness":0.16666666666666666,"signed_gap":0.06155064205328625,"absolute_gap":0.06155064205328625,"min_confidence":0.2256699502468109,"max_confidence":0.23126931488513947},{"index":9,"count":6,"confidence":0.23517999798059464,"empirical_correctness":0.0,"signed_gap":0.23517999798059464,"absolute_gap":0.23517999798059464,"min_confidence":0.23155462741851807,"max_confidence":0.24272370338439941}]}},"calibration_singleton_labels":{"equal_width":{"n":56,"bin_count":10,"ece":0.16214338902916228,"signed_mean_gap":0.16134619872484887,"absolute_mean_gap":0.16134619872484887,"bins":[{"index":0,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.0,"upper":0.1,"upper_inclusive":false},{"index":1,"count":5,"confidence":0.19553573429584503,"empirical_correctness":0.2,"signed_gap":-0.004464265704154979,"absolute_gap":0.004464265704154979,"lower":0.1,"upper":0.2,"upper_inclusive":false},{"index":2,"count":51,"confidence":0.21681781288455515,"empirical_correctness":0.0392156862745098,"signed_gap":0.17760212661004535,"absolute_gap":0.17760212661004535,"lower":0.2,"upper":0.3,"upper_inclusive":false},{"index":3,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.3,"upper":0.4,"upper_inclusive":false},{"index":4,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.4,"upper":0.5,"upper_inclusive":false},{"index":5,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.5,"upper":0.6,"upper_inclusive":false},{"index":6,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.6,"upper":0.7,"upper_inclusive":false},{"index":7,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.7,"upper":0.8,"upper_inclusive":false},{"index":8,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.8,"upper":0.9,"upper_inclusive":false},{"index":9,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.9,"upper":1.0,"upper_inclusive":true}]},"equal_mass":{"n":56,"bin_count":10,"ece":0.16214338902916228,"signed_mean_gap":0.16134619872484887,"absolute_mean_gap":0.16134619872484887,"bins":[{"index":0,"count":5,"confidence":0.19553573429584503,"empirical_correctness":0.2,"signed_gap":-0.004464265704154979,"absolute_gap":0.004464265704154979,"min_confidence":0.1904963254928589,"max_confidence":0.19951370358467102},{"index":1,"count":6,"confidence":0.20317169278860092,"empirical_correctness":0.0,"signed_gap":0.20317169278860092,"absolute_gap":0.20317169278860092,"min_confidence":0.2018505334854126,"max_confidence":0.2037724256515503},{"index":2,"count":5,"confidence":0.2052905172109604,"empirical_correctness":0.0,"signed_gap":0.2052905172109604,"absolute_gap":0.2052905172109604,"min_confidence":0.2037837654352188,"max_confidence":0.20696993172168732},{"index":3,"count":6,"confidence":0.2083556925257047,"empirical_correctness":0.0,"signed_gap":0.2083556925257047,"absolute_gap":0.2083556925257047,"min_confidence":0.20727050304412842,"max_confidence":0.20989613234996796},{"index":4,"count":6,"confidence":0.2128220001856486,"empirical_correctness":0.16666666666666666,"signed_gap":0.046155333518981934,"absolute_gap":0.046155333518981934,"min_confidence":0.21068306267261505,"max_confidence":0.21477286517620087},{"index":5,"count":5,"confidence":0.21643634140491486,"empirical_correctness":0.0,"signed_gap":0.21643634140491486,"absolute_gap":0.21643634140491486,"min_confidence":0.21499064564704895,"max_confidence":0.2176080197095871},{"index":6,"count":6,"confidence":0.21863829096158346,"empirical_correctness":0.0,"signed_gap":0.21863829096158346,"absolute_gap":0.21863829096158346,"min_confidence":0.2179182767868042,"max_confidence":0.22045476734638214},{"index":7,"count":5,"confidence":0.22215285301208496,"empirical_correctness":0.0,"signed_gap":0.22215285301208496,"absolute_gap":0.22215285301208496,"min_confidence":0.2205636203289032,"max_confidence":0.22451229393482208},{"index":8,"count":6,"confidence":0.2282173087199529,"empirical_correctness":0.16666666666666666,"signed_gap":0.06155064205328625,"absolute_gap":0.06155064205328625,"min_confidence":0.2256699502468109,"max_confidence":0.23126931488513947},{"index":9,"count":6,"confidence":0.23517999798059464,"empirical_correctness":0.0,"signed_gap":0.23517999798059464,"absolute_gap":0.23517999798059464,"min_confidence":0.23155462741851807,"max_confidence":0.24272370338439941}]}},"latency_ms":{"n":56,"median":93.55202497681603,"p95":95.02092101320159},"per_action":{"keep":{"n":8,"raw":{"n":8,"acceptable":2,"unsafe":0,"review":0,"automated_count":8,"automated_acceptable":2,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":0.25,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.25,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.071479212752109,"upper":0.5907245696898311,"confidence_level":0.95}},"effective":{"n":8,"acceptable":0,"unsafe":0,"review":8,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":8,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.32440756488388023,"confidence_level":0.95}}},"drop":{"n":22,"raw":{"n":22,"acceptable":0,"unsafe":10,"review":0,"automated_count":22,"automated_acceptable":0,"automated_unsafe":10,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":0.45454545454545453,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":0.45454545454545453,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.14865487460862087,"confidence_level":0.95}},"effective":{"n":22,"acceptable":0,"unsafe":0,"review":22,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":22,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.14865487460862087,"confidence_level":0.95}}},"mask":{"n":8,"raw":{"n":8,"acceptable":1,"unsafe":6,"review":1,"automated_count":7,"automated_acceptable":1,"automated_unsafe":6,"unnecessary_reviews":1,"accuracy":0.125,"unsafe_rate":0.75,"review_rate":0.125,"automated_coverage":0.875,"conditional_correctness":0.14285714285714285,"conditional_unsafe":0.8571428571428571,"unnecessary_review_rate":0.125,"accuracy_wilson_95":{"lower":0.02241749145005667,"upper":0.4708881822128535,"confidence_level":0.95}},"effective":{"n":8,"acceptable":0,"unsafe":0,"review":8,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":8,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.32440756488388023,"confidence_level":0.95}}},"pseudonymize":{"n":8,"raw":{"n":8,"acceptable":0,"unsafe":4,"review":0,"automated_count":8,"automated_acceptable":0,"automated_unsafe":4,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":0.5,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":0.5,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.32440756488388023,"confidence_level":0.95}},"effective":{"n":8,"acceptable":0,"unsafe":0,"review":8,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":8,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.32440756488388023,"confidence_level":0.95}}},"generalize":{"n":8,"raw":{"n":8,"acceptable":0,"unsafe":4,"review":0,"automated_count":8,"automated_acceptable":0,"automated_unsafe":4,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":0.5,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":0.5,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.32440756488388023,"confidence_level":0.95}},"effective":{"n":8,"acceptable":0,"unsafe":0,"review":8,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":8,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.32440756488388023,"confidence_level":0.95}}},"review":{"n":2,"raw":{"n":2,"acceptable":0,"unsafe":2,"review":0,"automated_count":2,"automated_acceptable":0,"automated_unsafe":2,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":1.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":1.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.6576197724933469,"confidence_level":0.95}},"effective":{"n":2,"acceptable":2,"unsafe":0,"review":2,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":1.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.34238022750665303,"upper":1.0,"confidence_level":0.95}}}}}},"composite":{"n":198,"scope":"descriptive sum across four correlated panels","raw":{"n":198,"acceptable":31,"unsafe":111,"review":2,"automated_count":196,"automated_acceptable":31,"automated_unsafe":111,"unnecessary_reviews":2,"accuracy":0.15656565656565657,"unsafe_rate":0.5606060606060606,"review_rate":0.010101010101010102,"automated_coverage":0.98989898989899,"conditional_correctness":0.15816326530612246,"conditional_unsafe":0.5663265306122449,"unnecessary_review_rate":0.010101010101010102},"effective":{"n":198,"acceptable":16,"unsafe":0,"review":189,"automated_count":9,"automated_acceptable":2,"automated_unsafe":0,"unnecessary_reviews":175,"accuracy":0.08080808080808081,"unsafe_rate":0.0,"review_rate":0.9545454545454546,"automated_coverage":0.045454545454545456,"conditional_correctness":0.2222222222222222,"conditional_unsafe":0.0,"unnecessary_review_rate":0.8838383838383839}}},{"id":"JoshuaAshkinaze--policyfx-nli","platform":"Local","model_id":"JoshuaAshkinaze/policyfx-nli","revision":"29cf6857e2fa9a76798e1577cb6e1b4842e570d3","loading":"float32 / none","scoring_kind":"Adapted NLI","settings":{"dtype":"float32","quantization":null,"max_tokens":4096,"effective_max_tokens":2048,"temperature":1.0,"scoring_batch_size":1,"trust_remote_code":false,"local_files_only":true,"use_safetensors":true,"guard_threshold":0.45,"seed":0,"tf32":false},"scoring_metadata":{"backend":"nli","protocol":"hf-local-sweep-backends-v1","probability_source":"adapted_entailment_softmax_not_calibrated_sixway","latency_scope":"synchronous_forward_scoring_including_transfer_excluding_tokenization","precision":"float32","equal_mass_convention":"rank bins with floor boundaries at floor(i*n/k), k=min(10,n//2); minimum two rows"},"load_ms":1460.4647539672442,"peak_allocated_bytes":665745408,"peak_reserved_bytes":681574400,"panels":{"diagnostic43-no_samples":{"n":43,"raw":{"n":43,"acceptable":12,"unsafe":31,"review":0,"automated_count":43,"automated_acceptable":12,"automated_unsafe":31,"unnecessary_reviews":0,"accuracy":0.27906976744186046,"unsafe_rate":0.7209302325581395,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.27906976744186046,"conditional_unsafe":0.7209302325581395,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.16746613786237488,"upper":0.42691028681878873,"confidence_level":0.95}},"effective":{"n":43,"acceptable":6,"unsafe":0,"review":38,"automated_count":5,"automated_acceptable":1,"automated_unsafe":0,"unnecessary_reviews":33,"accuracy":0.13953488372093023,"unsafe_rate":0.0,"review_rate":0.8837209302325582,"automated_coverage":0.11627906976744186,"conditional_correctness":0.2,"conditional_unsafe":0.0,"unnecessary_review_rate":0.7674418604651163,"accuracy_wilson_95":{"lower":0.06555723427964878,"upper":0.2726358796738286,"confidence_level":0.95}},"confusion":{"raw":{"keep":{"keep":12,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0},"drop":{"keep":15,"drop":0,"mask":0,"pseudonymize":0,"generalize":1,"review":0},"mask":{"keep":2,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0},"pseudonymize":{"keep":2,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0},"generalize":{"keep":6,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0},"review":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0}},"effective":{"keep":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":12},"drop":{"keep":0,"drop":0,"mask":4,"pseudonymize":0,"generalize":0,"review":12},"mask":{"keep":0,"drop":0,"mask":1,"pseudonymize":0,"generalize":0,"review":1},"pseudonymize":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":2},"generalize":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":6},"review":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0}}},"probability_metrics":{"singleton_n":38,"multiacceptable_n":5,"multiacceptable_coverage":0.11627906976744186,"nll":1.707879599881062,"nll_zero_mass_n":0,"brier":0.803495306318909,"acceptable_set_n":43,"acceptable_set_nll":1.600461589291823,"acceptable_set_zero_mass_n":0},"calibration_all_labels":{"equal_width":{"n":43,"bin_count":10,"ece":0.019902582085409848,"signed_mean_gap":-0.019902582085409848,"absolute_mean_gap":0.019902582085409848,"bins":[{"index":0,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.0,"upper":0.1,"upper_inclusive":false},{"index":1,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.1,"upper":0.2,"upper_inclusive":false},{"index":2,"count":43,"confidence":0.2591671853564506,"empirical_correctness":0.27906976744186046,"signed_gap":-0.019902582085409848,"absolute_gap":0.019902582085409848,"lower":0.2,"upper":0.3,"upper_inclusive":false},{"index":3,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.3,"upper":0.4,"upper_inclusive":false},{"index":4,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.4,"upper":0.5,"upper_inclusive":false},{"index":5,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.5,"upper":0.6,"upper_inclusive":false},{"index":6,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.6,"upper":0.7,"upper_inclusive":false},{"index":7,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.7,"upper":0.8,"upper_inclusive":false},{"index":8,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.8,"upper":0.9,"upper_inclusive":false},{"index":9,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.9,"upper":1.0,"upper_inclusive":true}]},"equal_mass":{"n":43,"bin_count":10,"ece":0.17034895059674285,"signed_mean_gap":-0.019902582085409834,"absolute_mean_gap":0.019902582085409834,"bins":[{"index":0,"count":4,"confidence":0.23466655984520912,"empirical_correctness":0.25,"signed_gap":-0.015333440154790878,"absolute_gap":0.015333440154790878,"min_confidence":0.2323085516691208,"max_confidence":0.23701980710029602},{"index":1,"count":4,"confidence":0.23891931027173996,"empirical_correctness":0.25,"signed_gap":-0.01108068972826004,"absolute_gap":0.01108068972826004,"min_confidence":0.23738934099674225,"max_confidence":0.24100100994110107},{"index":2,"count":4,"confidence":0.24230679869651794,"empirical_correctness":0.25,"signed_gap":-0.007693201303482056,"absolute_gap":0.007693201303482056,"min_confidence":0.24116480350494385,"max_confidence":0.24313035607337952},{"index":3,"count":5,"confidence":0.24386067390441896,"empirical_correctness":0.4,"signed_gap":-0.15613932609558107,"absolute_gap":0.15613932609558107,"min_confidence":0.24319249391555786,"max_confidence":0.24430061876773834},{"index":4,"count":4,"confidence":0.24809692427515984,"empirical_correctness":0.0,"signed_gap":0.24809692427515984,"absolute_gap":0.24809692427515984,"min_confidence":0.2444727122783661,"max_confidence":0.25176408886909485},{"index":5,"count":4,"confidence":0.2596913129091263,"empirical_correctness":0.5,"signed_gap":-0.24030868709087372,"absolute_gap":0.24030868709087372,"min_confidence":0.2570613622665405,"max_confidence":0.26187342405319214},{"index":6,"count":5,"confidence":0.26647503972053527,"empirical_correctness":0.4,"signed_gap":-0.13352496027946475,"absolute_gap":0.13352496027946475,"min_confidence":0.26206955313682556,"max_confidence":0.27180665731430054},{"index":7,"count":4,"confidence":0.2773907259106636,"empirical_correctness":0.0,"signed_gap":0.2773907259106636,"absolute_gap":0.2773907259106636,"min_confidence":0.273741751909256,"max_confidence":0.28037017583847046},{"index":8,"count":4,"confidence":0.28316158056259155,"empirical_correctness":0.0,"signed_gap":0.28316158056259155,"absolute_gap":0.28316158056259155,"min_confidence":0.2804921567440033,"max_confidence":0.28826308250427246},{"index":9,"count":5,"confidence":0.2911155104637146,"empirical_correctness":0.6,"signed_gap":-0.3088844895362854,"absolute_gap":0.3088844895362854,"min_confidence":0.288397878408432,"max_confidence":0.2967776656150818}]}},"calibration_singleton_labels":{"equal_width":{"n":38,"bin_count":10,"ece":0.05589585555227178,"signed_mean_gap":-0.05589585555227178,"absolute_mean_gap":0.05589585555227178,"bins":[{"index":0,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.0,"upper":0.1,"upper_inclusive":false},{"index":1,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.1,"upper":0.2,"upper_inclusive":false},{"index":2,"count":38,"confidence":0.25989361813193873,"empirical_correctness":0.3157894736842105,"signed_gap":-0.05589585555227178,"absolute_gap":0.05589585555227178,"lower":0.2,"upper":0.3,"upper_inclusive":false},{"index":3,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.3,"upper":0.4,"upper_inclusive":false},{"index":4,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.4,"upper":0.5,"upper_inclusive":false},{"index":5,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.5,"upper":0.6,"upper_inclusive":false},{"index":6,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.6,"upper":0.7,"upper_inclusive":false},{"index":7,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.7,"upper":0.8,"upper_inclusive":false},{"index":8,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.8,"upper":0.9,"upper_inclusive":false},{"index":9,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.9,"upper":1.0,"upper_inclusive":true}]},"equal_mass":{"n":38,"bin_count":10,"ece":0.1588820517063141,"signed_mean_gap":-0.05589585555227179,"absolute_mean_gap":0.05589585555227179,"bins":[{"index":0,"count":3,"confidence":0.2338821440935135,"empirical_correctness":0.0,"signed_gap":0.2338821440935135,"absolute_gap":0.2338821440935135,"min_confidence":0.2323085516691208,"max_confidence":0.23535777628421783},{"index":1,"count":4,"confidence":0.2388269267976284,"empirical_correctness":0.5,"signed_gap":-0.2611730732023716,"absolute_gap":0.2611730732023716,"min_confidence":0.23701980710029602,"max_confidence":0.24100100994110107},{"index":2,"count":4,"confidence":0.24239883571863174,"empirical_correctness":0.25,"signed_gap":-0.007601164281368256,"absolute_gap":0.007601164281368256,"min_confidence":0.24116480350494385,"max_confidence":0.24319249391555786},{"index":3,"count":4,"confidence":0.24413807317614555,"empirical_correctness":0.5,"signed_gap":-0.25586192682385445,"absolute_gap":0.25586192682385445,"min_confidence":0.24352674186229706,"max_confidence":0.2444727122783661},{"index":4,"count":4,"confidence":0.25124408677220345,"empirical_correctness":0.25,"signed_gap":0.0012440867722034454,"absolute_gap":0.0012440867722034454,"min_confidence":0.24702204763889313,"max_confidence":0.2570613622665405},{"index":5,"count":3,"confidence":0.26056796312332153,"empirical_correctness":0.3333333333333333,"signed_gap":-0.07276537021001178,"absolute_gap":0.07276537021001178,"min_confidence":0.25885528326034546,"max_confidence":0.26187342405319214},{"index":6,"count":4,"confidence":0.267220214009285,"empirical_correctness":0.5,"signed_gap":-0.23277978599071503,"absolute_gap":0.23277978599071503,"min_confidence":0.26206955313682556,"max_confidence":0.27180665731430054},{"index":7,"count":4,"confidence":0.2773907259106636,"empirical_correctness":0.0,"signed_gap":0.2773907259106636,"absolute_gap":0.2773907259106636,"min_confidence":0.273741751909256,"max_confidence":0.28037017583847046},{"index":8,"count":4,"confidence":0.28513801097869873,"empirical_correctness":0.25,"signed_gap":0.03513801097869873,"absolute_gap":0.03513801097869873,"min_confidence":0.2812163233757019,"max_confidence":0.288397878408432},{"index":9,"count":4,"confidence":0.29179491847753525,"empirical_correctness":0.5,"signed_gap":-0.20820508152246475,"absolute_gap":0.20820508152246475,"min_confidence":0.28913185000419617,"max_confidence":0.2967776656150818}]}},"latency_ms":{"n":43,"median":39.032533997669816,"p95":39.77602002560161},"per_action":{"keep":{"n":12,"raw":{"n":12,"acceptable":12,"unsafe":0,"review":0,"automated_count":12,"automated_acceptable":12,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":1.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.7575059933447592,"upper":1.0,"confidence_level":0.95}},"effective":{"n":12,"acceptable":0,"unsafe":0,"review":12,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":12,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":2.7755575615628914e-17,"upper":0.24249400665524085,"confidence_level":0.95}}},"drop":{"n":16,"raw":{"n":16,"acceptable":0,"unsafe":16,"review":0,"automated_count":16,"automated_acceptable":0,"automated_unsafe":16,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":1.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":1.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.1936076805344365,"confidence_level":0.95}},"effective":{"n":16,"acceptable":0,"unsafe":0,"review":12,"automated_count":4,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":12,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":0.75,"automated_coverage":0.25,"conditional_correctness":0.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.75,"accuracy_wilson_95":{"lower":0.0,"upper":0.1936076805344365,"confidence_level":0.95}}},"mask":{"n":2,"raw":{"n":2,"acceptable":0,"unsafe":2,"review":0,"automated_count":2,"automated_acceptable":0,"automated_unsafe":2,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":1.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":1.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.6576197724933469,"confidence_level":0.95}},"effective":{"n":2,"acceptable":1,"unsafe":0,"review":1,"automated_count":1,"automated_acceptable":1,"automated_unsafe":0,"unnecessary_reviews":1,"accuracy":0.5,"unsafe_rate":0.0,"review_rate":0.5,"automated_coverage":0.5,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.5,"accuracy_wilson_95":{"lower":0.09453120573423074,"upper":0.9054687942657693,"confidence_level":0.95}}},"pseudonymize":{"n":2,"raw":{"n":2,"acceptable":0,"unsafe":2,"review":0,"automated_count":2,"automated_acceptable":0,"automated_unsafe":2,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":1.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":1.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.6576197724933469,"confidence_level":0.95}},"effective":{"n":2,"acceptable":0,"unsafe":0,"review":2,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":2,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.6576197724933469,"confidence_level":0.95}}},"generalize":{"n":6,"raw":{"n":6,"acceptable":0,"unsafe":6,"review":0,"automated_count":6,"automated_acceptable":0,"automated_unsafe":6,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":1.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":1.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":2.7755575615628914e-17,"upper":0.3903342879021653,"confidence_level":0.95}},"effective":{"n":6,"acceptable":0,"unsafe":0,"review":6,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":6,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":2.7755575615628914e-17,"upper":0.3903342879021653,"confidence_level":0.95}}},"review":{"n":0,"raw":{"n":0,"acceptable":0,"unsafe":0,"review":0,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":null,"unsafe_rate":null,"review_rate":null,"automated_coverage":null,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":null},"effective":{"n":0,"acceptable":0,"unsafe":0,"review":0,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":null,"unsafe_rate":null,"review_rate":null,"automated_coverage":null,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":null}}}},"diagnostic43-compact_no_samples":{"n":43,"raw":{"n":43,"acceptable":12,"unsafe":31,"review":0,"automated_count":43,"automated_acceptable":12,"automated_unsafe":31,"unnecessary_reviews":0,"accuracy":0.27906976744186046,"unsafe_rate":0.7209302325581395,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.27906976744186046,"conditional_unsafe":0.7209302325581395,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.16746613786237488,"upper":0.42691028681878873,"confidence_level":0.95}},"effective":{"n":43,"acceptable":6,"unsafe":0,"review":38,"automated_count":5,"automated_acceptable":1,"automated_unsafe":0,"unnecessary_reviews":33,"accuracy":0.13953488372093023,"unsafe_rate":0.0,"review_rate":0.8837209302325582,"automated_coverage":0.11627906976744186,"conditional_correctness":0.2,"conditional_unsafe":0.0,"unnecessary_review_rate":0.7674418604651163,"accuracy_wilson_95":{"lower":0.06555723427964878,"upper":0.2726358796738286,"confidence_level":0.95}},"confusion":{"raw":{"keep":{"keep":12,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0},"drop":{"keep":16,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0},"mask":{"keep":2,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0},"pseudonymize":{"keep":2,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0},"generalize":{"keep":6,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0},"review":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0}},"effective":{"keep":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":12},"drop":{"keep":0,"drop":0,"mask":4,"pseudonymize":0,"generalize":0,"review":12},"mask":{"keep":0,"drop":0,"mask":1,"pseudonymize":0,"generalize":0,"review":1},"pseudonymize":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":2},"generalize":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":6},"review":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0}}},"probability_metrics":{"singleton_n":38,"multiacceptable_n":5,"multiacceptable_coverage":0.11627906976744186,"nll":1.721735897811196,"nll_zero_mass_n":0,"brier":0.8081842027136268,"acceptable_set_n":43,"acceptable_set_nll":1.6111819069603495,"acceptable_set_zero_mass_n":0},"calibration_all_labels":{"equal_width":{"n":43,"bin_count":10,"ece":0.0425894752491352,"signed_mean_gap":-0.0425894752491352,"absolute_mean_gap":0.0425894752491352,"bins":[{"index":0,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.0,"upper":0.1,"upper_inclusive":false},{"index":1,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.1,"upper":0.2,"upper_inclusive":false},{"index":2,"count":43,"confidence":0.23648029219272526,"empirical_correctness":0.27906976744186046,"signed_gap":-0.0425894752491352,"absolute_gap":0.0425894752491352,"lower":0.2,"upper":0.3,"upper_inclusive":false},{"index":3,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.3,"upper":0.4,"upper_inclusive":false},{"index":4,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.4,"upper":0.5,"upper_inclusive":false},{"index":5,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.5,"upper":0.6,"upper_inclusive":false},{"index":6,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.6,"upper":0.7,"upper_inclusive":false},{"index":7,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.7,"upper":0.8,"upper_inclusive":false},{"index":8,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.8,"upper":0.9,"upper_inclusive":false},{"index":9,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.9,"upper":1.0,"upper_inclusive":true}]},"equal_mass":{"n":43,"bin_count":10,"ece":0.22431341506714045,"signed_mean_gap":-0.042589475249135214,"absolute_mean_gap":0.042589475249135214,"bins":[{"index":0,"count":4,"confidence":0.21868955343961716,"empirical_correctness":0.0,"signed_gap":0.21868955343961716,"absolute_gap":0.21868955343961716,"min_confidence":0.21415725350379944,"max_confidence":0.22147901356220245},{"index":1,"count":4,"confidence":0.22244350612163544,"empirical_correctness":0.0,"signed_gap":0.22244350612163544,"absolute_gap":0.22244350612163544,"min_confidence":0.22151151299476624,"max_confidence":0.2229221761226654},{"index":2,"count":4,"confidence":0.2255970872938633,"empirical_correctness":0.25,"signed_gap":-0.024402912706136703,"absolute_gap":0.024402912706136703,"min_confidence":0.22496332228183746,"max_confidence":0.22598664462566376},{"index":3,"count":5,"confidence":0.22703829109668733,"empirical_correctness":0.2,"signed_gap":0.027038291096687317,"absolute_gap":0.027038291096687317,"min_confidence":0.22651273012161255,"max_confidence":0.2277025580406189},{"index":4,"count":4,"confidence":0.230042215436697,"empirical_correctness":0.25,"signed_gap":-0.019957784563302994,"absolute_gap":0.019957784563302994,"min_confidence":0.22835882008075714,"max_confidence":0.23195146024227142},{"index":5,"count":4,"confidence":0.23454183340072632,"empirical_correctness":0.75,"signed_gap":-0.5154581665992737,"absolute_gap":0.5154581665992737,"min_confidence":0.23271913826465607,"max_confidence":0.23649515211582184},{"index":6,"count":5,"confidence":0.23924767971038818,"empirical_correctness":0.6,"signed_gap":-0.3607523202896118,"absolute_gap":0.3607523202896118,"min_confidence":0.23668012022972107,"max_confidence":0.2435939610004425},{"index":7,"count":4,"confidence":0.24951694533228874,"empirical_correctness":0.0,"signed_gap":0.24951694533228874,"absolute_gap":0.24951694533228874,"min_confidence":0.2488696426153183,"max_confidence":0.2502773106098175},{"index":8,"count":4,"confidence":0.2523183077573776,"empirical_correctness":0.0,"signed_gap":0.2523183077573776,"absolute_gap":0.2523183077573776,"min_confidence":0.2514435946941376,"max_confidence":0.2528737783432007},{"index":9,"count":5,"confidence":0.26092498302459716,"empirical_correctness":0.6,"signed_gap":-0.3390750169754028,"absolute_gap":0.3390750169754028,"min_confidence":0.2540304362773895,"max_confidence":0.2698674499988556}]}},"calibration_singleton_labels":{"equal_width":{"n":38,"bin_count":10,"ece":0.07909984220015373,"signed_mean_gap":-0.07909984220015373,"absolute_mean_gap":0.07909984220015373,"bins":[{"index":0,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.0,"upper":0.1,"upper_inclusive":false},{"index":1,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.1,"upper":0.2,"upper_inclusive":false},{"index":2,"count":38,"confidence":0.23668963148405678,"empirical_correctness":0.3157894736842105,"signed_gap":-0.07909984220015373,"absolute_gap":0.07909984220015373,"lower":0.2,"upper":0.3,"upper_inclusive":false},{"index":3,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.3,"upper":0.4,"upper_inclusive":false},{"index":4,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.4,"upper":0.5,"upper_inclusive":false},{"index":5,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.5,"upper":0.6,"upper_inclusive":false},{"index":6,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.6,"upper":0.7,"upper_inclusive":false},{"index":7,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.7,"upper":0.8,"upper_inclusive":false},{"index":8,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.8,"upper":0.9,"upper_inclusive":false},{"index":9,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.9,"upper":1.0,"upper_inclusive":true}]},"equal_mass":{"n":38,"bin_count":10,"ece":0.313653822400068,"signed_mean_gap":-0.07909984220015376,"absolute_mean_gap":0.07909984220015376,"bins":[{"index":0,"count":3,"confidence":0.2177597333987554,"empirical_correctness":0.0,"signed_gap":0.2177597333987554,"absolute_gap":0.2177597333987554,"min_confidence":0.21415725350379944,"max_confidence":0.22015778720378876},{"index":1,"count":4,"confidence":0.22210412845015526,"empirical_correctness":0.0,"signed_gap":0.22210412845015526,"absolute_gap":0.22210412845015526,"min_confidence":0.22147901356220245,"max_confidence":0.2229221761226654},{"index":2,"count":4,"confidence":0.2255970872938633,"empirical_correctness":0.25,"signed_gap":-0.024402912706136703,"absolute_gap":0.024402912706136703,"min_confidence":0.22496332228183746,"max_confidence":0.22598664462566376},{"index":3,"count":4,"confidence":0.22687222436070442,"empirical_correctness":0.0,"signed_gap":0.22687222436070442,"absolute_gap":0.22687222436070442,"min_confidence":0.22651273012161255,"max_confidence":0.22710919380187988},{"index":4,"count":4,"confidence":0.23018299415707588,"empirical_correctness":0.75,"signed_gap":-0.5198170058429241,"absolute_gap":0.5198170058429241,"min_confidence":0.2277025580406189,"max_confidence":0.23271913826465607},{"index":5,"count":3,"confidence":0.23514939844608307,"empirical_correctness":0.6666666666666666,"signed_gap":-0.43151726822058356,"absolute_gap":0.43151726822058356,"min_confidence":0.23409560322761536,"max_confidence":0.23649515211582184},{"index":6,"count":4,"confidence":0.23988956958055496,"empirical_correctness":0.75,"signed_gap":-0.510110430419445,"absolute_gap":0.510110430419445,"min_confidence":0.237631693482399,"max_confidence":0.2435939610004425},{"index":7,"count":4,"confidence":0.24951694533228874,"empirical_correctness":0.0,"signed_gap":0.24951694533228874,"absolute_gap":0.24951694533228874,"min_confidence":0.2488696426153183,"max_confidence":0.2502773106098175},{"index":8,"count":4,"confidence":0.2523183077573776,"empirical_correctness":0.0,"signed_gap":0.2523183077573776,"absolute_gap":0.2523183077573776,"min_confidence":0.2514435946941376,"max_confidence":0.2528737783432007},{"index":9,"count":4,"confidence":0.2623883932828903,"empirical_correctness":0.75,"signed_gap":-0.4876116067171097,"absolute_gap":0.4876116067171097,"min_confidence":0.2540304362773895,"max_confidence":0.2698674499988556}]}},"latency_ms":{"n":43,"median":48.22288599098101,"p95":49.07378313364461},"per_action":{"keep":{"n":12,"raw":{"n":12,"acceptable":12,"unsafe":0,"review":0,"automated_count":12,"automated_acceptable":12,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":1.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.7575059933447592,"upper":1.0,"confidence_level":0.95}},"effective":{"n":12,"acceptable":0,"unsafe":0,"review":12,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":12,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":2.7755575615628914e-17,"upper":0.24249400665524085,"confidence_level":0.95}}},"drop":{"n":16,"raw":{"n":16,"acceptable":0,"unsafe":16,"review":0,"automated_count":16,"automated_acceptable":0,"automated_unsafe":16,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":1.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":1.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.1936076805344365,"confidence_level":0.95}},"effective":{"n":16,"acceptable":0,"unsafe":0,"review":12,"automated_count":4,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":12,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":0.75,"automated_coverage":0.25,"conditional_correctness":0.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.75,"accuracy_wilson_95":{"lower":0.0,"upper":0.1936076805344365,"confidence_level":0.95}}},"mask":{"n":2,"raw":{"n":2,"acceptable":0,"unsafe":2,"review":0,"automated_count":2,"automated_acceptable":0,"automated_unsafe":2,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":1.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":1.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.6576197724933469,"confidence_level":0.95}},"effective":{"n":2,"acceptable":1,"unsafe":0,"review":1,"automated_count":1,"automated_acceptable":1,"automated_unsafe":0,"unnecessary_reviews":1,"accuracy":0.5,"unsafe_rate":0.0,"review_rate":0.5,"automated_coverage":0.5,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.5,"accuracy_wilson_95":{"lower":0.09453120573423074,"upper":0.9054687942657693,"confidence_level":0.95}}},"pseudonymize":{"n":2,"raw":{"n":2,"acceptable":0,"unsafe":2,"review":0,"automated_count":2,"automated_acceptable":0,"automated_unsafe":2,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":1.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":1.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.6576197724933469,"confidence_level":0.95}},"effective":{"n":2,"acceptable":0,"unsafe":0,"review":2,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":2,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.6576197724933469,"confidence_level":0.95}}},"generalize":{"n":6,"raw":{"n":6,"acceptable":0,"unsafe":6,"review":0,"automated_count":6,"automated_acceptable":0,"automated_unsafe":6,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":1.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":1.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":2.7755575615628914e-17,"upper":0.3903342879021653,"confidence_level":0.95}},"effective":{"n":6,"acceptable":0,"unsafe":0,"review":6,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":6,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":2.7755575615628914e-17,"upper":0.3903342879021653,"confidence_level":0.95}}},"review":{"n":0,"raw":{"n":0,"acceptable":0,"unsafe":0,"review":0,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":null,"unsafe_rate":null,"review_rate":null,"automated_coverage":null,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":null},"effective":{"n":0,"acceptable":0,"unsafe":0,"review":0,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":null,"unsafe_rate":null,"review_rate":null,"automated_coverage":null,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":null}}}},"challenge56-no_samples":{"n":56,"raw":{"n":56,"acceptable":11,"unsafe":45,"review":0,"automated_count":56,"automated_acceptable":11,"automated_unsafe":45,"unnecessary_reviews":0,"accuracy":0.19642857142857142,"unsafe_rate":0.8035714285714286,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.19642857142857142,"conditional_unsafe":0.8035714285714286,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.11338595674323658,"upper":0.31844607626458404,"confidence_level":0.95}},"effective":{"n":56,"acceptable":2,"unsafe":0,"review":56,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":54,"accuracy":0.03571428571428571,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":0.9642857142857143,"accuracy_wilson_95":{"lower":0.00984942514823639,"upper":0.12118780180490107,"confidence_level":0.95}},"confusion":{"raw":{"keep":{"keep":8,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0},"drop":{"keep":22,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0},"mask":{"keep":8,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0},"pseudonymize":{"keep":8,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0},"generalize":{"keep":5,"drop":0,"mask":0,"pseudonymize":0,"generalize":3,"review":0},"review":{"keep":2,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0}},"effective":{"keep":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":8},"drop":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":22},"mask":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":8},"pseudonymize":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":8},"generalize":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":8},"review":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":2}}},"probability_metrics":{"singleton_n":56,"multiacceptable_n":0,"multiacceptable_coverage":0.0,"nll":1.7384964543430785,"nll_zero_mass_n":0,"brier":0.8150770436028986,"acceptable_set_n":56,"acceptable_set_nll":1.7384964543430785,"acceptable_set_zero_mass_n":0},"calibration_all_labels":{"equal_width":{"n":56,"bin_count":10,"ece":0.04620225248592241,"signed_mean_gap":0.04620225248592241,"absolute_mean_gap":0.04620225248592241,"bins":[{"index":0,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.0,"upper":0.1,"upper_inclusive":false},{"index":1,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.1,"upper":0.2,"upper_inclusive":false},{"index":2,"count":56,"confidence":0.24263082391449384,"empirical_correctness":0.19642857142857142,"signed_gap":0.04620225248592241,"absolute_gap":0.04620225248592241,"lower":0.2,"upper":0.3,"upper_inclusive":false},{"index":3,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.3,"upper":0.4,"upper_inclusive":false},{"index":4,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.4,"upper":0.5,"upper_inclusive":false},{"index":5,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.5,"upper":0.6,"upper_inclusive":false},{"index":6,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.6,"upper":0.7,"upper_inclusive":false},{"index":7,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.7,"upper":0.8,"upper_inclusive":false},{"index":8,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.8,"upper":0.9,"upper_inclusive":false},{"index":9,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.9,"upper":1.0,"upper_inclusive":true}]},"equal_mass":{"n":56,"bin_count":10,"ece":0.1776047402194568,"signed_mean_gap":0.04620225248592241,"absolute_mean_gap":0.04620225248592241,"bins":[{"index":0,"count":5,"confidence":0.2154470682144165,"empirical_correctness":0.0,"signed_gap":0.2154470682144165,"absolute_gap":0.2154470682144165,"min_confidence":0.20683002471923828,"max_confidence":0.22002004086971283},{"index":1,"count":6,"confidence":0.22735556463400522,"empirical_correctness":0.0,"signed_gap":0.22735556463400522,"absolute_gap":0.22735556463400522,"min_confidence":0.22034163773059845,"max_confidence":0.23388387262821198},{"index":2,"count":5,"confidence":0.23513534367084504,"empirical_correctness":0.2,"signed_gap":0.03513534367084503,"absolute_gap":0.03513534367084503,"min_confidence":0.23410937190055847,"max_confidence":0.23595455288887024},{"index":3,"count":6,"confidence":0.23885243386030197,"empirical_correctness":0.0,"signed_gap":0.23885243386030197,"absolute_gap":0.23885243386030197,"min_confidence":0.2378797084093094,"max_confidence":0.23939402401447296},{"index":4,"count":6,"confidence":0.24077114711205164,"empirical_correctness":0.0,"signed_gap":0.24077114711205164,"absolute_gap":0.24077114711205164,"min_confidence":0.24013486504554749,"max_confidence":0.24170801043510437},{"index":5,"count":5,"confidence":0.24305417835712434,"empirical_correctness":0.4,"signed_gap":-0.15694582164287568,"absolute_gap":0.15694582164287568,"min_confidence":0.2420770227909088,"max_confidence":0.24422802031040192},{"index":6,"count":6,"confidence":0.24700693786144257,"empirical_correctness":0.3333333333333333,"signed_gap":-0.08632639547189075,"absolute_gap":0.08632639547189075,"min_confidence":0.2457784116268158,"max_confidence":0.24832987785339355},{"index":7,"count":5,"confidence":0.2497921347618103,"empirical_correctness":0.2,"signed_gap":0.0497921347618103,"absolute_gap":0.0497921347618103,"min_confidence":0.24861523509025574,"max_confidence":0.25158432126045227},{"index":8,"count":6,"confidence":0.25380803147951764,"empirical_correctness":0.16666666666666666,"signed_gap":0.08714136481285098,"absolute_gap":0.08714136481285098,"min_confidence":0.2519809901714325,"max_confidence":0.2566836476325989},{"index":9,"count":6,"confidence":0.2705696374177933,"empirical_correctness":0.6666666666666666,"signed_gap":-0.39609702924887336,"absolute_gap":0.39609702924887336,"min_confidence":0.25710031390190125,"max_confidence":0.28427886962890625}]}},"calibration_singleton_labels":{"equal_width":{"n":56,"bin_count":10,"ece":0.04620225248592241,"signed_mean_gap":0.04620225248592241,"absolute_mean_gap":0.04620225248592241,"bins":[{"index":0,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.0,"upper":0.1,"upper_inclusive":false},{"index":1,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.1,"upper":0.2,"upper_inclusive":false},{"index":2,"count":56,"confidence":0.24263082391449384,"empirical_correctness":0.19642857142857142,"signed_gap":0.04620225248592241,"absolute_gap":0.04620225248592241,"lower":0.2,"upper":0.3,"upper_inclusive":false},{"index":3,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.3,"upper":0.4,"upper_inclusive":false},{"index":4,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.4,"upper":0.5,"upper_inclusive":false},{"index":5,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.5,"upper":0.6,"upper_inclusive":false},{"index":6,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.6,"upper":0.7,"upper_inclusive":false},{"index":7,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.7,"upper":0.8,"upper_inclusive":false},{"index":8,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.8,"upper":0.9,"upper_inclusive":false},{"index":9,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.9,"upper":1.0,"upper_inclusive":true}]},"equal_mass":{"n":56,"bin_count":10,"ece":0.1776047402194568,"signed_mean_gap":0.04620225248592241,"absolute_mean_gap":0.04620225248592241,"bins":[{"index":0,"count":5,"confidence":0.2154470682144165,"empirical_correctness":0.0,"signed_gap":0.2154470682144165,"absolute_gap":0.2154470682144165,"min_confidence":0.20683002471923828,"max_confidence":0.22002004086971283},{"index":1,"count":6,"confidence":0.22735556463400522,"empirical_correctness":0.0,"signed_gap":0.22735556463400522,"absolute_gap":0.22735556463400522,"min_confidence":0.22034163773059845,"max_confidence":0.23388387262821198},{"index":2,"count":5,"confidence":0.23513534367084504,"empirical_correctness":0.2,"signed_gap":0.03513534367084503,"absolute_gap":0.03513534367084503,"min_confidence":0.23410937190055847,"max_confidence":0.23595455288887024},{"index":3,"count":6,"confidence":0.23885243386030197,"empirical_correctness":0.0,"signed_gap":0.23885243386030197,"absolute_gap":0.23885243386030197,"min_confidence":0.2378797084093094,"max_confidence":0.23939402401447296},{"index":4,"count":6,"confidence":0.24077114711205164,"empirical_correctness":0.0,"signed_gap":0.24077114711205164,"absolute_gap":0.24077114711205164,"min_confidence":0.24013486504554749,"max_confidence":0.24170801043510437},{"index":5,"count":5,"confidence":0.24305417835712434,"empirical_correctness":0.4,"signed_gap":-0.15694582164287568,"absolute_gap":0.15694582164287568,"min_confidence":0.2420770227909088,"max_confidence":0.24422802031040192},{"index":6,"count":6,"confidence":0.24700693786144257,"empirical_correctness":0.3333333333333333,"signed_gap":-0.08632639547189075,"absolute_gap":0.08632639547189075,"min_confidence":0.2457784116268158,"max_confidence":0.24832987785339355},{"index":7,"count":5,"confidence":0.2497921347618103,"empirical_correctness":0.2,"signed_gap":0.0497921347618103,"absolute_gap":0.0497921347618103,"min_confidence":0.24861523509025574,"max_confidence":0.25158432126045227},{"index":8,"count":6,"confidence":0.25380803147951764,"empirical_correctness":0.16666666666666666,"signed_gap":0.08714136481285098,"absolute_gap":0.08714136481285098,"min_confidence":0.2519809901714325,"max_confidence":0.2566836476325989},{"index":9,"count":6,"confidence":0.2705696374177933,"empirical_correctness":0.6666666666666666,"signed_gap":-0.39609702924887336,"absolute_gap":0.39609702924887336,"min_confidence":0.25710031390190125,"max_confidence":0.28427886962890625}]}},"latency_ms":{"n":56,"median":36.30354200140573,"p95":38.49188801541459},"per_action":{"keep":{"n":8,"raw":{"n":8,"acceptable":8,"unsafe":0,"review":0,"automated_count":8,"automated_acceptable":8,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":1.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.6755924351161198,"upper":1.0,"confidence_level":0.95}},"effective":{"n":8,"acceptable":0,"unsafe":0,"review":8,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":8,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.32440756488388023,"confidence_level":0.95}}},"drop":{"n":22,"raw":{"n":22,"acceptable":0,"unsafe":22,"review":0,"automated_count":22,"automated_acceptable":0,"automated_unsafe":22,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":1.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":1.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.14865487460862087,"confidence_level":0.95}},"effective":{"n":22,"acceptable":0,"unsafe":0,"review":22,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":22,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.14865487460862087,"confidence_level":0.95}}},"mask":{"n":8,"raw":{"n":8,"acceptable":0,"unsafe":8,"review":0,"automated_count":8,"automated_acceptable":0,"automated_unsafe":8,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":1.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":1.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.32440756488388023,"confidence_level":0.95}},"effective":{"n":8,"acceptable":0,"unsafe":0,"review":8,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":8,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.32440756488388023,"confidence_level":0.95}}},"pseudonymize":{"n":8,"raw":{"n":8,"acceptable":0,"unsafe":8,"review":0,"automated_count":8,"automated_acceptable":0,"automated_unsafe":8,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":1.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":1.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.32440756488388023,"confidence_level":0.95}},"effective":{"n":8,"acceptable":0,"unsafe":0,"review":8,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":8,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.32440756488388023,"confidence_level":0.95}}},"generalize":{"n":8,"raw":{"n":8,"acceptable":3,"unsafe":5,"review":0,"automated_count":8,"automated_acceptable":3,"automated_unsafe":5,"unnecessary_reviews":0,"accuracy":0.375,"unsafe_rate":0.625,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.375,"conditional_unsafe":0.625,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.13684428582359742,"upper":0.6942576053973727,"confidence_level":0.95}},"effective":{"n":8,"acceptable":0,"unsafe":0,"review":8,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":8,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.32440756488388023,"confidence_level":0.95}}},"review":{"n":2,"raw":{"n":2,"acceptable":0,"unsafe":2,"review":0,"automated_count":2,"automated_acceptable":0,"automated_unsafe":2,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":1.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":1.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.6576197724933469,"confidence_level":0.95}},"effective":{"n":2,"acceptable":2,"unsafe":0,"review":2,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":1.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.34238022750665303,"upper":1.0,"confidence_level":0.95}}}}},"challenge56-compact_no_samples":{"n":56,"raw":{"n":56,"acceptable":11,"unsafe":45,"review":0,"automated_count":56,"automated_acceptable":11,"automated_unsafe":45,"unnecessary_reviews":0,"accuracy":0.19642857142857142,"unsafe_rate":0.8035714285714286,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.19642857142857142,"conditional_unsafe":0.8035714285714286,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.11338595674323658,"upper":0.31844607626458404,"confidence_level":0.95}},"effective":{"n":56,"acceptable":2,"unsafe":0,"review":56,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":54,"accuracy":0.03571428571428571,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":0.9642857142857143,"accuracy_wilson_95":{"lower":0.00984942514823639,"upper":0.12118780180490107,"confidence_level":0.95}},"confusion":{"raw":{"keep":{"keep":8,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0},"drop":{"keep":22,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0},"mask":{"keep":8,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0},"pseudonymize":{"keep":8,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0},"generalize":{"keep":5,"drop":0,"mask":0,"pseudonymize":0,"generalize":3,"review":0},"review":{"keep":2,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0}},"effective":{"keep":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":8},"drop":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":22},"mask":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":8},"pseudonymize":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":8},"generalize":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":8},"review":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":2}}},"probability_metrics":{"singleton_n":56,"multiacceptable_n":0,"multiacceptable_coverage":0.0,"nll":1.766713223862057,"nll_zero_mass_n":0,"brier":0.8242194081599321,"acceptable_set_n":56,"acceptable_set_nll":1.766713223862057,"acceptable_set_zero_mass_n":0},"calibration_all_labels":{"equal_width":{"n":56,"bin_count":10,"ece":0.03310293225305422,"signed_mean_gap":0.03310293225305422,"absolute_mean_gap":0.03310293225305422,"bins":[{"index":0,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.0,"upper":0.1,"upper_inclusive":false},{"index":1,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.1,"upper":0.2,"upper_inclusive":false},{"index":2,"count":56,"confidence":0.22953150368162564,"empirical_correctness":0.19642857142857142,"signed_gap":0.03310293225305422,"absolute_gap":0.03310293225305422,"lower":0.2,"upper":0.3,"upper_inclusive":false},{"index":3,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.3,"upper":0.4,"upper_inclusive":false},{"index":4,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.4,"upper":0.5,"upper_inclusive":false},{"index":5,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.5,"upper":0.6,"upper_inclusive":false},{"index":6,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.6,"upper":0.7,"upper_inclusive":false},{"index":7,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.7,"upper":0.8,"upper_inclusive":false},{"index":8,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.8,"upper":0.9,"upper_inclusive":false},{"index":9,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.9,"upper":1.0,"upper_inclusive":true}]},"equal_mass":{"n":56,"bin_count":10,"ece":0.16860240218894823,"signed_mean_gap":0.033102932253054225,"absolute_mean_gap":0.033102932253054225,"bins":[{"index":0,"count":5,"confidence":0.21845707595348357,"empirical_correctness":0.0,"signed_gap":0.21845707595348357,"absolute_gap":0.21845707595348357,"min_confidence":0.21572093665599823,"max_confidence":0.22015345096588135},{"index":1,"count":6,"confidence":0.22240003943443298,"empirical_correctness":0.16666666666666666,"signed_gap":0.055733372767766326,"absolute_gap":0.055733372767766326,"min_confidence":0.22099237143993378,"max_confidence":0.2238769829273224},{"index":2,"count":5,"confidence":0.2251793384552002,"empirical_correctness":0.0,"signed_gap":0.2251793384552002,"absolute_gap":0.2251793384552002,"min_confidence":0.22418540716171265,"max_confidence":0.22595623135566711},{"index":3,"count":6,"confidence":0.2267658238609632,"empirical_correctness":0.3333333333333333,"signed_gap":-0.10656750947237012,"absolute_gap":0.10656750947237012,"min_confidence":0.2259914129972458,"max_confidence":0.2277001142501831},{"index":4,"count":6,"confidence":0.22927931199471155,"empirical_correctness":0.0,"signed_gap":0.22927931199471155,"absolute_gap":0.22927931199471155,"min_confidence":0.22839540243148804,"max_confidence":0.2304070144891739},{"index":5,"count":5,"confidence":0.23112043142318725,"empirical_correctness":0.0,"signed_gap":0.23112043142318725,"absolute_gap":0.23112043142318725,"min_confidence":0.23079842329025269,"max_confidence":0.23152762651443481},{"index":6,"count":6,"confidence":0.23252423852682114,"empirical_correctness":0.16666666666666666,"signed_gap":0.06585757186015448,"absolute_gap":0.06585757186015448,"min_confidence":0.23200179636478424,"max_confidence":0.23301269114017487},{"index":7,"count":5,"confidence":0.23374871909618378,"empirical_correctness":0.2,"signed_gap":0.033748719096183766,"absolute_gap":0.033748719096183766,"min_confidence":0.23331893980503082,"max_confidence":0.23412717878818512},{"index":8,"count":6,"confidence":0.23495294898748398,"empirical_correctness":0.3333333333333333,"signed_gap":-0.09838038434584934,"absolute_gap":0.09838038434584934,"min_confidence":0.23438166081905365,"max_confidence":0.23543263971805573},{"index":9,"count":6,"confidence":0.23928370078404745,"empirical_correctness":0.6666666666666666,"signed_gap":-0.42738296588261915,"absolute_gap":0.42738296588261915,"min_confidence":0.23662520945072174,"max_confidence":0.2414226084947586}]}},"calibration_singleton_labels":{"equal_width":{"n":56,"bin_count":10,"ece":0.03310293225305422,"signed_mean_gap":0.03310293225305422,"absolute_mean_gap":0.03310293225305422,"bins":[{"index":0,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.0,"upper":0.1,"upper_inclusive":false},{"index":1,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.1,"upper":0.2,"upper_inclusive":false},{"index":2,"count":56,"confidence":0.22953150368162564,"empirical_correctness":0.19642857142857142,"signed_gap":0.03310293225305422,"absolute_gap":0.03310293225305422,"lower":0.2,"upper":0.3,"upper_inclusive":false},{"index":3,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.3,"upper":0.4,"upper_inclusive":false},{"index":4,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.4,"upper":0.5,"upper_inclusive":false},{"index":5,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.5,"upper":0.6,"upper_inclusive":false},{"index":6,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.6,"upper":0.7,"upper_inclusive":false},{"index":7,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.7,"upper":0.8,"upper_inclusive":false},{"index":8,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.8,"upper":0.9,"upper_inclusive":false},{"index":9,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.9,"upper":1.0,"upper_inclusive":true}]},"equal_mass":{"n":56,"bin_count":10,"ece":0.16860240218894823,"signed_mean_gap":0.033102932253054225,"absolute_mean_gap":0.033102932253054225,"bins":[{"index":0,"count":5,"confidence":0.21845707595348357,"empirical_correctness":0.0,"signed_gap":0.21845707595348357,"absolute_gap":0.21845707595348357,"min_confidence":0.21572093665599823,"max_confidence":0.22015345096588135},{"index":1,"count":6,"confidence":0.22240003943443298,"empirical_correctness":0.16666666666666666,"signed_gap":0.055733372767766326,"absolute_gap":0.055733372767766326,"min_confidence":0.22099237143993378,"max_confidence":0.2238769829273224},{"index":2,"count":5,"confidence":0.2251793384552002,"empirical_correctness":0.0,"signed_gap":0.2251793384552002,"absolute_gap":0.2251793384552002,"min_confidence":0.22418540716171265,"max_confidence":0.22595623135566711},{"index":3,"count":6,"confidence":0.2267658238609632,"empirical_correctness":0.3333333333333333,"signed_gap":-0.10656750947237012,"absolute_gap":0.10656750947237012,"min_confidence":0.2259914129972458,"max_confidence":0.2277001142501831},{"index":4,"count":6,"confidence":0.22927931199471155,"empirical_correctness":0.0,"signed_gap":0.22927931199471155,"absolute_gap":0.22927931199471155,"min_confidence":0.22839540243148804,"max_confidence":0.2304070144891739},{"index":5,"count":5,"confidence":0.23112043142318725,"empirical_correctness":0.0,"signed_gap":0.23112043142318725,"absolute_gap":0.23112043142318725,"min_confidence":0.23079842329025269,"max_confidence":0.23152762651443481},{"index":6,"count":6,"confidence":0.23252423852682114,"empirical_correctness":0.16666666666666666,"signed_gap":0.06585757186015448,"absolute_gap":0.06585757186015448,"min_confidence":0.23200179636478424,"max_confidence":0.23301269114017487},{"index":7,"count":5,"confidence":0.23374871909618378,"empirical_correctness":0.2,"signed_gap":0.033748719096183766,"absolute_gap":0.033748719096183766,"min_confidence":0.23331893980503082,"max_confidence":0.23412717878818512},{"index":8,"count":6,"confidence":0.23495294898748398,"empirical_correctness":0.3333333333333333,"signed_gap":-0.09838038434584934,"absolute_gap":0.09838038434584934,"min_confidence":0.23438166081905365,"max_confidence":0.23543263971805573},{"index":9,"count":6,"confidence":0.23928370078404745,"empirical_correctness":0.6666666666666666,"signed_gap":-0.42738296588261915,"absolute_gap":0.42738296588261915,"min_confidence":0.23662520945072174,"max_confidence":0.2414226084947586}]}},"latency_ms":{"n":56,"median":39.2315095232334,"p95":40.1529987284448},"per_action":{"keep":{"n":8,"raw":{"n":8,"acceptable":8,"unsafe":0,"review":0,"automated_count":8,"automated_acceptable":8,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":1.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.6755924351161198,"upper":1.0,"confidence_level":0.95}},"effective":{"n":8,"acceptable":0,"unsafe":0,"review":8,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":8,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.32440756488388023,"confidence_level":0.95}}},"drop":{"n":22,"raw":{"n":22,"acceptable":0,"unsafe":22,"review":0,"automated_count":22,"automated_acceptable":0,"automated_unsafe":22,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":1.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":1.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.14865487460862087,"confidence_level":0.95}},"effective":{"n":22,"acceptable":0,"unsafe":0,"review":22,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":22,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.14865487460862087,"confidence_level":0.95}}},"mask":{"n":8,"raw":{"n":8,"acceptable":0,"unsafe":8,"review":0,"automated_count":8,"automated_acceptable":0,"automated_unsafe":8,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":1.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":1.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.32440756488388023,"confidence_level":0.95}},"effective":{"n":8,"acceptable":0,"unsafe":0,"review":8,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":8,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.32440756488388023,"confidence_level":0.95}}},"pseudonymize":{"n":8,"raw":{"n":8,"acceptable":0,"unsafe":8,"review":0,"automated_count":8,"automated_acceptable":0,"automated_unsafe":8,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":1.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":1.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.32440756488388023,"confidence_level":0.95}},"effective":{"n":8,"acceptable":0,"unsafe":0,"review":8,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":8,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.32440756488388023,"confidence_level":0.95}}},"generalize":{"n":8,"raw":{"n":8,"acceptable":3,"unsafe":5,"review":0,"automated_count":8,"automated_acceptable":3,"automated_unsafe":5,"unnecessary_reviews":0,"accuracy":0.375,"unsafe_rate":0.625,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.375,"conditional_unsafe":0.625,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.13684428582359742,"upper":0.6942576053973727,"confidence_level":0.95}},"effective":{"n":8,"acceptable":0,"unsafe":0,"review":8,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":8,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.32440756488388023,"confidence_level":0.95}}},"review":{"n":2,"raw":{"n":2,"acceptable":0,"unsafe":2,"review":0,"automated_count":2,"automated_acceptable":0,"automated_unsafe":2,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":1.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":1.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.6576197724933469,"confidence_level":0.95}},"effective":{"n":2,"acceptable":2,"unsafe":0,"review":2,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":1.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.34238022750665303,"upper":1.0,"confidence_level":0.95}}}}}},"composite":{"n":198,"scope":"descriptive sum across four correlated panels","raw":{"n":198,"acceptable":46,"unsafe":152,"review":0,"automated_count":198,"automated_acceptable":46,"automated_unsafe":152,"unnecessary_reviews":0,"accuracy":0.23232323232323232,"unsafe_rate":0.7676767676767676,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.23232323232323232,"conditional_unsafe":0.7676767676767676,"unnecessary_review_rate":0.0},"effective":{"n":198,"acceptable":16,"unsafe":0,"review":188,"automated_count":10,"automated_acceptable":2,"automated_unsafe":0,"unnecessary_reviews":174,"accuracy":0.08080808080808081,"unsafe_rate":0.0,"review_rate":0.9494949494949495,"automated_coverage":0.050505050505050504,"conditional_correctness":0.2,"conditional_unsafe":0.0,"unnecessary_review_rate":0.8787878787878788}}},{"id":"cross-encoder--nli-distilroberta-base","platform":"Local","model_id":"cross-encoder/nli-distilroberta-base","revision":"b14d131f9d32668a5e6a982729b57ff6ed5dfcbd","loading":"float32 / none","scoring_kind":"Adapted NLI","settings":{"dtype":"float32","quantization":null,"max_tokens":4096,"effective_max_tokens":512,"temperature":1.0,"scoring_batch_size":1,"trust_remote_code":false,"local_files_only":true,"use_safetensors":true,"guard_threshold":0.45,"seed":0,"tf32":false},"scoring_metadata":{"backend":"nli","protocol":"hf-local-sweep-backends-v1","probability_source":"adapted_entailment_softmax_not_calibrated_sixway","latency_scope":"synchronous_forward_scoring_including_transfer_excluding_tokenization","precision":"float32","equal_mass_convention":"rank bins with floor boundaries at floor(i*n/k), k=min(10,n//2); minimum two rows"},"load_ms":1356.7743790335953,"peak_allocated_bytes":380356608,"peak_reserved_bytes":406847488,"panels":{"diagnostic43-no_samples":{"n":43,"raw":{"n":43,"acceptable":6,"unsafe":0,"review":26,"automated_count":17,"automated_acceptable":3,"automated_unsafe":0,"unnecessary_reviews":23,"accuracy":0.13953488372093023,"unsafe_rate":0.0,"review_rate":0.6046511627906976,"automated_coverage":0.3953488372093023,"conditional_correctness":0.17647058823529413,"conditional_unsafe":0.0,"unnecessary_review_rate":0.5348837209302325,"accuracy_wilson_95":{"lower":0.06555723427964878,"upper":0.2726358796738286,"confidence_level":0.95}},"effective":{"n":43,"acceptable":5,"unsafe":0,"review":43,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":38,"accuracy":0.11627906976744186,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":0.8837209302325582,"accuracy_wilson_95":{"lower":0.05070427739804166,"upper":0.24479161810082128,"confidence_level":0.95}},"confusion":{"raw":{"keep":{"keep":0,"drop":0,"mask":5,"pseudonymize":0,"generalize":0,"review":7},"drop":{"keep":0,"drop":0,"mask":8,"pseudonymize":0,"generalize":0,"review":8},"mask":{"keep":0,"drop":0,"mask":1,"pseudonymize":0,"generalize":0,"review":1},"pseudonymize":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":2},"generalize":{"keep":0,"drop":0,"mask":1,"pseudonymize":0,"generalize":0,"review":5},"review":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0}},"effective":{"keep":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":12},"drop":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":16},"mask":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":2},"pseudonymize":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":2},"generalize":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":6},"review":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0}}},"probability_metrics":{"singleton_n":38,"multiacceptable_n":5,"multiacceptable_coverage":0.11627906976744186,"nll":1.9225895779458946,"nll_zero_mass_n":0,"brier":0.871107498482255,"acceptable_set_n":43,"acceptable_set_nll":1.7546685716855233,"acceptable_set_zero_mass_n":0},"calibration_all_labels":{"equal_width":{"n":43,"bin_count":10,"ece":0.08418038452780525,"signed_mean_gap":0.08418038452780525,"absolute_mean_gap":0.08418038452780525,"bins":[{"index":0,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.0,"upper":0.1,"upper_inclusive":false},{"index":1,"count":1,"confidence":0.19876635074615479,"empirical_correctness":0.0,"signed_gap":0.19876635074615479,"absolute_gap":0.19876635074615479,"lower":0.1,"upper":0.2,"upper_inclusive":false},{"index":2,"count":42,"confidence":0.22430929009403502,"empirical_correctness":0.14285714285714285,"signed_gap":0.08145214723689218,"absolute_gap":0.08145214723689218,"lower":0.2,"upper":0.3,"upper_inclusive":false},{"index":3,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.3,"upper":0.4,"upper_inclusive":false},{"index":4,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.4,"upper":0.5,"upper_inclusive":false},{"index":5,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.5,"upper":0.6,"upper_inclusive":false},{"index":6,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.6,"upper":0.7,"upper_inclusive":false},{"index":7,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.7,"upper":0.8,"upper_inclusive":false},{"index":8,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.8,"upper":0.9,"upper_inclusive":false},{"index":9,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.9,"upper":1.0,"upper_inclusive":true}]},"equal_mass":{"n":43,"bin_count":10,"ece":0.09776700270730396,"signed_mean_gap":0.08418038452780524,"absolute_mean_gap":0.08418038452780524,"bins":[{"index":0,"count":4,"confidence":0.20044827461242676,"empirical_correctness":0.0,"signed_gap":0.20044827461242676,"absolute_gap":0.20044827461242676,"min_confidence":0.19876635074615479,"max_confidence":0.2014317810535431},{"index":1,"count":4,"confidence":0.2032933533191681,"empirical_correctness":0.0,"signed_gap":0.2032933533191681,"absolute_gap":0.2032933533191681,"min_confidence":0.20178894698619843,"max_confidence":0.20447708666324615},{"index":2,"count":4,"confidence":0.20572799071669579,"empirical_correctness":0.0,"signed_gap":0.20572799071669579,"absolute_gap":0.20572799071669579,"min_confidence":0.20449994504451752,"max_confidence":0.2076769322156906},{"index":3,"count":5,"confidence":0.2140512079000473,"empirical_correctness":0.2,"signed_gap":0.014051207900047297,"absolute_gap":0.014051207900047297,"min_confidence":0.20907314121723175,"max_confidence":0.2220534384250641},{"index":4,"count":4,"confidence":0.22304505109786987,"empirical_correctness":0.25,"signed_gap":-0.026954948902130127,"absolute_gap":0.026954948902130127,"min_confidence":0.22223897278308868,"max_confidence":0.22375480830669403},{"index":5,"count":4,"confidence":0.22566724568605423,"empirical_correctness":0.25,"signed_gap":-0.02433275431394577,"absolute_gap":0.02433275431394577,"min_confidence":0.22394058108329773,"max_confidence":0.22666914761066437},{"index":6,"count":5,"confidence":0.2332111716270447,"empirical_correctness":0.0,"signed_gap":0.2332111716270447,"absolute_gap":0.2332111716270447,"min_confidence":0.2313946783542633,"max_confidence":0.23575039207935333},{"index":7,"count":4,"confidence":0.23776127770543098,"empirical_correctness":0.25,"signed_gap":-0.012238722294569016,"absolute_gap":0.012238722294569016,"min_confidence":0.235931858420372,"max_confidence":0.2395111322402954},{"index":8,"count":4,"confidence":0.2404983527958393,"empirical_correctness":0.25,"signed_gap":-0.00950164720416069,"absolute_gap":0.00950164720416069,"min_confidence":0.23990993201732635,"max_confidence":0.24162371456623077},{"index":9,"count":5,"confidence":0.24753569066524506,"empirical_correctness":0.2,"signed_gap":0.047535690665245045,"absolute_gap":0.047535690665245045,"min_confidence":0.24334004521369934,"max_confidence":0.25421127676963806}]}},"calibration_singleton_labels":{"equal_width":{"n":38,"bin_count":10,"ece":0.19665281749085375,"signed_mean_gap":0.19665281749085375,"absolute_mean_gap":0.19665281749085375,"bins":[{"index":0,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.0,"upper":0.1,"upper_inclusive":false},{"index":1,"count":1,"confidence":0.19876635074615479,"empirical_correctness":0.0,"signed_gap":0.19876635074615479,"absolute_gap":0.19876635074615479,"lower":0.1,"upper":0.2,"upper_inclusive":false},{"index":2,"count":37,"confidence":0.22362272199746724,"empirical_correctness":0.02702702702702703,"signed_gap":0.19659569497044022,"absolute_gap":0.19659569497044022,"lower":0.2,"upper":0.3,"upper_inclusive":false},{"index":3,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.3,"upper":0.4,"upper_inclusive":false},{"index":4,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.4,"upper":0.5,"upper_inclusive":false},{"index":5,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.5,"upper":0.6,"upper_inclusive":false},{"index":6,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.6,"upper":0.7,"upper_inclusive":false},{"index":7,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.7,"upper":0.8,"upper_inclusive":false},{"index":8,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.8,"upper":0.9,"upper_inclusive":false},{"index":9,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.9,"upper":1.0,"upper_inclusive":true}]},"equal_mass":{"n":38,"bin_count":10,"ece":0.19869143437398107,"signed_mean_gap":0.19665281749085375,"absolute_mean_gap":0.19665281749085375,"bins":[{"index":0,"count":3,"confidence":0.20012043913205466,"empirical_correctness":0.0,"signed_gap":0.20012043913205466,"absolute_gap":0.20012043913205466,"min_confidence":0.19876635074615479,"max_confidence":0.20139241218566895},{"index":1,"count":4,"confidence":0.20253202691674232,"empirical_correctness":0.0,"signed_gap":0.20253202691674232,"absolute_gap":0.20253202691674232,"min_confidence":0.2014317810535431,"max_confidence":0.2043667733669281},{"index":2,"count":4,"confidence":0.20492802932858467,"empirical_correctness":0.0,"signed_gap":0.20492802932858467,"absolute_gap":0.20492802932858467,"min_confidence":0.20447708666324615,"max_confidence":0.20552855730056763},{"index":3,"count":4,"confidence":0.21170159801840782,"empirical_correctness":0.0,"signed_gap":0.21170159801840782,"absolute_gap":0.21170159801840782,"min_confidence":0.2076769322156906,"max_confidence":0.21522851288318634},{"index":4,"count":4,"confidence":0.2227528728544712,"empirical_correctness":0.0,"signed_gap":0.2227528728544712,"absolute_gap":0.2227528728544712,"min_confidence":0.2220534384250641,"max_confidence":0.22375480830669403},{"index":5,"count":3,"confidence":0.22552989423274994,"empirical_correctness":0.0,"signed_gap":0.22552989423274994,"absolute_gap":0.22552989423274994,"min_confidence":0.22394058108329773,"max_confidence":0.22666914761066437},{"index":6,"count":4,"confidence":0.23257636651396751,"empirical_correctness":0.0,"signed_gap":0.23257636651396751,"absolute_gap":0.23257636651396751,"min_confidence":0.2313946783542633,"max_confidence":0.23472757637500763},{"index":7,"count":4,"confidence":0.23682109266519547,"empirical_correctness":0.0,"signed_gap":0.23682109266519547,"absolute_gap":0.23682109266519547,"min_confidence":0.23575039207935333,"max_confidence":0.23886306583881378},{"index":8,"count":4,"confidence":0.24031656980514526,"empirical_correctness":0.25,"signed_gap":-0.009683430194854736,"absolute_gap":0.009683430194854736,"min_confidence":0.2395111322402954,"max_confidence":0.24162371456623077},{"index":9,"count":4,"confidence":0.24733546003699303,"empirical_correctness":0.0,"signed_gap":0.24733546003699303,"absolute_gap":0.24733546003699303,"min_confidence":0.24334004521369934,"max_confidence":0.25421127676963806}]}},"latency_ms":{"n":43,"median":13.13478802330792,"p95":13.633277983171865},"per_action":{"keep":{"n":12,"raw":{"n":12,"acceptable":0,"unsafe":0,"review":7,"automated_count":5,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":7,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":0.5833333333333334,"automated_coverage":0.4166666666666667,"conditional_correctness":0.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.5833333333333334,"accuracy_wilson_95":{"lower":2.7755575615628914e-17,"upper":0.24249400665524085,"confidence_level":0.95}},"effective":{"n":12,"acceptable":0,"unsafe":0,"review":12,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":12,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":2.7755575615628914e-17,"upper":0.24249400665524085,"confidence_level":0.95}}},"drop":{"n":16,"raw":{"n":16,"acceptable":0,"unsafe":0,"review":8,"automated_count":8,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":8,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":0.5,"automated_coverage":0.5,"conditional_correctness":0.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.5,"accuracy_wilson_95":{"lower":0.0,"upper":0.1936076805344365,"confidence_level":0.95}},"effective":{"n":16,"acceptable":0,"unsafe":0,"review":16,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":16,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.1936076805344365,"confidence_level":0.95}}},"mask":{"n":2,"raw":{"n":2,"acceptable":1,"unsafe":0,"review":1,"automated_count":1,"automated_acceptable":1,"automated_unsafe":0,"unnecessary_reviews":1,"accuracy":0.5,"unsafe_rate":0.0,"review_rate":0.5,"automated_coverage":0.5,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.5,"accuracy_wilson_95":{"lower":0.09453120573423074,"upper":0.9054687942657693,"confidence_level":0.95}},"effective":{"n":2,"acceptable":0,"unsafe":0,"review":2,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":2,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.6576197724933469,"confidence_level":0.95}}},"pseudonymize":{"n":2,"raw":{"n":2,"acceptable":0,"unsafe":0,"review":2,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":2,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.6576197724933469,"confidence_level":0.95}},"effective":{"n":2,"acceptable":0,"unsafe":0,"review":2,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":2,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.6576197724933469,"confidence_level":0.95}}},"generalize":{"n":6,"raw":{"n":6,"acceptable":0,"unsafe":0,"review":5,"automated_count":1,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":5,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":0.8333333333333334,"automated_coverage":0.16666666666666666,"conditional_correctness":0.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.8333333333333334,"accuracy_wilson_95":{"lower":2.7755575615628914e-17,"upper":0.3903342879021653,"confidence_level":0.95}},"effective":{"n":6,"acceptable":0,"unsafe":0,"review":6,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":6,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":2.7755575615628914e-17,"upper":0.3903342879021653,"confidence_level":0.95}}},"review":{"n":0,"raw":{"n":0,"acceptable":0,"unsafe":0,"review":0,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":null,"unsafe_rate":null,"review_rate":null,"automated_coverage":null,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":null},"effective":{"n":0,"acceptable":0,"unsafe":0,"review":0,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":null,"unsafe_rate":null,"review_rate":null,"automated_coverage":null,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":null}}}},"diagnostic43-compact_no_samples":{"n":43,"raw":{"n":43,"acceptable":14,"unsafe":0,"review":12,"automated_count":31,"automated_acceptable":13,"automated_unsafe":0,"unnecessary_reviews":11,"accuracy":0.32558139534883723,"unsafe_rate":0.0,"review_rate":0.27906976744186046,"automated_coverage":0.7209302325581395,"conditional_correctness":0.41935483870967744,"conditional_unsafe":0.0,"unnecessary_review_rate":0.2558139534883721,"accuracy_wilson_95":{"lower":0.20493297126568708,"upper":0.47483789032470525,"confidence_level":0.95}},"effective":{"n":43,"acceptable":5,"unsafe":0,"review":43,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":38,"accuracy":0.11627906976744186,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":0.8837209302325582,"accuracy_wilson_95":{"lower":0.05070427739804166,"upper":0.24479161810082128,"confidence_level":0.95}},"confusion":{"raw":{"keep":{"keep":0,"drop":7,"mask":2,"pseudonymize":0,"generalize":1,"review":2},"drop":{"keep":0,"drop":6,"mask":3,"pseudonymize":0,"generalize":1,"review":6},"mask":{"keep":0,"drop":1,"mask":1,"pseudonymize":0,"generalize":0,"review":0},"pseudonymize":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":1,"review":1},"generalize":{"keep":0,"drop":2,"mask":0,"pseudonymize":0,"generalize":2,"review":2},"review":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0}},"effective":{"keep":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":12},"drop":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":16},"mask":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":2},"pseudonymize":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":2},"generalize":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":6},"review":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0}}},"probability_metrics":{"singleton_n":38,"multiacceptable_n":5,"multiacceptable_coverage":0.11627906976744186,"nll":1.8137377480428143,"nll_zero_mass_n":0,"brier":0.8396844828116412,"acceptable_set_n":43,"acceptable_set_nll":1.6727117373225258,"acceptable_set_zero_mass_n":0},"calibration_all_labels":{"equal_width":{"n":43,"bin_count":10,"ece":0.12883219712002333,"signed_mean_gap":-0.12883219712002333,"absolute_mean_gap":0.12883219712002333,"bins":[{"index":0,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.0,"upper":0.1,"upper_inclusive":false},{"index":1,"count":27,"confidence":0.19086833574153758,"empirical_correctness":0.3333333333333333,"signed_gap":-0.14246499759179573,"absolute_gap":0.14246499759179573,"lower":0.1,"upper":0.2,"upper_inclusive":false},{"index":2,"count":16,"confidence":0.20667315367609262,"empirical_correctness":0.3125,"signed_gap":-0.10582684632390738,"absolute_gap":0.10582684632390738,"lower":0.2,"upper":0.3,"upper_inclusive":false},{"index":3,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.3,"upper":0.4,"upper_inclusive":false},{"index":4,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.4,"upper":0.5,"upper_inclusive":false},{"index":5,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.5,"upper":0.6,"upper_inclusive":false},{"index":6,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.6,"upper":0.7,"upper_inclusive":false},{"index":7,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.7,"upper":0.8,"upper_inclusive":false},{"index":8,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.8,"upper":0.9,"upper_inclusive":false},{"index":9,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.9,"upper":1.0,"upper_inclusive":true}]},"equal_mass":{"n":43,"bin_count":10,"ece":0.17368454676727915,"signed_mean_gap":-0.12883219712002333,"absolute_mean_gap":0.12883219712002333,"bins":[{"index":0,"count":4,"confidence":0.18232156708836555,"empirical_correctness":0.5,"signed_gap":-0.31767843291163445,"absolute_gap":0.31767843291163445,"min_confidence":0.1799422800540924,"max_confidence":0.18392176926136017},{"index":1,"count":4,"confidence":0.18495134636759758,"empirical_correctness":0.25,"signed_gap":-0.06504865363240242,"absolute_gap":0.06504865363240242,"min_confidence":0.18420866131782532,"max_confidence":0.18592417240142822},{"index":2,"count":4,"confidence":0.18816878646612167,"empirical_correctness":0.25,"signed_gap":-0.061831213533878326,"absolute_gap":0.061831213533878326,"min_confidence":0.18761982023715973,"max_confidence":0.188643679022789},{"index":3,"count":5,"confidence":0.1918705314397812,"empirical_correctness":0.0,"signed_gap":0.1918705314397812,"absolute_gap":0.1918705314397812,"min_confidence":0.1896272450685501,"max_confidence":0.1942681223154068},{"index":4,"count":4,"confidence":0.19524797052145004,"empirical_correctness":0.75,"signed_gap":-0.55475202947855,"absolute_gap":0.55475202947855,"min_confidence":0.19484156370162964,"max_confidence":0.1957465559244156},{"index":5,"count":4,"confidence":0.19804984331130981,"empirical_correctness":0.5,"signed_gap":-0.3019501566886902,"absolute_gap":0.3019501566886902,"min_confidence":0.19603601098060608,"max_confidence":0.19951492547988892},{"index":6,"count":5,"confidence":0.2009945720434189,"empirical_correctness":0.2,"signed_gap":0.0009945720434188787,"absolute_gap":0.0009945720434188787,"min_confidence":0.19955050945281982,"max_confidence":0.20343640446662903},{"index":7,"count":4,"confidence":0.2048817127943039,"empirical_correctness":0.25,"signed_gap":-0.045118287205696106,"absolute_gap":0.045118287205696106,"min_confidence":0.20431335270404816,"max_confidence":0.20555466413497925},{"index":8,"count":4,"confidence":0.20695549249649048,"empirical_correctness":0.25,"signed_gap":-0.04304450750350952,"absolute_gap":0.04304450750350952,"min_confidence":0.20574155449867249,"max_confidence":0.20788709819316864},{"index":9,"count":5,"confidence":0.21071662604808808,"empirical_correctness":0.4,"signed_gap":-0.18928337395191194,"absolute_gap":0.18928337395191194,"min_confidence":0.20804353058338165,"max_confidence":0.21410727500915527}]}},"calibration_singleton_labels":{"equal_width":{"n":38,"bin_count":10,"ece":0.039999588147590026,"signed_mean_gap":-0.039999588147590026,"absolute_mean_gap":0.039999588147590026,"bins":[{"index":0,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.0,"upper":0.1,"upper_inclusive":false},{"index":1,"count":23,"confidence":0.19075445506883704,"empirical_correctness":0.21739130434782608,"signed_gap":-0.026636849278989044,"absolute_gap":0.026636849278989044,"lower":0.1,"upper":0.2,"upper_inclusive":false},{"index":2,"count":15,"confidence":0.2061775455872218,"empirical_correctness":0.26666666666666666,"signed_gap":-0.060489121079444874,"absolute_gap":0.060489121079444874,"lower":0.2,"upper":0.3,"upper_inclusive":false},{"index":3,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.3,"upper":0.4,"upper_inclusive":false},{"index":4,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.4,"upper":0.5,"upper_inclusive":false},{"index":5,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.5,"upper":0.6,"upper_inclusive":false},{"index":6,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.6,"upper":0.7,"upper_inclusive":false},{"index":7,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.7,"upper":0.8,"upper_inclusive":false},{"index":8,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.8,"upper":0.9,"upper_inclusive":false},{"index":9,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.9,"upper":1.0,"upper_inclusive":true}]},"equal_mass":{"n":38,"bin_count":10,"ece":0.08026719524672157,"signed_mean_gap":-0.03999958814759003,"absolute_mean_gap":0.03999958814759003,"bins":[{"index":0,"count":3,"confidence":0.1831146627664566,"empirical_correctness":0.3333333333333333,"signed_gap":-0.1502186705668767,"absolute_gap":0.1502186705668767,"min_confidence":0.1825195997953415,"max_confidence":0.18392176926136017},{"index":1,"count":4,"confidence":0.18495134636759758,"empirical_correctness":0.25,"signed_gap":-0.06504865363240242,"absolute_gap":0.06504865363240242,"min_confidence":0.18420866131782532,"max_confidence":0.18592417240142822},{"index":2,"count":4,"confidence":0.18816878646612167,"empirical_correctness":0.25,"signed_gap":-0.061831213533878326,"absolute_gap":0.061831213533878326,"min_confidence":0.18761982023715973,"max_confidence":0.188643679022789},{"index":3,"count":4,"confidence":0.1912711337208748,"empirical_correctness":0.0,"signed_gap":0.1912711337208748,"absolute_gap":0.1912711337208748,"min_confidence":0.1896272450685501,"max_confidence":0.1931816190481186},{"index":4,"count":4,"confidence":0.19575197994709015,"empirical_correctness":0.25,"signed_gap":-0.05424802005290985,"absolute_gap":0.05424802005290985,"min_confidence":0.1942681223154068,"max_confidence":0.19786222279071808},{"index":5,"count":3,"confidence":0.1992838829755783,"empirical_correctness":0.3333333333333333,"signed_gap":-0.134049450357755,"absolute_gap":0.134049450357755,"min_confidence":0.19878621399402618,"max_confidence":0.19955050945281982},{"index":6,"count":4,"confidence":0.20135558769106865,"empirical_correctness":0.25,"signed_gap":-0.04864441230893135,"absolute_gap":0.04864441230893135,"min_confidence":0.19958384335041046,"max_confidence":0.20343640446662903},{"index":7,"count":4,"confidence":0.2048817127943039,"empirical_correctness":0.25,"signed_gap":-0.045118287205696106,"absolute_gap":0.045118287205696106,"min_confidence":0.20431335270404816,"max_confidence":0.20555466413497925},{"index":8,"count":4,"confidence":0.20695549249649048,"empirical_correctness":0.25,"signed_gap":-0.04304450750350952,"absolute_gap":0.04304450750350952,"min_confidence":0.20574155449867249,"max_confidence":0.20788709819316864},{"index":9,"count":4,"confidence":0.20986896380782127,"empirical_correctness":0.25,"signed_gap":-0.040131036192178726,"absolute_gap":0.040131036192178726,"min_confidence":0.20804353058338165,"max_confidence":0.2131982445716858}]}},"latency_ms":{"n":43,"median":17.298955004662275,"p95":17.876506882021204},"per_action":{"keep":{"n":12,"raw":{"n":12,"acceptable":0,"unsafe":0,"review":2,"automated_count":10,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":2,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":0.16666666666666666,"automated_coverage":0.8333333333333334,"conditional_correctness":0.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.16666666666666666,"accuracy_wilson_95":{"lower":2.7755575615628914e-17,"upper":0.24249400665524085,"confidence_level":0.95}},"effective":{"n":12,"acceptable":0,"unsafe":0,"review":12,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":12,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":2.7755575615628914e-17,"upper":0.24249400665524085,"confidence_level":0.95}}},"drop":{"n":16,"raw":{"n":16,"acceptable":6,"unsafe":0,"review":6,"automated_count":10,"automated_acceptable":6,"automated_unsafe":0,"unnecessary_reviews":6,"accuracy":0.375,"unsafe_rate":0.0,"review_rate":0.375,"automated_coverage":0.625,"conditional_correctness":0.6,"conditional_unsafe":0.0,"unnecessary_review_rate":0.375,"accuracy_wilson_95":{"lower":0.18481232558863633,"upper":0.6135895945449727,"confidence_level":0.95}},"effective":{"n":16,"acceptable":0,"unsafe":0,"review":16,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":16,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.1936076805344365,"confidence_level":0.95}}},"mask":{"n":2,"raw":{"n":2,"acceptable":1,"unsafe":0,"review":0,"automated_count":2,"automated_acceptable":1,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":0.5,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.5,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.09453120573423074,"upper":0.9054687942657693,"confidence_level":0.95}},"effective":{"n":2,"acceptable":0,"unsafe":0,"review":2,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":2,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.6576197724933469,"confidence_level":0.95}}},"pseudonymize":{"n":2,"raw":{"n":2,"acceptable":0,"unsafe":0,"review":1,"automated_count":1,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":1,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":0.5,"automated_coverage":0.5,"conditional_correctness":0.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.5,"accuracy_wilson_95":{"lower":0.0,"upper":0.6576197724933469,"confidence_level":0.95}},"effective":{"n":2,"acceptable":0,"unsafe":0,"review":2,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":2,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.6576197724933469,"confidence_level":0.95}}},"generalize":{"n":6,"raw":{"n":6,"acceptable":2,"unsafe":0,"review":2,"automated_count":4,"automated_acceptable":2,"automated_unsafe":0,"unnecessary_reviews":2,"accuracy":0.3333333333333333,"unsafe_rate":0.0,"review_rate":0.3333333333333333,"automated_coverage":0.6666666666666666,"conditional_correctness":0.5,"conditional_unsafe":0.0,"unnecessary_review_rate":0.3333333333333333,"accuracy_wilson_95":{"lower":0.09677141110578041,"upper":0.700006684861608,"confidence_level":0.95}},"effective":{"n":6,"acceptable":0,"unsafe":0,"review":6,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":6,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":2.7755575615628914e-17,"upper":0.3903342879021653,"confidence_level":0.95}}},"review":{"n":0,"raw":{"n":0,"acceptable":0,"unsafe":0,"review":0,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":null,"unsafe_rate":null,"review_rate":null,"automated_coverage":null,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":null},"effective":{"n":0,"acceptable":0,"unsafe":0,"review":0,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":null,"unsafe_rate":null,"review_rate":null,"automated_coverage":null,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":null}}}},"challenge56-no_samples":{"n":56,"raw":{"n":56,"acceptable":13,"unsafe":0,"review":9,"automated_count":47,"automated_acceptable":12,"automated_unsafe":0,"unnecessary_reviews":8,"accuracy":0.23214285714285715,"unsafe_rate":0.0,"review_rate":0.16071428571428573,"automated_coverage":0.8392857142857143,"conditional_correctness":0.2553191489361702,"conditional_unsafe":0.0,"unnecessary_review_rate":0.14285714285714285,"accuracy_wilson_95":{"lower":0.1409937910920862,"upper":0.35768153215010856,"confidence_level":0.95}},"effective":{"n":56,"acceptable":2,"unsafe":0,"review":56,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":54,"accuracy":0.03571428571428571,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":0.9642857142857143,"accuracy_wilson_95":{"lower":0.00984942514823639,"upper":0.12118780180490107,"confidence_level":0.95}},"confusion":{"raw":{"keep":{"keep":0,"drop":0,"mask":4,"pseudonymize":0,"generalize":2,"review":2},"drop":{"keep":0,"drop":3,"mask":11,"pseudonymize":0,"generalize":3,"review":5},"mask":{"keep":0,"drop":0,"mask":8,"pseudonymize":0,"generalize":0,"review":0},"pseudonymize":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":8,"review":0},"generalize":{"keep":0,"drop":0,"mask":6,"pseudonymize":0,"generalize":1,"review":1},"review":{"keep":0,"drop":0,"mask":1,"pseudonymize":0,"generalize":0,"review":1}},"effective":{"keep":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":8},"drop":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":22},"mask":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":8},"pseudonymize":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":8},"generalize":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":8},"review":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":2}}},"probability_metrics":{"singleton_n":56,"multiacceptable_n":0,"multiacceptable_coverage":0.0,"nll":1.749378896585745,"nll_zero_mass_n":0,"brier":0.8162388346855849,"acceptable_set_n":56,"acceptable_set_nll":1.749378896585745,"acceptable_set_zero_mass_n":0},"calibration_all_labels":{"equal_width":{"n":56,"bin_count":10,"ece":0.170208590105176,"signed_mean_gap":-0.03696588666311333,"absolute_mean_gap":0.03696588666311333,"bins":[{"index":0,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.0,"upper":0.1,"upper_inclusive":false},{"index":1,"count":42,"confidence":0.18406656419947034,"empirical_correctness":0.09523809523809523,"signed_gap":0.08882846896137511,"absolute_gap":0.08882846896137511,"lower":0.1,"upper":0.2,"upper_inclusive":false},{"index":2,"count":14,"confidence":0.22850818932056427,"empirical_correctness":0.6428571428571429,"signed_gap":-0.41434895353657863,"absolute_gap":0.41434895353657863,"lower":0.2,"upper":0.3,"upper_inclusive":false},{"index":3,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.3,"upper":0.4,"upper_inclusive":false},{"index":4,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.4,"upper":0.5,"upper_inclusive":false},{"index":5,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.5,"upper":0.6,"upper_inclusive":false},{"index":6,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.6,"upper":0.7,"upper_inclusive":false},{"index":7,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.7,"upper":0.8,"upper_inclusive":false},{"index":8,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.8,"upper":0.9,"upper_inclusive":false},{"index":9,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.9,"upper":1.0,"upper_inclusive":true}]},"equal_mass":{"n":56,"bin_count":10,"ece":0.18574068934789725,"signed_mean_gap":-0.036965886663113325,"absolute_mean_gap":0.036965886663113325,"bins":[{"index":0,"count":5,"confidence":0.17699548602104187,"empirical_correctness":0.0,"signed_gap":0.17699548602104187,"absolute_gap":0.17699548602104187,"min_confidence":0.17560848593711853,"max_confidence":0.1783580780029297},{"index":1,"count":6,"confidence":0.17927241325378418,"empirical_correctness":0.0,"signed_gap":0.17927241325378418,"absolute_gap":0.17927241325378418,"min_confidence":0.17866508662700653,"max_confidence":0.17995759844779968},{"index":2,"count":5,"confidence":0.18109453320503235,"empirical_correctness":0.2,"signed_gap":-0.018905466794967657,"absolute_gap":0.018905466794967657,"min_confidence":0.18042047321796417,"max_confidence":0.18201656639575958},{"index":3,"count":6,"confidence":0.18285592397054037,"empirical_correctness":0.0,"signed_gap":0.18285592397054037,"absolute_gap":0.18285592397054037,"min_confidence":0.1823931187391281,"max_confidence":0.18362441658973694},{"index":4,"count":6,"confidence":0.18465783695379892,"empirical_correctness":0.0,"signed_gap":0.18465783695379892,"absolute_gap":0.18465783695379892,"min_confidence":0.18364548683166504,"max_confidence":0.18614615499973297},{"index":5,"count":5,"confidence":0.1871667504310608,"empirical_correctness":0.2,"signed_gap":-0.01283324956893922,"absolute_gap":0.01283324956893922,"min_confidence":0.18627935647964478,"max_confidence":0.18812550604343414},{"index":6,"count":6,"confidence":0.18943122029304504,"empirical_correctness":0.3333333333333333,"signed_gap":-0.14390211304028827,"absolute_gap":0.14390211304028827,"min_confidence":0.18831555545330048,"max_confidence":0.19174860417842865},{"index":7,"count":5,"confidence":0.19817653596401213,"empirical_correctness":0.2,"signed_gap":-0.0018234640359878762,"absolute_gap":0.0018234640359878762,"min_confidence":0.19457504153251648,"max_confidence":0.20259930193424225},{"index":8,"count":6,"confidence":0.2114422544836998,"empirical_correctness":0.3333333333333333,"signed_gap":-0.12189107884963352,"absolute_gap":0.12189107884963352,"min_confidence":0.20263038575649261,"max_confidence":0.23173515498638153},{"index":9,"count":6,"confidence":0.2544643208384514,"empirical_correctness":1.0,"signed_gap":-0.7455356791615486,"absolute_gap":0.7455356791615486,"min_confidence":0.2468193620443344,"max_confidence":0.259489506483078}]}},"calibration_singleton_labels":{"equal_width":{"n":56,"bin_count":10,"ece":0.170208590105176,"signed_mean_gap":-0.03696588666311333,"absolute_mean_gap":0.03696588666311333,"bins":[{"index":0,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.0,"upper":0.1,"upper_inclusive":false},{"index":1,"count":42,"confidence":0.18406656419947034,"empirical_correctness":0.09523809523809523,"signed_gap":0.08882846896137511,"absolute_gap":0.08882846896137511,"lower":0.1,"upper":0.2,"upper_inclusive":false},{"index":2,"count":14,"confidence":0.22850818932056427,"empirical_correctness":0.6428571428571429,"signed_gap":-0.41434895353657863,"absolute_gap":0.41434895353657863,"lower":0.2,"upper":0.3,"upper_inclusive":false},{"index":3,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.3,"upper":0.4,"upper_inclusive":false},{"index":4,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.4,"upper":0.5,"upper_inclusive":false},{"index":5,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.5,"upper":0.6,"upper_inclusive":false},{"index":6,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.6,"upper":0.7,"upper_inclusive":false},{"index":7,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.7,"upper":0.8,"upper_inclusive":false},{"index":8,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.8,"upper":0.9,"upper_inclusive":false},{"index":9,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.9,"upper":1.0,"upper_inclusive":true}]},"equal_mass":{"n":56,"bin_count":10,"ece":0.18574068934789725,"signed_mean_gap":-0.036965886663113325,"absolute_mean_gap":0.036965886663113325,"bins":[{"index":0,"count":5,"confidence":0.17699548602104187,"empirical_correctness":0.0,"signed_gap":0.17699548602104187,"absolute_gap":0.17699548602104187,"min_confidence":0.17560848593711853,"max_confidence":0.1783580780029297},{"index":1,"count":6,"confidence":0.17927241325378418,"empirical_correctness":0.0,"signed_gap":0.17927241325378418,"absolute_gap":0.17927241325378418,"min_confidence":0.17866508662700653,"max_confidence":0.17995759844779968},{"index":2,"count":5,"confidence":0.18109453320503235,"empirical_correctness":0.2,"signed_gap":-0.018905466794967657,"absolute_gap":0.018905466794967657,"min_confidence":0.18042047321796417,"max_confidence":0.18201656639575958},{"index":3,"count":6,"confidence":0.18285592397054037,"empirical_correctness":0.0,"signed_gap":0.18285592397054037,"absolute_gap":0.18285592397054037,"min_confidence":0.1823931187391281,"max_confidence":0.18362441658973694},{"index":4,"count":6,"confidence":0.18465783695379892,"empirical_correctness":0.0,"signed_gap":0.18465783695379892,"absolute_gap":0.18465783695379892,"min_confidence":0.18364548683166504,"max_confidence":0.18614615499973297},{"index":5,"count":5,"confidence":0.1871667504310608,"empirical_correctness":0.2,"signed_gap":-0.01283324956893922,"absolute_gap":0.01283324956893922,"min_confidence":0.18627935647964478,"max_confidence":0.18812550604343414},{"index":6,"count":6,"confidence":0.18943122029304504,"empirical_correctness":0.3333333333333333,"signed_gap":-0.14390211304028827,"absolute_gap":0.14390211304028827,"min_confidence":0.18831555545330048,"max_confidence":0.19174860417842865},{"index":7,"count":5,"confidence":0.19817653596401213,"empirical_correctness":0.2,"signed_gap":-0.0018234640359878762,"absolute_gap":0.0018234640359878762,"min_confidence":0.19457504153251648,"max_confidence":0.20259930193424225},{"index":8,"count":6,"confidence":0.2114422544836998,"empirical_correctness":0.3333333333333333,"signed_gap":-0.12189107884963352,"absolute_gap":0.12189107884963352,"min_confidence":0.20263038575649261,"max_confidence":0.23173515498638153},{"index":9,"count":6,"confidence":0.2544643208384514,"empirical_correctness":1.0,"signed_gap":-0.7455356791615486,"absolute_gap":0.7455356791615486,"min_confidence":0.2468193620443344,"max_confidence":0.259489506483078}]}},"latency_ms":{"n":56,"median":12.152413983130828,"p95":12.834358727559447},"per_action":{"keep":{"n":8,"raw":{"n":8,"acceptable":0,"unsafe":0,"review":2,"automated_count":6,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":2,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":0.25,"automated_coverage":0.75,"conditional_correctness":0.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.25,"accuracy_wilson_95":{"lower":0.0,"upper":0.32440756488388023,"confidence_level":0.95}},"effective":{"n":8,"acceptable":0,"unsafe":0,"review":8,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":8,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.32440756488388023,"confidence_level":0.95}}},"drop":{"n":22,"raw":{"n":22,"acceptable":3,"unsafe":0,"review":5,"automated_count":17,"automated_acceptable":3,"automated_unsafe":0,"unnecessary_reviews":5,"accuracy":0.13636363636363635,"unsafe_rate":0.0,"review_rate":0.22727272727272727,"automated_coverage":0.7727272727272727,"conditional_correctness":0.17647058823529413,"conditional_unsafe":0.0,"unnecessary_review_rate":0.22727272727272727,"accuracy_wilson_95":{"lower":0.04749003714055253,"upper":0.33334987166571717,"confidence_level":0.95}},"effective":{"n":22,"acceptable":0,"unsafe":0,"review":22,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":22,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.14865487460862087,"confidence_level":0.95}}},"mask":{"n":8,"raw":{"n":8,"acceptable":8,"unsafe":0,"review":0,"automated_count":8,"automated_acceptable":8,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":1.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.6755924351161198,"upper":1.0,"confidence_level":0.95}},"effective":{"n":8,"acceptable":0,"unsafe":0,"review":8,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":8,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.32440756488388023,"confidence_level":0.95}}},"pseudonymize":{"n":8,"raw":{"n":8,"acceptable":0,"unsafe":0,"review":0,"automated_count":8,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.32440756488388023,"confidence_level":0.95}},"effective":{"n":8,"acceptable":0,"unsafe":0,"review":8,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":8,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.32440756488388023,"confidence_level":0.95}}},"generalize":{"n":8,"raw":{"n":8,"acceptable":1,"unsafe":0,"review":1,"automated_count":7,"automated_acceptable":1,"automated_unsafe":0,"unnecessary_reviews":1,"accuracy":0.125,"unsafe_rate":0.0,"review_rate":0.125,"automated_coverage":0.875,"conditional_correctness":0.14285714285714285,"conditional_unsafe":0.0,"unnecessary_review_rate":0.125,"accuracy_wilson_95":{"lower":0.02241749145005667,"upper":0.4708881822128535,"confidence_level":0.95}},"effective":{"n":8,"acceptable":0,"unsafe":0,"review":8,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":8,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.32440756488388023,"confidence_level":0.95}}},"review":{"n":2,"raw":{"n":2,"acceptable":1,"unsafe":0,"review":1,"automated_count":1,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":0.5,"unsafe_rate":0.0,"review_rate":0.5,"automated_coverage":0.5,"conditional_correctness":0.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.09453120573423074,"upper":0.9054687942657693,"confidence_level":0.95}},"effective":{"n":2,"acceptable":2,"unsafe":0,"review":2,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":1.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.34238022750665303,"upper":1.0,"confidence_level":0.95}}}}},"challenge56-compact_no_samples":{"n":56,"raw":{"n":56,"acceptable":2,"unsafe":0,"review":56,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":54,"accuracy":0.03571428571428571,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":0.9642857142857143,"accuracy_wilson_95":{"lower":0.00984942514823639,"upper":0.12118780180490107,"confidence_level":0.95}},"effective":{"n":56,"acceptable":2,"unsafe":0,"review":56,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":54,"accuracy":0.03571428571428571,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":0.9642857142857143,"accuracy_wilson_95":{"lower":0.00984942514823639,"upper":0.12118780180490107,"confidence_level":0.95}},"confusion":{"raw":{"keep":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":8},"drop":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":22},"mask":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":8},"pseudonymize":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":8},"generalize":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":8},"review":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":2}},"effective":{"keep":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":8},"drop":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":22},"mask":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":8},"pseudonymize":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":8},"generalize":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":8},"review":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":2}}},"probability_metrics":{"singleton_n":56,"multiacceptable_n":0,"multiacceptable_coverage":0.0,"nll":1.8040135715548637,"nll_zero_mass_n":0,"brier":0.8387177326085066,"acceptable_set_n":56,"acceptable_set_nll":1.8040135715548637,"acceptable_set_zero_mass_n":0},"calibration_all_labels":{"equal_width":{"n":56,"bin_count":10,"ece":0.19975294172763824,"signed_mean_gap":0.17672913361872947,"absolute_mean_gap":0.17672913361872947,"bins":[{"index":0,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.0,"upper":0.1,"upper_inclusive":false},{"index":1,"count":7,"confidence":0.19361905327865056,"empirical_correctness":0.2857142857142857,"signed_gap":-0.09209523243563514,"absolute_gap":0.09209523243563514,"lower":0.1,"upper":0.2,"upper_inclusive":false},{"index":2,"count":49,"confidence":0.21513261448363868,"empirical_correctness":0.0,"signed_gap":0.21513261448363868,"absolute_gap":0.21513261448363868,"lower":0.2,"upper":0.3,"upper_inclusive":false},{"index":3,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.3,"upper":0.4,"upper_inclusive":false},{"index":4,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.4,"upper":0.5,"upper_inclusive":false},{"index":5,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.5,"upper":0.6,"upper_inclusive":false},{"index":6,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.6,"upper":0.7,"upper_inclusive":false},{"index":7,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.7,"upper":0.8,"upper_inclusive":false},{"index":8,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.8,"upper":0.9,"upper_inclusive":false},{"index":9,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.9,"upper":1.0,"upper_inclusive":true}]},"equal_mass":{"n":56,"bin_count":10,"ece":0.2139663121529988,"signed_mean_gap":0.17672913361872947,"absolute_mean_gap":0.17672913361872947,"bins":[{"index":0,"count":5,"confidence":0.19147180020809174,"empirical_correctness":0.4,"signed_gap":-0.2085281997919083,"absolute_gap":0.2085281997919083,"min_confidence":0.18355035781860352,"max_confidence":0.19584901630878448},{"index":1,"count":6,"confidence":0.20131356020768484,"empirical_correctness":0.0,"signed_gap":0.20131356020768484,"absolute_gap":0.20131356020768484,"min_confidence":0.198141947388649,"max_confidence":0.2044960856437683},{"index":2,"count":5,"confidence":0.20518094599246978,"empirical_correctness":0.0,"signed_gap":0.20518094599246978,"absolute_gap":0.20518094599246978,"min_confidence":0.20453213155269623,"max_confidence":0.20605824887752533},{"index":3,"count":6,"confidence":0.20788060128688812,"empirical_correctness":0.0,"signed_gap":0.20788060128688812,"absolute_gap":0.20788060128688812,"min_confidence":0.20658713579177856,"max_confidence":0.2094603329896927},{"index":4,"count":6,"confidence":0.21078563233216605,"empirical_correctness":0.0,"signed_gap":0.21078563233216605,"absolute_gap":0.21078563233216605,"min_confidence":0.21034251153469086,"max_confidence":0.2118794023990631},{"index":5,"count":5,"confidence":0.21310669779777527,"empirical_correctness":0.0,"signed_gap":0.21310669779777527,"absolute_gap":0.21310669779777527,"min_confidence":0.2121003121137619,"max_confidence":0.21367968618869781},{"index":6,"count":6,"confidence":0.21476980795462927,"empirical_correctness":0.0,"signed_gap":0.21476980795462927,"absolute_gap":0.21476980795462927,"min_confidence":0.21397046744823456,"max_confidence":0.21606473624706268},{"index":7,"count":5,"confidence":0.22014724016189574,"empirical_correctness":0.0,"signed_gap":0.22014724016189574,"absolute_gap":0.22014724016189574,"min_confidence":0.21832385659217834,"max_confidence":0.22225502133369446},{"index":8,"count":6,"confidence":0.2257355252901713,"empirical_correctness":0.0,"signed_gap":0.2257355252901713,"absolute_gap":0.2257355252901713,"min_confidence":0.2232498973608017,"max_confidence":0.22769489884376526},{"index":9,"count":6,"confidence":0.23073121656974158,"empirical_correctness":0.0,"signed_gap":0.23073121656974158,"absolute_gap":0.23073121656974158,"min_confidence":0.22885578870773315,"max_confidence":0.23504501581192017}]}},"calibration_singleton_labels":{"equal_width":{"n":56,"bin_count":10,"ece":0.19975294172763824,"signed_mean_gap":0.17672913361872947,"absolute_mean_gap":0.17672913361872947,"bins":[{"index":0,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.0,"upper":0.1,"upper_inclusive":false},{"index":1,"count":7,"confidence":0.19361905327865056,"empirical_correctness":0.2857142857142857,"signed_gap":-0.09209523243563514,"absolute_gap":0.09209523243563514,"lower":0.1,"upper":0.2,"upper_inclusive":false},{"index":2,"count":49,"confidence":0.21513261448363868,"empirical_correctness":0.0,"signed_gap":0.21513261448363868,"absolute_gap":0.21513261448363868,"lower":0.2,"upper":0.3,"upper_inclusive":false},{"index":3,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.3,"upper":0.4,"upper_inclusive":false},{"index":4,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.4,"upper":0.5,"upper_inclusive":false},{"index":5,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.5,"upper":0.6,"upper_inclusive":false},{"index":6,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.6,"upper":0.7,"upper_inclusive":false},{"index":7,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.7,"upper":0.8,"upper_inclusive":false},{"index":8,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.8,"upper":0.9,"upper_inclusive":false},{"index":9,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.9,"upper":1.0,"upper_inclusive":true}]},"equal_mass":{"n":56,"bin_count":10,"ece":0.2139663121529988,"signed_mean_gap":0.17672913361872947,"absolute_mean_gap":0.17672913361872947,"bins":[{"index":0,"count":5,"confidence":0.19147180020809174,"empirical_correctness":0.4,"signed_gap":-0.2085281997919083,"absolute_gap":0.2085281997919083,"min_confidence":0.18355035781860352,"max_confidence":0.19584901630878448},{"index":1,"count":6,"confidence":0.20131356020768484,"empirical_correctness":0.0,"signed_gap":0.20131356020768484,"absolute_gap":0.20131356020768484,"min_confidence":0.198141947388649,"max_confidence":0.2044960856437683},{"index":2,"count":5,"confidence":0.20518094599246978,"empirical_correctness":0.0,"signed_gap":0.20518094599246978,"absolute_gap":0.20518094599246978,"min_confidence":0.20453213155269623,"max_confidence":0.20605824887752533},{"index":3,"count":6,"confidence":0.20788060128688812,"empirical_correctness":0.0,"signed_gap":0.20788060128688812,"absolute_gap":0.20788060128688812,"min_confidence":0.20658713579177856,"max_confidence":0.2094603329896927},{"index":4,"count":6,"confidence":0.21078563233216605,"empirical_correctness":0.0,"signed_gap":0.21078563233216605,"absolute_gap":0.21078563233216605,"min_confidence":0.21034251153469086,"max_confidence":0.2118794023990631},{"index":5,"count":5,"confidence":0.21310669779777527,"empirical_correctness":0.0,"signed_gap":0.21310669779777527,"absolute_gap":0.21310669779777527,"min_confidence":0.2121003121137619,"max_confidence":0.21367968618869781},{"index":6,"count":6,"confidence":0.21476980795462927,"empirical_correctness":0.0,"signed_gap":0.21476980795462927,"absolute_gap":0.21476980795462927,"min_confidence":0.21397046744823456,"max_confidence":0.21606473624706268},{"index":7,"count":5,"confidence":0.22014724016189574,"empirical_correctness":0.0,"signed_gap":0.22014724016189574,"absolute_gap":0.22014724016189574,"min_confidence":0.21832385659217834,"max_confidence":0.22225502133369446},{"index":8,"count":6,"confidence":0.2257355252901713,"empirical_correctness":0.0,"signed_gap":0.2257355252901713,"absolute_gap":0.2257355252901713,"min_confidence":0.2232498973608017,"max_confidence":0.22769489884376526},{"index":9,"count":6,"confidence":0.23073121656974158,"empirical_correctness":0.0,"signed_gap":0.23073121656974158,"absolute_gap":0.23073121656974158,"min_confidence":0.22885578870773315,"max_confidence":0.23504501581192017}]}},"latency_ms":{"n":56,"median":13.224193011410534,"p95":14.30137423449196},"per_action":{"keep":{"n":8,"raw":{"n":8,"acceptable":0,"unsafe":0,"review":8,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":8,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.32440756488388023,"confidence_level":0.95}},"effective":{"n":8,"acceptable":0,"unsafe":0,"review":8,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":8,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.32440756488388023,"confidence_level":0.95}}},"drop":{"n":22,"raw":{"n":22,"acceptable":0,"unsafe":0,"review":22,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":22,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.14865487460862087,"confidence_level":0.95}},"effective":{"n":22,"acceptable":0,"unsafe":0,"review":22,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":22,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.14865487460862087,"confidence_level":0.95}}},"mask":{"n":8,"raw":{"n":8,"acceptable":0,"unsafe":0,"review":8,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":8,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.32440756488388023,"confidence_level":0.95}},"effective":{"n":8,"acceptable":0,"unsafe":0,"review":8,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":8,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.32440756488388023,"confidence_level":0.95}}},"pseudonymize":{"n":8,"raw":{"n":8,"acceptable":0,"unsafe":0,"review":8,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":8,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.32440756488388023,"confidence_level":0.95}},"effective":{"n":8,"acceptable":0,"unsafe":0,"review":8,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":8,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.32440756488388023,"confidence_level":0.95}}},"generalize":{"n":8,"raw":{"n":8,"acceptable":0,"unsafe":0,"review":8,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":8,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.32440756488388023,"confidence_level":0.95}},"effective":{"n":8,"acceptable":0,"unsafe":0,"review":8,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":8,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.32440756488388023,"confidence_level":0.95}}},"review":{"n":2,"raw":{"n":2,"acceptable":2,"unsafe":0,"review":2,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":1.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.34238022750665303,"upper":1.0,"confidence_level":0.95}},"effective":{"n":2,"acceptable":2,"unsafe":0,"review":2,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":1.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.34238022750665303,"upper":1.0,"confidence_level":0.95}}}}}},"composite":{"n":198,"scope":"descriptive sum across four correlated panels","raw":{"n":198,"acceptable":35,"unsafe":0,"review":103,"automated_count":95,"automated_acceptable":28,"automated_unsafe":0,"unnecessary_reviews":96,"accuracy":0.17676767676767677,"unsafe_rate":0.0,"review_rate":0.5202020202020202,"automated_coverage":0.4797979797979798,"conditional_correctness":0.29473684210526313,"conditional_unsafe":0.0,"unnecessary_review_rate":0.48484848484848486},"effective":{"n":198,"acceptable":14,"unsafe":0,"review":198,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":184,"accuracy":0.0707070707070707,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":0.9292929292929293}}},{"id":"Arc53-native-v4","platform":"Local","model_id":"Arc53/arc-decide-0.6b-v0.1","revision":"8c61f24c50f8093ad6066683c0886ef660aa4ce0","loading":"float16 / none","scoring_kind":"Native decision readout","settings":{"dtype":"float16","quantization":"none","max_tokens":4096,"effective_max_tokens":4096,"temperature":1.4125375446227555,"trust_remote_code":false,"local_files_only":true,"use_safetensors":true,"guard_threshold":0.45,"seed":null,"tf32":null},"scoring_metadata":{"backend":"arc_decision","protocol":"hf-local-sweep-arc-v1","probability_source":"native_choice_probability_unrounded_generic_temperature_not_privacy_calibrated","latency_scope":"synchronous_forward_scoring_including_transfer_excluding_tokenization","equal_mass_convention":"rank bins with floor boundaries at floor(i*n/k), k=min(10,n//2); minimum two rows"},"load_ms":682.1363599738106,"peak_allocated_bytes":1667047936,"peak_reserved_bytes":2057306112,"panels":{"diagnostic43-no_samples":{"n":43,"raw":{"n":43,"acceptable":13,"unsafe":3,"review":0,"automated_count":43,"automated_acceptable":13,"automated_unsafe":3,"unnecessary_reviews":0,"accuracy":0.3023255813953488,"unsafe_rate":0.06976744186046512,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.3023255813953488,"conditional_unsafe":0.06976744186046512,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.1860200013584818,"upper":0.45105364177729607,"confidence_level":0.95}},"effective":{"n":43,"acceptable":5,"unsafe":0,"review":43,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":38,"accuracy":0.11627906976744186,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":0.8837209302325582,"accuracy_wilson_95":{"lower":0.05070427739804166,"upper":0.24479161810082128,"confidence_level":0.95}},"confusion":{"raw":{"keep":{"keep":0,"drop":3,"mask":8,"pseudonymize":1,"generalize":0,"review":0},"drop":{"keep":0,"drop":5,"mask":9,"pseudonymize":2,"generalize":0,"review":0},"mask":{"keep":0,"drop":0,"mask":2,"pseudonymize":0,"generalize":0,"review":0},"pseudonymize":{"keep":0,"drop":0,"mask":0,"pseudonymize":2,"generalize":0,"review":0},"generalize":{"keep":0,"drop":3,"mask":1,"pseudonymize":2,"generalize":0,"review":0},"review":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0}},"effective":{"keep":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":12},"drop":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":16},"mask":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":2},"pseudonymize":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":2},"generalize":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":6},"review":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0}}},"probability_metrics":{"singleton_n":38,"multiacceptable_n":5,"multiacceptable_coverage":0.11627906976744186,"nll":1.7255516254053513,"nll_zero_mass_n":0,"brier":0.8266853686731074,"acceptable_set_n":43,"acceptable_set_nll":1.5766112096479796,"acceptable_set_zero_mass_n":0},"calibration_all_labels":{"equal_width":{"n":43,"bin_count":10,"ece":0.11663615149120952,"signed_mean_gap":0.04362920273182003,"absolute_mean_gap":0.04362920273182003,"bins":[{"index":0,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.0,"upper":0.1,"upper_inclusive":false},{"index":1,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.1,"upper":0.2,"upper_inclusive":false},{"index":2,"count":6,"confidence":0.29014670848846436,"empirical_correctness":0.5,"signed_gap":-0.20985329151153564,"absolute_gap":0.20985329151153564,"lower":0.2,"upper":0.3,"upper_inclusive":false},{"index":3,"count":33,"confidence":0.34683954896348895,"empirical_correctness":0.24242424242424243,"signed_gap":0.10441530653924652,"absolute_gap":0.10441530653924652,"lower":0.3,"upper":0.4,"upper_inclusive":false},{"index":4,"count":4,"confidence":0.422367587685585,"empirical_correctness":0.5,"signed_gap":-0.07763241231441498,"absolute_gap":0.07763241231441498,"lower":0.4,"upper":0.5,"upper_inclusive":false},{"index":5,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.5,"upper":0.6,"upper_inclusive":false},{"index":6,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.6,"upper":0.7,"upper_inclusive":false},{"index":7,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.7,"upper":0.8,"upper_inclusive":false},{"index":8,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.8,"upper":0.9,"upper_inclusive":false},{"index":9,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.9,"upper":1.0,"upper_inclusive":true}]},"equal_mass":{"n":43,"bin_count":10,"ece":0.21106912923413654,"signed_mean_gap":0.04362920273182004,"absolute_mean_gap":0.04362920273182004,"bins":[{"index":0,"count":4,"confidence":0.287159763276577,"empirical_correctness":0.75,"signed_gap":-0.462840236723423,"absolute_gap":0.462840236723423,"min_confidence":0.2827463150024414,"max_confidence":0.2912598252296448},{"index":1,"count":4,"confidence":0.3017519414424896,"empirical_correctness":0.25,"signed_gap":0.051751941442489624,"absolute_gap":0.051751941442489624,"min_confidence":0.2924439013004303,"max_confidence":0.3104417026042938},{"index":2,"count":4,"confidence":0.31589552015066147,"empirical_correctness":0.25,"signed_gap":0.06589552015066147,"absolute_gap":0.06589552015066147,"min_confidence":0.31351685523986816,"max_confidence":0.3185324966907501},{"index":3,"count":5,"confidence":0.3280922114849091,"empirical_correctness":0.0,"signed_gap":0.3280922114849091,"absolute_gap":0.3280922114849091,"min_confidence":0.3230893611907959,"max_confidence":0.33428746461868286},{"index":4,"count":4,"confidence":0.3384741023182869,"empirical_correctness":0.0,"signed_gap":0.3384741023182869,"absolute_gap":0.3384741023182869,"min_confidence":0.3358688950538635,"max_confidence":0.3433215022087097},{"index":5,"count":4,"confidence":0.3466109558939934,"empirical_correctness":0.25,"signed_gap":0.09661095589399338,"absolute_gap":0.09661095589399338,"min_confidence":0.3445439338684082,"max_confidence":0.3487171530723572},{"index":6,"count":5,"confidence":0.3539267063140869,"empirical_correctness":0.6,"signed_gap":-0.24607329368591307,"absolute_gap":0.24607329368591307,"min_confidence":0.34927698969841003,"max_confidence":0.36372995376586914},{"index":7,"count":4,"confidence":0.3704422488808632,"empirical_correctness":0.5,"signed_gap":-0.1295577511191368,"absolute_gap":0.1295577511191368,"min_confidence":0.36491063237190247,"max_confidence":0.37302374839782715},{"index":8,"count":4,"confidence":0.38542329519987106,"empirical_correctness":0.0,"signed_gap":0.38542329519987106,"absolute_gap":0.38542329519987106,"min_confidence":0.37500548362731934,"max_confidence":0.39319607615470886},{"index":9,"count":5,"confidence":0.4165859639644623,"empirical_correctness":0.4,"signed_gap":0.01658596396446227,"absolute_gap":0.01658596396446227,"min_confidence":0.3934594690799713,"max_confidence":0.428832471370697}]}},"calibration_singleton_labels":{"equal_width":{"n":38,"bin_count":10,"ece":0.1726572152815367,"signed_mean_gap":0.10638775480420966,"absolute_mean_gap":0.10638775480420966,"bins":[{"index":0,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.0,"upper":0.1,"upper_inclusive":false},{"index":1,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.1,"upper":0.2,"upper_inclusive":false},{"index":2,"count":6,"confidence":0.29014670848846436,"empirical_correctness":0.5,"signed_gap":-0.20985329151153564,"absolute_gap":0.20985329151153564,"lower":0.2,"upper":0.3,"upper_inclusive":false},{"index":3,"count":29,"confidence":0.3462488466295703,"empirical_correctness":0.1724137931034483,"signed_gap":0.173835053526122,"absolute_gap":0.173835053526122,"lower":0.3,"upper":0.4,"upper_inclusive":false},{"index":4,"count":3,"confidence":0.42021262645721436,"empirical_correctness":0.3333333333333333,"signed_gap":0.08687929312388104,"absolute_gap":0.08687929312388104,"lower":0.4,"upper":0.5,"upper_inclusive":false},{"index":5,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.5,"upper":0.6,"upper_inclusive":false},{"index":6,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.6,"upper":0.7,"upper_inclusive":false},{"index":7,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.7,"upper":0.8,"upper_inclusive":false},{"index":8,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.8,"upper":0.9,"upper_inclusive":false},{"index":9,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.9,"upper":1.0,"upper_inclusive":true}]},"equal_mass":{"n":38,"bin_count":10,"ece":0.2791532936849092,"signed_mean_gap":0.10638775480420966,"absolute_mean_gap":0.10638775480420966,"bins":[{"index":0,"count":3,"confidence":0.28579307595888775,"empirical_correctness":1.0,"signed_gap":-0.7142069240411122,"absolute_gap":0.7142069240411122,"min_confidence":0.2827463150024414,"max_confidence":0.2880402207374573},{"index":1,"count":4,"confidence":0.29695647209882736,"empirical_correctness":0.25,"signed_gap":0.04695647209882736,"absolute_gap":0.04695647209882736,"min_confidence":0.2912598252296448,"max_confidence":0.3043248653411865},{"index":2,"count":4,"confidence":0.314312644302845,"empirical_correctness":0.0,"signed_gap":0.314312644302845,"absolute_gap":0.314312644302845,"min_confidence":0.3104417026042938,"max_confidence":0.3185324966907501},{"index":3,"count":4,"confidence":0.3265433982014656,"empirical_correctness":0.0,"signed_gap":0.3265433982014656,"absolute_gap":0.3265433982014656,"min_confidence":0.3230893611907959,"max_confidence":0.3340860605239868},{"index":4,"count":4,"confidence":0.3362155929207802,"empirical_correctness":0.0,"signed_gap":0.3362155929207802,"absolute_gap":0.3362155929207802,"min_confidence":0.33428746461868286,"max_confidence":0.3379892110824585},{"index":5,"count":3,"confidence":0.3444589873154958,"empirical_correctness":0.0,"signed_gap":0.3444589873154958,"absolute_gap":0.3444589873154958,"min_confidence":0.3433215022087097,"max_confidence":0.3455115258693695},{"index":6,"count":4,"confidence":0.3494590073823929,"empirical_correctness":0.5,"signed_gap":-0.15054099261760712,"absolute_gap":0.15054099261760712,"min_confidence":0.3476712107658386,"max_confidence":0.3521706759929657},{"index":7,"count":4,"confidence":0.3655598759651184,"empirical_correctness":0.5,"signed_gap":-0.1344401240348816,"absolute_gap":0.1344401240348816,"min_confidence":0.353494256734848,"max_confidence":0.37298262119293213},{"index":8,"count":4,"confidence":0.38542329519987106,"empirical_correctness":0.0,"signed_gap":0.38542329519987106,"absolute_gap":0.38542329519987106,"min_confidence":0.37500548362731934,"max_confidence":0.39319607615470886},{"index":9,"count":4,"confidence":0.4135243371129036,"empirical_correctness":0.25,"signed_gap":0.1635243371129036,"absolute_gap":0.1635243371129036,"min_confidence":0.3934594690799713,"max_confidence":0.42568913102149963}]}},"latency_ms":{"n":43,"median":13.036797987297177,"p95":13.696058082859963},"per_action":{"keep":{"n":12,"raw":{"n":12,"acceptable":0,"unsafe":0,"review":0,"automated_count":12,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":2.7755575615628914e-17,"upper":0.24249400665524085,"confidence_level":0.95}},"effective":{"n":12,"acceptable":0,"unsafe":0,"review":12,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":12,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":2.7755575615628914e-17,"upper":0.24249400665524085,"confidence_level":0.95}}},"drop":{"n":16,"raw":{"n":16,"acceptable":5,"unsafe":2,"review":0,"automated_count":16,"automated_acceptable":5,"automated_unsafe":2,"unnecessary_reviews":0,"accuracy":0.3125,"unsafe_rate":0.125,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.3125,"conditional_unsafe":0.125,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.14164643854782039,"upper":0.5559564416525933,"confidence_level":0.95}},"effective":{"n":16,"acceptable":0,"unsafe":0,"review":16,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":16,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.1936076805344365,"confidence_level":0.95}}},"mask":{"n":2,"raw":{"n":2,"acceptable":2,"unsafe":0,"review":0,"automated_count":2,"automated_acceptable":2,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":1.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.34238022750665303,"upper":1.0,"confidence_level":0.95}},"effective":{"n":2,"acceptable":0,"unsafe":0,"review":2,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":2,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.6576197724933469,"confidence_level":0.95}}},"pseudonymize":{"n":2,"raw":{"n":2,"acceptable":2,"unsafe":0,"review":0,"automated_count":2,"automated_acceptable":2,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":1.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.34238022750665303,"upper":1.0,"confidence_level":0.95}},"effective":{"n":2,"acceptable":0,"unsafe":0,"review":2,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":2,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.6576197724933469,"confidence_level":0.95}}},"generalize":{"n":6,"raw":{"n":6,"acceptable":0,"unsafe":0,"review":0,"automated_count":6,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":2.7755575615628914e-17,"upper":0.3903342879021653,"confidence_level":0.95}},"effective":{"n":6,"acceptable":0,"unsafe":0,"review":6,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":6,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":2.7755575615628914e-17,"upper":0.3903342879021653,"confidence_level":0.95}}},"review":{"n":0,"raw":{"n":0,"acceptable":0,"unsafe":0,"review":0,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":null,"unsafe_rate":null,"review_rate":null,"automated_coverage":null,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":null},"effective":{"n":0,"acceptable":0,"unsafe":0,"review":0,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":null,"unsafe_rate":null,"review_rate":null,"automated_coverage":null,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":null}}}},"diagnostic43-compact_no_samples":{"n":43,"raw":{"n":43,"acceptable":19,"unsafe":0,"review":0,"automated_count":43,"automated_acceptable":19,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":0.4418604651162791,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.4418604651162791,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.30433309839170514,"upper":0.5889238554717591,"confidence_level":0.95}},"effective":{"n":43,"acceptable":5,"unsafe":0,"review":43,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":38,"accuracy":0.11627906976744186,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":0.8837209302325582,"accuracy_wilson_95":{"lower":0.05070427739804166,"upper":0.24479161810082128,"confidence_level":0.95}},"confusion":{"raw":{"keep":{"keep":1,"drop":4,"mask":7,"pseudonymize":0,"generalize":0,"review":0},"drop":{"keep":0,"drop":10,"mask":6,"pseudonymize":0,"generalize":0,"review":0},"mask":{"keep":0,"drop":0,"mask":2,"pseudonymize":0,"generalize":0,"review":0},"pseudonymize":{"keep":0,"drop":1,"mask":0,"pseudonymize":1,"generalize":0,"review":0},"generalize":{"keep":0,"drop":4,"mask":2,"pseudonymize":0,"generalize":0,"review":0},"review":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0}},"effective":{"keep":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":12},"drop":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":16},"mask":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":2},"pseudonymize":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":2},"generalize":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":6},"review":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0}}},"probability_metrics":{"singleton_n":38,"multiacceptable_n":5,"multiacceptable_coverage":0.11627906976744186,"nll":1.5102641072489196,"nll_zero_mass_n":0,"brier":0.7368808628046937,"acceptable_set_n":43,"acceptable_set_nll":1.3876529892316258,"acceptable_set_zero_mass_n":0},"calibration_all_labels":{"equal_width":{"n":43,"bin_count":10,"ece":0.16226001811581986,"signed_mean_gap":-0.11318373680114745,"absolute_mean_gap":0.11318373680114745,"bins":[{"index":0,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.0,"upper":0.1,"upper_inclusive":false},{"index":1,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.1,"upper":0.2,"upper_inclusive":false},{"index":2,"count":11,"confidence":0.27774000438776886,"empirical_correctness":0.18181818181818182,"signed_gap":0.09592182256958703,"absolute_gap":0.09592182256958703,"lower":0.2,"upper":0.3,"upper_inclusive":false},{"index":3,"count":31,"confidence":0.3441295248846854,"empirical_correctness":0.5161290322580645,"signed_gap":-0.1719995073733791,"absolute_gap":0.1719995073733791,"lower":0.3,"upper":0.4,"upper_inclusive":false},{"index":4,"count":1,"confidence":0.40994399785995483,"empirical_correctness":1.0,"signed_gap":-0.5900560021400452,"absolute_gap":0.5900560021400452,"lower":0.4,"upper":0.5,"upper_inclusive":false},{"index":5,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.5,"upper":0.6,"upper_inclusive":false},{"index":6,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.6,"upper":0.7,"upper_inclusive":false},{"index":7,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.7,"upper":0.8,"upper_inclusive":false},{"index":8,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.8,"upper":0.9,"upper_inclusive":false},{"index":9,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.9,"upper":1.0,"upper_inclusive":true}]},"equal_mass":{"n":43,"bin_count":10,"ece":0.20058731422867887,"signed_mean_gap":-0.11318373680114746,"absolute_mean_gap":0.11318373680114746,"bins":[{"index":0,"count":4,"confidence":0.2613106966018677,"empirical_correctness":0.0,"signed_gap":0.2613106966018677,"absolute_gap":0.2613106966018677,"min_confidence":0.25140494108200073,"max_confidence":0.27007511258125305},{"index":1,"count":4,"confidence":0.2817223370075226,"empirical_correctness":0.25,"signed_gap":0.03172233700752258,"absolute_gap":0.03172233700752258,"min_confidence":0.27169176936149597,"max_confidence":0.29093846678733826},{"index":2,"count":4,"confidence":0.29603710025548935,"empirical_correctness":0.25,"signed_gap":0.04603710025548935,"absolute_gap":0.04603710025548935,"min_confidence":0.29339343309402466,"max_confidence":0.3011404871940613},{"index":3,"count":5,"confidence":0.3045792758464813,"empirical_correctness":0.2,"signed_gap":0.1045792758464813,"absolute_gap":0.1045792758464813,"min_confidence":0.3022262156009674,"max_confidence":0.30963218212127686},{"index":4,"count":4,"confidence":0.32088982313871384,"empirical_correctness":0.5,"signed_gap":-0.17911017686128616,"absolute_gap":0.17911017686128616,"min_confidence":0.3162144124507904,"max_confidence":0.32540708780288696},{"index":5,"count":4,"confidence":0.32831592857837677,"empirical_correctness":0.5,"signed_gap":-0.17168407142162323,"absolute_gap":0.17168407142162323,"min_confidence":0.3262646198272705,"max_confidence":0.33013060688972473},{"index":6,"count":5,"confidence":0.3456520438194275,"empirical_correctness":0.6,"signed_gap":-0.2543479561805725,"absolute_gap":0.2543479561805725,"min_confidence":0.33446136116981506,"max_confidence":0.3581322729587555},{"index":7,"count":4,"confidence":0.3651752695441246,"empirical_correctness":0.75,"signed_gap":-0.3848247304558754,"absolute_gap":0.3848247304558754,"min_confidence":0.3609054982662201,"max_confidence":0.36981669068336487},{"index":8,"count":4,"confidence":0.3727237954735756,"empirical_correctness":0.75,"signed_gap":-0.3772762045264244,"absolute_gap":0.3772762045264244,"min_confidence":0.3703886568546295,"max_confidence":0.3758070766925812},{"index":9,"count":5,"confidence":0.3954485833644867,"empirical_correctness":0.6,"signed_gap":-0.2045514166355133,"absolute_gap":0.2045514166355133,"min_confidence":0.38436657190322876,"max_confidence":0.40994399785995483}]}},"calibration_singleton_labels":{"equal_width":{"n":38,"bin_count":10,"ece":0.16883941543729683,"signed_mean_gap":-0.03877309905855279,"absolute_mean_gap":0.03877309905855279,"bins":[{"index":0,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.0,"upper":0.1,"upper_inclusive":false},{"index":1,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.1,"upper":0.2,"upper_inclusive":false},{"index":2,"count":9,"confidence":0.27458444568845963,"empirical_correctness":0.0,"signed_gap":0.27458444568845963,"absolute_gap":0.27458444568845963,"lower":0.2,"upper":0.3,"upper_inclusive":false},{"index":3,"count":28,"confidence":0.344479222382818,"empirical_correctness":0.4642857142857143,"signed_gap":-0.11980649190289633,"absolute_gap":0.11980649190289633,"lower":0.3,"upper":0.4,"upper_inclusive":false},{"index":4,"count":1,"confidence":0.40994399785995483,"empirical_correctness":1.0,"signed_gap":-0.5900560021400452,"absolute_gap":0.5900560021400452,"lower":0.4,"upper":0.5,"upper_inclusive":false},{"index":5,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.5,"upper":0.6,"upper_inclusive":false},{"index":6,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.6,"upper":0.7,"upper_inclusive":false},{"index":7,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.7,"upper":0.8,"upper_inclusive":false},{"index":8,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.8,"upper":0.9,"upper_inclusive":false},{"index":9,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.9,"upper":1.0,"upper_inclusive":true}]},"equal_mass":{"n":38,"bin_count":10,"ece":0.22721985609907852,"signed_mean_gap":-0.038773099058552786,"absolute_mean_gap":0.038773099058552786,"bins":[{"index":0,"count":3,"confidence":0.2583892246087392,"empirical_correctness":0.0,"signed_gap":0.2583892246087392,"absolute_gap":0.2583892246087392,"min_confidence":0.25140494108200073,"max_confidence":0.26512423157691956},{"index":1,"count":4,"confidence":0.27661946415901184,"empirical_correctness":0.0,"signed_gap":0.27661946415901184,"absolute_gap":0.27661946415901184,"min_confidence":0.27007511258125305,"max_confidence":0.29093846678733826},{"index":2,"count":4,"confidence":0.29824529588222504,"empirical_correctness":0.25,"signed_gap":0.04824529588222504,"absolute_gap":0.04824529588222504,"min_confidence":0.29447802901268005,"max_confidence":0.3022262156009674},{"index":3,"count":4,"confidence":0.3051675409078598,"empirical_correctness":0.0,"signed_gap":0.3051675409078598,"absolute_gap":0.3051675409078598,"min_confidence":0.302547425031662,"max_confidence":0.30963218212127686},{"index":4,"count":4,"confidence":0.32129787653684616,"empirical_correctness":0.25,"signed_gap":0.07129787653684616,"absolute_gap":0.07129787653684616,"min_confidence":0.3162144124507904,"max_confidence":0.32703930139541626},{"index":5,"count":3,"confidence":0.3314737180868785,"empirical_correctness":0.6666666666666666,"signed_gap":-0.33519294857978815,"absolute_gap":0.33519294857978815,"min_confidence":0.3298291862010956,"max_confidence":0.33446136116981506},{"index":6,"count":4,"confidence":0.3484497144818306,"empirical_correctness":0.5,"signed_gap":-0.1515502855181694,"absolute_gap":0.1515502855181694,"min_confidence":0.3360605239868164,"max_confidence":0.3581322729587555},{"index":7,"count":4,"confidence":0.3651752695441246,"empirical_correctness":0.75,"signed_gap":-0.3848247304558754,"absolute_gap":0.3848247304558754,"min_confidence":0.3609054982662201,"max_confidence":0.36981669068336487},{"index":8,"count":4,"confidence":0.376084104180336,"empirical_correctness":0.5,"signed_gap":-0.123915895819664,"absolute_gap":0.123915895819664,"min_confidence":0.3703886568546295,"max_confidence":0.38436657190322876},{"index":9,"count":4,"confidence":0.3982190862298012,"empirical_correctness":0.75,"signed_gap":-0.3517809137701988,"absolute_gap":0.3517809137701988,"min_confidence":0.39139118790626526,"max_confidence":0.40994399785995483}]}},"latency_ms":{"n":43,"median":14.314645028207451,"p95":15.34570191288367},"per_action":{"keep":{"n":12,"raw":{"n":12,"acceptable":1,"unsafe":0,"review":0,"automated_count":12,"automated_acceptable":1,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":0.08333333333333333,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.08333333333333333,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.014865094404917095,"upper":0.35387991114111694,"confidence_level":0.95}},"effective":{"n":12,"acceptable":0,"unsafe":0,"review":12,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":12,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":2.7755575615628914e-17,"upper":0.24249400665524085,"confidence_level":0.95}}},"drop":{"n":16,"raw":{"n":16,"acceptable":10,"unsafe":0,"review":0,"automated_count":16,"automated_acceptable":10,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":0.625,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.625,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.38641040545502714,"upper":0.8151876744113636,"confidence_level":0.95}},"effective":{"n":16,"acceptable":0,"unsafe":0,"review":16,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":16,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.1936076805344365,"confidence_level":0.95}}},"mask":{"n":2,"raw":{"n":2,"acceptable":2,"unsafe":0,"review":0,"automated_count":2,"automated_acceptable":2,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":1.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.34238022750665303,"upper":1.0,"confidence_level":0.95}},"effective":{"n":2,"acceptable":0,"unsafe":0,"review":2,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":2,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.6576197724933469,"confidence_level":0.95}}},"pseudonymize":{"n":2,"raw":{"n":2,"acceptable":1,"unsafe":0,"review":0,"automated_count":2,"automated_acceptable":1,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":0.5,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.5,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.09453120573423074,"upper":0.9054687942657693,"confidence_level":0.95}},"effective":{"n":2,"acceptable":0,"unsafe":0,"review":2,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":2,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.6576197724933469,"confidence_level":0.95}}},"generalize":{"n":6,"raw":{"n":6,"acceptable":0,"unsafe":0,"review":0,"automated_count":6,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":2.7755575615628914e-17,"upper":0.3903342879021653,"confidence_level":0.95}},"effective":{"n":6,"acceptable":0,"unsafe":0,"review":6,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":6,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":2.7755575615628914e-17,"upper":0.3903342879021653,"confidence_level":0.95}}},"review":{"n":0,"raw":{"n":0,"acceptable":0,"unsafe":0,"review":0,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":null,"unsafe_rate":null,"review_rate":null,"automated_coverage":null,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":null},"effective":{"n":0,"acceptable":0,"unsafe":0,"review":0,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":null,"unsafe_rate":null,"review_rate":null,"automated_coverage":null,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":null}}}},"challenge56-no_samples":{"n":56,"raw":{"n":56,"acceptable":43,"unsafe":3,"review":0,"automated_count":56,"automated_acceptable":43,"automated_unsafe":3,"unnecessary_reviews":0,"accuracy":0.7678571428571429,"unsafe_rate":0.05357142857142857,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.7678571428571429,"conditional_unsafe":0.05357142857142857,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.6423184678498916,"upper":0.8590062089079139,"confidence_level":0.95}},"effective":{"n":56,"acceptable":14,"unsafe":0,"review":44,"automated_count":12,"automated_acceptable":12,"automated_unsafe":0,"unnecessary_reviews":42,"accuracy":0.25,"unsafe_rate":0.0,"review_rate":0.7857142857142857,"automated_coverage":0.21428571428571427,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.75,"accuracy_wilson_95":{"lower":0.15517054688270684,"upper":0.376926421476675,"confidence_level":0.95}},"confusion":{"raw":{"keep":{"keep":6,"drop":2,"mask":0,"pseudonymize":0,"generalize":0,"review":0},"drop":{"keep":1,"drop":13,"mask":8,"pseudonymize":0,"generalize":0,"review":0},"mask":{"keep":0,"drop":0,"mask":8,"pseudonymize":0,"generalize":0,"review":0},"pseudonymize":{"keep":0,"drop":0,"mask":0,"pseudonymize":8,"generalize":0,"review":0},"generalize":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":8,"review":0},"review":{"keep":1,"drop":0,"mask":0,"pseudonymize":1,"generalize":0,"review":0}},"effective":{"keep":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":8},"drop":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":22},"mask":{"keep":0,"drop":0,"mask":8,"pseudonymize":0,"generalize":0,"review":0},"pseudonymize":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":8},"generalize":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":4,"review":4},"review":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":2}}},"probability_metrics":{"singleton_n":56,"multiacceptable_n":0,"multiacceptable_coverage":0.0,"nll":1.1227318626016531,"nll_zero_mass_n":0,"brier":0.5455753409709068,"acceptable_set_n":56,"acceptable_set_nll":1.1227318626016531,"acceptable_set_zero_mass_n":0},"calibration_all_labels":{"equal_width":{"n":56,"bin_count":10,"ece":0.3984278738498688,"signed_mean_gap":-0.3984278738498688,"absolute_mean_gap":0.3984278738498688,"bins":[{"index":0,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.0,"upper":0.1,"upper_inclusive":false},{"index":1,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.1,"upper":0.2,"upper_inclusive":false},{"index":2,"count":21,"confidence":0.2702217400074005,"empirical_correctness":0.38095238095238093,"signed_gap":-0.11073064094498042,"absolute_gap":0.11073064094498042,"lower":0.2,"upper":0.3,"upper_inclusive":false},{"index":3,"count":17,"confidence":0.35440531898947325,"empirical_correctness":1.0,"signed_gap":-0.6455946810105267,"absolute_gap":0.6455946810105267,"lower":0.3,"upper":0.4,"upper_inclusive":false},{"index":4,"count":10,"confidence":0.43702492415905,"empirical_correctness":1.0,"signed_gap":-0.56297507584095,"absolute_gap":0.56297507584095,"lower":0.4,"upper":0.5,"upper_inclusive":false},{"index":5,"count":6,"confidence":0.5634948412577311,"empirical_correctness":1.0,"signed_gap":-0.4365051587422689,"absolute_gap":0.4365051587422689,"lower":0.5,"upper":0.6,"upper_inclusive":false},{"index":6,"count":2,"confidence":0.6186369061470032,"empirical_correctness":1.0,"signed_gap":-0.3813630938529968,"absolute_gap":0.3813630938529968,"lower":0.6,"upper":0.7,"upper_inclusive":false},{"index":7,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.7,"upper":0.8,"upper_inclusive":false},{"index":8,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.8,"upper":0.9,"upper_inclusive":false},{"index":9,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.9,"upper":1.0,"upper_inclusive":true}]},"equal_mass":{"n":56,"bin_count":10,"ece":0.49830994116408484,"signed_mean_gap":-0.3984278738498688,"absolute_mean_gap":0.3984278738498688,"bins":[{"index":0,"count":5,"confidence":0.24354930520057677,"empirical_correctness":0.0,"signed_gap":0.24354930520057677,"absolute_gap":0.24354930520057677,"min_confidence":0.23980183899402618,"max_confidence":0.24965126812458038},{"index":1,"count":6,"confidence":0.26315855979919434,"empirical_correctness":0.0,"signed_gap":0.26315855979919434,"absolute_gap":0.26315855979919434,"min_confidence":0.2535601258277893,"max_confidence":0.26914748549461365},{"index":2,"count":5,"confidence":0.2784122586250305,"empirical_correctness":0.6,"signed_gap":-0.3215877413749695,"absolute_gap":0.3215877413749695,"min_confidence":0.2744397222995758,"max_confidence":0.2836395502090454},{"index":3,"count":6,"confidence":0.29942547778288525,"empirical_correctness":1.0,"signed_gap":-0.7005745222171147,"absolute_gap":0.7005745222171147,"min_confidence":0.2941024601459503,"max_confidence":0.31065550446510315},{"index":4,"count":6,"confidence":0.3251022646824519,"empirical_correctness":1.0,"signed_gap":-0.674897735317548,"absolute_gap":0.674897735317548,"min_confidence":0.3146480619907379,"max_confidence":0.34041231870651245},{"index":5,"count":5,"confidence":0.3629587233066559,"empirical_correctness":1.0,"signed_gap":-0.6370412766933441,"absolute_gap":0.6370412766933441,"min_confidence":0.35026249289512634,"max_confidence":0.3793348968029022},{"index":6,"count":6,"confidence":0.3926406055688858,"empirical_correctness":1.0,"signed_gap":-0.6073593944311142,"absolute_gap":0.6073593944311142,"min_confidence":0.3837064206600189,"max_confidence":0.40701591968536377},{"index":7,"count":5,"confidence":0.4225404918193817,"empirical_correctness":1.0,"signed_gap":-0.5774595081806183,"absolute_gap":0.5774595081806183,"min_confidence":0.41002777218818665,"max_confidence":0.445632666349411},{"index":8,"count":6,"confidence":0.48976927002271015,"empirical_correctness":1.0,"signed_gap":-0.5102307299772899,"absolute_gap":0.5102307299772899,"min_confidence":0.4548131823539734,"max_confidence":0.5460519790649414},{"index":9,"count":6,"confidence":0.5883596837520599,"empirical_correctness":1.0,"signed_gap":-0.41164031624794006,"absolute_gap":0.41164031624794006,"min_confidence":0.5466581583023071,"max_confidence":0.6299730539321899}]}},"calibration_singleton_labels":{"equal_width":{"n":56,"bin_count":10,"ece":0.3984278738498688,"signed_mean_gap":-0.3984278738498688,"absolute_mean_gap":0.3984278738498688,"bins":[{"index":0,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.0,"upper":0.1,"upper_inclusive":false},{"index":1,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.1,"upper":0.2,"upper_inclusive":false},{"index":2,"count":21,"confidence":0.2702217400074005,"empirical_correctness":0.38095238095238093,"signed_gap":-0.11073064094498042,"absolute_gap":0.11073064094498042,"lower":0.2,"upper":0.3,"upper_inclusive":false},{"index":3,"count":17,"confidence":0.35440531898947325,"empirical_correctness":1.0,"signed_gap":-0.6455946810105267,"absolute_gap":0.6455946810105267,"lower":0.3,"upper":0.4,"upper_inclusive":false},{"index":4,"count":10,"confidence":0.43702492415905,"empirical_correctness":1.0,"signed_gap":-0.56297507584095,"absolute_gap":0.56297507584095,"lower":0.4,"upper":0.5,"upper_inclusive":false},{"index":5,"count":6,"confidence":0.5634948412577311,"empirical_correctness":1.0,"signed_gap":-0.4365051587422689,"absolute_gap":0.4365051587422689,"lower":0.5,"upper":0.6,"upper_inclusive":false},{"index":6,"count":2,"confidence":0.6186369061470032,"empirical_correctness":1.0,"signed_gap":-0.3813630938529968,"absolute_gap":0.3813630938529968,"lower":0.6,"upper":0.7,"upper_inclusive":false},{"index":7,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.7,"upper":0.8,"upper_inclusive":false},{"index":8,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.8,"upper":0.9,"upper_inclusive":false},{"index":9,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.9,"upper":1.0,"upper_inclusive":true}]},"equal_mass":{"n":56,"bin_count":10,"ece":0.49830994116408484,"signed_mean_gap":-0.3984278738498688,"absolute_mean_gap":0.3984278738498688,"bins":[{"index":0,"count":5,"confidence":0.24354930520057677,"empirical_correctness":0.0,"signed_gap":0.24354930520057677,"absolute_gap":0.24354930520057677,"min_confidence":0.23980183899402618,"max_confidence":0.24965126812458038},{"index":1,"count":6,"confidence":0.26315855979919434,"empirical_correctness":0.0,"signed_gap":0.26315855979919434,"absolute_gap":0.26315855979919434,"min_confidence":0.2535601258277893,"max_confidence":0.26914748549461365},{"index":2,"count":5,"confidence":0.2784122586250305,"empirical_correctness":0.6,"signed_gap":-0.3215877413749695,"absolute_gap":0.3215877413749695,"min_confidence":0.2744397222995758,"max_confidence":0.2836395502090454},{"index":3,"count":6,"confidence":0.29942547778288525,"empirical_correctness":1.0,"signed_gap":-0.7005745222171147,"absolute_gap":0.7005745222171147,"min_confidence":0.2941024601459503,"max_confidence":0.31065550446510315},{"index":4,"count":6,"confidence":0.3251022646824519,"empirical_correctness":1.0,"signed_gap":-0.674897735317548,"absolute_gap":0.674897735317548,"min_confidence":0.3146480619907379,"max_confidence":0.34041231870651245},{"index":5,"count":5,"confidence":0.3629587233066559,"empirical_correctness":1.0,"signed_gap":-0.6370412766933441,"absolute_gap":0.6370412766933441,"min_confidence":0.35026249289512634,"max_confidence":0.3793348968029022},{"index":6,"count":6,"confidence":0.3926406055688858,"empirical_correctness":1.0,"signed_gap":-0.6073593944311142,"absolute_gap":0.6073593944311142,"min_confidence":0.3837064206600189,"max_confidence":0.40701591968536377},{"index":7,"count":5,"confidence":0.4225404918193817,"empirical_correctness":1.0,"signed_gap":-0.5774595081806183,"absolute_gap":0.5774595081806183,"min_confidence":0.41002777218818665,"max_confidence":0.445632666349411},{"index":8,"count":6,"confidence":0.48976927002271015,"empirical_correctness":1.0,"signed_gap":-0.5102307299772899,"absolute_gap":0.5102307299772899,"min_confidence":0.4548131823539734,"max_confidence":0.5460519790649414},{"index":9,"count":6,"confidence":0.5883596837520599,"empirical_correctness":1.0,"signed_gap":-0.41164031624794006,"absolute_gap":0.41164031624794006,"min_confidence":0.5466581583023071,"max_confidence":0.6299730539321899}]}},"latency_ms":{"n":56,"median":12.357253493973985,"p95":12.831130254198797},"per_action":{"keep":{"n":8,"raw":{"n":8,"acceptable":6,"unsafe":0,"review":0,"automated_count":8,"automated_acceptable":6,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":0.75,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.75,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.40927543031016883,"upper":0.9285207872478909,"confidence_level":0.95}},"effective":{"n":8,"acceptable":0,"unsafe":0,"review":8,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":8,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.32440756488388023,"confidence_level":0.95}}},"drop":{"n":22,"raw":{"n":22,"acceptable":13,"unsafe":1,"review":0,"automated_count":22,"automated_acceptable":13,"automated_unsafe":1,"unnecessary_reviews":0,"accuracy":0.5909090909090909,"unsafe_rate":0.045454545454545456,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.5909090909090909,"conditional_unsafe":0.045454545454545456,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.3873481847631513,"upper":0.7674418380352814,"confidence_level":0.95}},"effective":{"n":22,"acceptable":0,"unsafe":0,"review":22,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":22,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.14865487460862087,"confidence_level":0.95}}},"mask":{"n":8,"raw":{"n":8,"acceptable":8,"unsafe":0,"review":0,"automated_count":8,"automated_acceptable":8,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":1.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.6755924351161198,"upper":1.0,"confidence_level":0.95}},"effective":{"n":8,"acceptable":8,"unsafe":0,"review":0,"automated_count":8,"automated_acceptable":8,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":1.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.6755924351161198,"upper":1.0,"confidence_level":0.95}}},"pseudonymize":{"n":8,"raw":{"n":8,"acceptable":8,"unsafe":0,"review":0,"automated_count":8,"automated_acceptable":8,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":1.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.6755924351161198,"upper":1.0,"confidence_level":0.95}},"effective":{"n":8,"acceptable":0,"unsafe":0,"review":8,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":8,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.32440756488388023,"confidence_level":0.95}}},"generalize":{"n":8,"raw":{"n":8,"acceptable":8,"unsafe":0,"review":0,"automated_count":8,"automated_acceptable":8,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":1.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.6755924351161198,"upper":1.0,"confidence_level":0.95}},"effective":{"n":8,"acceptable":4,"unsafe":0,"review":4,"automated_count":4,"automated_acceptable":4,"automated_unsafe":0,"unnecessary_reviews":4,"accuracy":0.5,"unsafe_rate":0.0,"review_rate":0.5,"automated_coverage":0.5,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.5,"accuracy_wilson_95":{"lower":0.21521606221387757,"upper":0.7847839377861224,"confidence_level":0.95}}},"review":{"n":2,"raw":{"n":2,"acceptable":0,"unsafe":2,"review":0,"automated_count":2,"automated_acceptable":0,"automated_unsafe":2,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":1.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":1.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.6576197724933469,"confidence_level":0.95}},"effective":{"n":2,"acceptable":2,"unsafe":0,"review":2,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":1.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.34238022750665303,"upper":1.0,"confidence_level":0.95}}}}},"challenge56-compact_no_samples":{"n":56,"raw":{"n":56,"acceptable":33,"unsafe":21,"review":0,"automated_count":56,"automated_acceptable":33,"automated_unsafe":21,"unnecessary_reviews":0,"accuracy":0.5892857142857143,"unsafe_rate":0.375,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.5892857142857143,"conditional_unsafe":0.375,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.4587759684643971,"upper":0.708332257121538,"confidence_level":0.95}},"effective":{"n":56,"acceptable":10,"unsafe":0,"review":48,"automated_count":8,"automated_acceptable":8,"automated_unsafe":0,"unnecessary_reviews":46,"accuracy":0.17857142857142858,"unsafe_rate":0.0,"review_rate":0.8571428571428571,"automated_coverage":0.14285714285714285,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.8214285714285714,"accuracy_wilson_95":{"lower":0.09999856752412305,"upper":0.2984118203665106,"confidence_level":0.95}},"confusion":{"raw":{"keep":{"keep":8,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0},"drop":{"keep":10,"drop":9,"mask":0,"pseudonymize":1,"generalize":2,"review":0},"mask":{"keep":0,"drop":0,"mask":8,"pseudonymize":0,"generalize":0,"review":0},"pseudonymize":{"keep":5,"drop":0,"mask":0,"pseudonymize":3,"generalize":0,"review":0},"generalize":{"keep":3,"drop":0,"mask":0,"pseudonymize":0,"generalize":5,"review":0},"review":{"keep":2,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0}},"effective":{"keep":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":8},"drop":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":22},"mask":{"keep":0,"drop":0,"mask":8,"pseudonymize":0,"generalize":0,"review":0},"pseudonymize":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":8},"generalize":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":8},"review":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":2}}},"probability_metrics":{"singleton_n":56,"multiacceptable_n":0,"multiacceptable_coverage":0.0,"nll":1.2961518161664125,"nll_zero_mass_n":0,"brier":0.6303361391199641,"acceptable_set_n":56,"acceptable_set_nll":1.2961518161664125,"acceptable_set_zero_mass_n":0},"calibration_all_labels":{"equal_width":{"n":56,"bin_count":10,"ece":0.27126181977135794,"signed_mean_gap":-0.27126181977135794,"absolute_mean_gap":0.27126181977135794,"bins":[{"index":0,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.0,"upper":0.1,"upper_inclusive":false},{"index":1,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.1,"upper":0.2,"upper_inclusive":false},{"index":2,"count":30,"confidence":0.25937420229117075,"empirical_correctness":0.43333333333333335,"signed_gap":-0.1739591310421626,"absolute_gap":0.1739591310421626,"lower":0.2,"upper":0.3,"upper_inclusive":false},{"index":3,"count":15,"confidence":0.31782671213150027,"empirical_correctness":0.6,"signed_gap":-0.2821732878684997,"absolute_gap":0.2821732878684997,"lower":0.3,"upper":0.4,"upper_inclusive":false},{"index":4,"count":9,"confidence":0.46999213761753506,"empirical_correctness":1.0,"signed_gap":-0.5300078623824649,"absolute_gap":0.5300078623824649,"lower":0.4,"upper":0.5,"upper_inclusive":false},{"index":5,"count":2,"confidence":0.5153910517692566,"empirical_correctness":1.0,"signed_gap":-0.4846089482307434,"absolute_gap":0.4846089482307434,"lower":0.5,"upper":0.6,"upper_inclusive":false},{"index":6,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.6,"upper":0.7,"upper_inclusive":false},{"index":7,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.7,"upper":0.8,"upper_inclusive":false},{"index":8,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.8,"upper":0.9,"upper_inclusive":false},{"index":9,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.9,"upper":1.0,"upper_inclusive":true}]},"equal_mass":{"n":56,"bin_count":10,"ece":0.3275742099753448,"signed_mean_gap":-0.27126181977135794,"absolute_mean_gap":0.27126181977135794,"bins":[{"index":0,"count":5,"confidence":0.22366036474704742,"empirical_correctness":0.0,"signed_gap":0.22366036474704742,"absolute_gap":0.22366036474704742,"min_confidence":0.21832117438316345,"max_confidence":0.23059147596359253},{"index":1,"count":6,"confidence":0.24307418366273245,"empirical_correctness":0.16666666666666666,"signed_gap":0.0764075169960658,"absolute_gap":0.0764075169960658,"min_confidence":0.23836521804332733,"max_confidence":0.24791152775287628},{"index":2,"count":5,"confidence":0.2557348221540451,"empirical_correctness":0.4,"signed_gap":-0.14426517784595494,"absolute_gap":0.14426517784595494,"min_confidence":0.24817059934139252,"max_confidence":0.26608365774154663},{"index":3,"count":6,"confidence":0.27384068568547565,"empirical_correctness":0.6666666666666666,"signed_gap":-0.392825980981191,"absolute_gap":0.392825980981191,"min_confidence":0.26713430881500244,"max_confidence":0.2787376642227173},{"index":4,"count":6,"confidence":0.28277816375096637,"empirical_correctness":0.8333333333333334,"signed_gap":-0.5505551695823669,"absolute_gap":0.5505551695823669,"min_confidence":0.27915850281715393,"max_confidence":0.28755778074264526},{"index":5,"count":5,"confidence":0.29813127517700194,"empirical_correctness":0.6,"signed_gap":-0.30186872482299804,"absolute_gap":0.30186872482299804,"min_confidence":0.2926071584224701,"max_confidence":0.30251049995422363},{"index":6,"count":6,"confidence":0.31128155688444775,"empirical_correctness":0.6666666666666666,"signed_gap":-0.3553851097822189,"absolute_gap":0.3553851097822189,"min_confidence":0.3064998984336853,"max_confidence":0.31357359886169434},{"index":7,"count":5,"confidence":0.33040146827697753,"empirical_correctness":0.6,"signed_gap":-0.26959853172302245,"absolute_gap":0.26959853172302245,"min_confidence":0.3162708282470703,"max_confidence":0.3430556356906891},{"index":8,"count":6,"confidence":0.43418102463086444,"empirical_correctness":0.8333333333333334,"signed_gap":-0.3991523087024689,"absolute_gap":0.3991523087024689,"min_confidence":0.34313955903053284,"max_confidence":0.48490023612976074},{"index":9,"count":6,"confidence":0.4997941255569458,"empirical_correctness":1.0,"signed_gap":-0.5002058744430542,"absolute_gap":0.5002058744430542,"min_confidence":0.4858803153038025,"max_confidence":0.5228119492530823}]}},"calibration_singleton_labels":{"equal_width":{"n":56,"bin_count":10,"ece":0.27126181977135794,"signed_mean_gap":-0.27126181977135794,"absolute_mean_gap":0.27126181977135794,"bins":[{"index":0,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.0,"upper":0.1,"upper_inclusive":false},{"index":1,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.1,"upper":0.2,"upper_inclusive":false},{"index":2,"count":30,"confidence":0.25937420229117075,"empirical_correctness":0.43333333333333335,"signed_gap":-0.1739591310421626,"absolute_gap":0.1739591310421626,"lower":0.2,"upper":0.3,"upper_inclusive":false},{"index":3,"count":15,"confidence":0.31782671213150027,"empirical_correctness":0.6,"signed_gap":-0.2821732878684997,"absolute_gap":0.2821732878684997,"lower":0.3,"upper":0.4,"upper_inclusive":false},{"index":4,"count":9,"confidence":0.46999213761753506,"empirical_correctness":1.0,"signed_gap":-0.5300078623824649,"absolute_gap":0.5300078623824649,"lower":0.4,"upper":0.5,"upper_inclusive":false},{"index":5,"count":2,"confidence":0.5153910517692566,"empirical_correctness":1.0,"signed_gap":-0.4846089482307434,"absolute_gap":0.4846089482307434,"lower":0.5,"upper":0.6,"upper_inclusive":false},{"index":6,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.6,"upper":0.7,"upper_inclusive":false},{"index":7,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.7,"upper":0.8,"upper_inclusive":false},{"index":8,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.8,"upper":0.9,"upper_inclusive":false},{"index":9,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.9,"upper":1.0,"upper_inclusive":true}]},"equal_mass":{"n":56,"bin_count":10,"ece":0.3275742099753448,"signed_mean_gap":-0.27126181977135794,"absolute_mean_gap":0.27126181977135794,"bins":[{"index":0,"count":5,"confidence":0.22366036474704742,"empirical_correctness":0.0,"signed_gap":0.22366036474704742,"absolute_gap":0.22366036474704742,"min_confidence":0.21832117438316345,"max_confidence":0.23059147596359253},{"index":1,"count":6,"confidence":0.24307418366273245,"empirical_correctness":0.16666666666666666,"signed_gap":0.0764075169960658,"absolute_gap":0.0764075169960658,"min_confidence":0.23836521804332733,"max_confidence":0.24791152775287628},{"index":2,"count":5,"confidence":0.2557348221540451,"empirical_correctness":0.4,"signed_gap":-0.14426517784595494,"absolute_gap":0.14426517784595494,"min_confidence":0.24817059934139252,"max_confidence":0.26608365774154663},{"index":3,"count":6,"confidence":0.27384068568547565,"empirical_correctness":0.6666666666666666,"signed_gap":-0.392825980981191,"absolute_gap":0.392825980981191,"min_confidence":0.26713430881500244,"max_confidence":0.2787376642227173},{"index":4,"count":6,"confidence":0.28277816375096637,"empirical_correctness":0.8333333333333334,"signed_gap":-0.5505551695823669,"absolute_gap":0.5505551695823669,"min_confidence":0.27915850281715393,"max_confidence":0.28755778074264526},{"index":5,"count":5,"confidence":0.29813127517700194,"empirical_correctness":0.6,"signed_gap":-0.30186872482299804,"absolute_gap":0.30186872482299804,"min_confidence":0.2926071584224701,"max_confidence":0.30251049995422363},{"index":6,"count":6,"confidence":0.31128155688444775,"empirical_correctness":0.6666666666666666,"signed_gap":-0.3553851097822189,"absolute_gap":0.3553851097822189,"min_confidence":0.3064998984336853,"max_confidence":0.31357359886169434},{"index":7,"count":5,"confidence":0.33040146827697753,"empirical_correctness":0.6,"signed_gap":-0.26959853172302245,"absolute_gap":0.26959853172302245,"min_confidence":0.3162708282470703,"max_confidence":0.3430556356906891},{"index":8,"count":6,"confidence":0.43418102463086444,"empirical_correctness":0.8333333333333334,"signed_gap":-0.3991523087024689,"absolute_gap":0.3991523087024689,"min_confidence":0.34313955903053284,"max_confidence":0.48490023612976074},{"index":9,"count":6,"confidence":0.4997941255569458,"empirical_correctness":1.0,"signed_gap":-0.5002058744430542,"absolute_gap":0.5002058744430542,"min_confidence":0.4858803153038025,"max_confidence":0.5228119492530823}]}},"latency_ms":{"n":56,"median":13.050852488959208,"p95":13.456014232360758},"per_action":{"keep":{"n":8,"raw":{"n":8,"acceptable":8,"unsafe":0,"review":0,"automated_count":8,"automated_acceptable":8,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":1.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.6755924351161198,"upper":1.0,"confidence_level":0.95}},"effective":{"n":8,"acceptable":0,"unsafe":0,"review":8,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":8,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.32440756488388023,"confidence_level":0.95}}},"drop":{"n":22,"raw":{"n":22,"acceptable":9,"unsafe":11,"review":0,"automated_count":22,"automated_acceptable":9,"automated_unsafe":11,"unnecessary_reviews":0,"accuracy":0.4090909090909091,"unsafe_rate":0.5,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.4090909090909091,"conditional_unsafe":0.5,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.2325581619647187,"upper":0.6126518152368488,"confidence_level":0.95}},"effective":{"n":22,"acceptable":0,"unsafe":0,"review":22,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":22,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.14865487460862087,"confidence_level":0.95}}},"mask":{"n":8,"raw":{"n":8,"acceptable":8,"unsafe":0,"review":0,"automated_count":8,"automated_acceptable":8,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":1.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.6755924351161198,"upper":1.0,"confidence_level":0.95}},"effective":{"n":8,"acceptable":8,"unsafe":0,"review":0,"automated_count":8,"automated_acceptable":8,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":1.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.6755924351161198,"upper":1.0,"confidence_level":0.95}}},"pseudonymize":{"n":8,"raw":{"n":8,"acceptable":3,"unsafe":5,"review":0,"automated_count":8,"automated_acceptable":3,"automated_unsafe":5,"unnecessary_reviews":0,"accuracy":0.375,"unsafe_rate":0.625,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.375,"conditional_unsafe":0.625,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.13684428582359742,"upper":0.6942576053973727,"confidence_level":0.95}},"effective":{"n":8,"acceptable":0,"unsafe":0,"review":8,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":8,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.32440756488388023,"confidence_level":0.95}}},"generalize":{"n":8,"raw":{"n":8,"acceptable":5,"unsafe":3,"review":0,"automated_count":8,"automated_acceptable":5,"automated_unsafe":3,"unnecessary_reviews":0,"accuracy":0.625,"unsafe_rate":0.375,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.625,"conditional_unsafe":0.375,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.30574239460262737,"upper":0.8631557141764026,"confidence_level":0.95}},"effective":{"n":8,"acceptable":0,"unsafe":0,"review":8,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":8,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.32440756488388023,"confidence_level":0.95}}},"review":{"n":2,"raw":{"n":2,"acceptable":0,"unsafe":2,"review":0,"automated_count":2,"automated_acceptable":0,"automated_unsafe":2,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":1.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":1.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.6576197724933469,"confidence_level":0.95}},"effective":{"n":2,"acceptable":2,"unsafe":0,"review":2,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":1.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.34238022750665303,"upper":1.0,"confidence_level":0.95}}}}}},"composite":{"n":198,"scope":"descriptive sum across four correlated panels","raw":{"n":198,"acceptable":108,"unsafe":27,"review":0,"automated_count":198,"automated_acceptable":108,"automated_unsafe":27,"unnecessary_reviews":0,"accuracy":0.5454545454545454,"unsafe_rate":0.13636363636363635,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.5454545454545454,"conditional_unsafe":0.13636363636363635,"unnecessary_review_rate":0.0},"effective":{"n":198,"acceptable":34,"unsafe":0,"review":178,"automated_count":20,"automated_acceptable":20,"automated_unsafe":0,"unnecessary_reviews":164,"accuracy":0.1717171717171717,"unsafe_rate":0.0,"review_rate":0.898989898989899,"automated_coverage":0.10101010101010101,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.8282828282828283}}},{"id":"Intern2B-native-v3","platform":"Local","model_id":"internlm/Intern-Decision-2B","revision":"8797836c65fc91a2435b1fb6850b5f0aabd75cc3","loading":"float16 / none","scoring_kind":"Native decision readout","settings":{"dtype":"float16","quantization":"none","max_tokens":4096,"effective_max_tokens":4096,"temperature":2.100509348278,"truncation":false,"cpu_offload":false,"scoring_batch_size":1,"trust_remote_code":false,"local_files_only":true,"use_safetensors":true,"guard_threshold":0.45,"seed":null,"tf32":null},"scoring_metadata":{"backend":"intern2b_decision","protocol":"hf-local-sweep-intern2b-v3","readout":"builtin_full_vocabulary_logits_immediately_before_decision","latency_scope":"synchronous_forward_scoring_including_transfer_excluding_tokenization","equal_mass_convention":"rank bins with floor boundaries at floor(i*n/k), k=min(10,n//2); minimum two rows"},"load_ms":7637.356353981886,"peak_allocated_bytes":4530725376,"peak_reserved_bytes":4630511616,"panels":{"diagnostic43-no_samples":{"n":43,"raw":{"n":43,"acceptable":24,"unsafe":0,"review":0,"automated_count":43,"automated_acceptable":24,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":0.5581395348837209,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.5581395348837209,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.41107614452824104,"upper":0.6956669016082949,"confidence_level":0.95}},"effective":{"n":43,"acceptable":22,"unsafe":0,"review":6,"automated_count":37,"automated_acceptable":22,"automated_unsafe":0,"unnecessary_reviews":6,"accuracy":0.5116279069767442,"unsafe_rate":0.0,"review_rate":0.13953488372093023,"automated_coverage":0.8604651162790697,"conditional_correctness":0.5945945945945946,"conditional_unsafe":0.0,"unnecessary_review_rate":0.13953488372093023,"accuracy_wilson_95":{"lower":0.3675230846944931,"upper":0.6538255245328142,"confidence_level":0.95}},"confusion":{"raw":{"keep":{"keep":0,"drop":3,"mask":8,"pseudonymize":0,"generalize":1,"review":0},"drop":{"keep":0,"drop":12,"mask":4,"pseudonymize":0,"generalize":0,"review":0},"mask":{"keep":0,"drop":0,"mask":2,"pseudonymize":0,"generalize":0,"review":0},"pseudonymize":{"keep":0,"drop":1,"mask":1,"pseudonymize":0,"generalize":0,"review":0},"generalize":{"keep":0,"drop":1,"mask":0,"pseudonymize":0,"generalize":5,"review":0},"review":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0}},"effective":{"keep":{"keep":0,"drop":2,"mask":6,"pseudonymize":0,"generalize":0,"review":4},"drop":{"keep":0,"drop":12,"mask":4,"pseudonymize":0,"generalize":0,"review":0},"mask":{"keep":0,"drop":0,"mask":2,"pseudonymize":0,"generalize":0,"review":0},"pseudonymize":{"keep":0,"drop":1,"mask":1,"pseudonymize":0,"generalize":0,"review":0},"generalize":{"keep":0,"drop":1,"mask":0,"pseudonymize":0,"generalize":3,"review":2},"review":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0}}},"probability_metrics":{"singleton_n":38,"multiacceptable_n":5,"multiacceptable_coverage":0.11627906976744186,"nll":1.4816552533753091,"nll_zero_mass_n":0,"brier":0.7178471660657654,"acceptable_set_n":43,"acceptable_set_nll":1.3194912922435853,"acceptable_set_zero_mass_n":0},"calibration_all_labels":{"equal_width":{"n":43,"bin_count":10,"ece":0.1568723318139138,"signed_mean_gap":0.0685521927288729,"absolute_mean_gap":0.0685521927288729,"bins":[{"index":0,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.0,"upper":0.1,"upper_inclusive":false},{"index":1,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.1,"upper":0.2,"upper_inclusive":false},{"index":2,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.2,"upper":0.3,"upper_inclusive":false},{"index":3,"count":3,"confidence":0.3496320392204479,"empirical_correctness":0.0,"signed_gap":0.3496320392204479,"absolute_gap":0.3496320392204479,"lower":0.3,"upper":0.4,"upper_inclusive":false},{"index":4,"count":7,"confidence":0.45185536072154103,"empirical_correctness":0.7142857142857143,"signed_gap":-0.26243035356417327,"absolute_gap":0.26243035356417327,"lower":0.4,"upper":0.5,"upper_inclusive":false},{"index":5,"count":9,"confidence":0.5486810538467594,"empirical_correctness":0.5555555555555556,"signed_gap":-0.006874501708796221,"absolute_gap":0.006874501708796221,"lower":0.5,"upper":0.6,"upper_inclusive":false},{"index":6,"count":11,"confidence":0.6652892576532673,"empirical_correctness":0.6363636363636364,"signed_gap":0.028925621289630943,"absolute_gap":0.028925621289630943,"lower":0.6,"upper":0.7,"upper_inclusive":false},{"index":7,"count":7,"confidence":0.7692910934952354,"empirical_correctness":0.5714285714285714,"signed_gap":0.197862522066664,"absolute_gap":0.197862522066664,"lower":0.7,"upper":0.8,"upper_inclusive":false},{"index":8,"count":6,"confidence":0.8490852785593304,"empirical_correctness":0.5,"signed_gap":0.34908527855933036,"absolute_gap":0.34908527855933036,"lower":0.8,"upper":0.9,"upper_inclusive":false},{"index":9,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.9,"upper":1.0,"upper_inclusive":true}]},"equal_mass":{"n":43,"bin_count":10,"ece":0.1812702580893627,"signed_mean_gap":0.0685521927288729,"absolute_mean_gap":0.0685521927288729,"bins":[{"index":0,"count":4,"confidence":0.3639587142930224,"empirical_correctness":0.0,"signed_gap":0.3639587142930224,"absolute_gap":0.3639587142930224,"min_confidence":0.30979111044980195,"max_confidence":0.40693873951074616},{"index":1,"count":4,"confidence":0.44104539740720006,"empirical_correctness":0.75,"signed_gap":-0.30895460259279994,"absolute_gap":0.30895460259279994,"min_confidence":0.41609415849314324,"max_confidence":0.45716165793806285},{"index":2,"count":4,"confidence":0.4991555761279841,"empirical_correctness":0.5,"signed_gap":-0.000844423872015887,"absolute_gap":0.000844423872015887,"min_confidence":0.4919613904925917,"max_confidence":0.5042382791211444},{"index":3,"count":5,"confidence":0.5490083958268434,"empirical_correctness":0.6,"signed_gap":-0.050991604173156624,"absolute_gap":0.050991604173156624,"min_confidence":0.5073628039154153,"max_confidence":0.5779711228509653},{"index":4,"count":4,"confidence":0.6106094996474293,"empirical_correctness":0.75,"signed_gap":-0.13939050035257072,"absolute_gap":0.13939050035257072,"min_confidence":0.5907667343625321,"max_confidence":0.6425922727671144},{"index":5,"count":4,"confidence":0.6570694307211997,"empirical_correctness":0.75,"signed_gap":-0.09293056927880028,"absolute_gap":0.09293056927880028,"min_confidence":0.6461457492725078,"max_confidence":0.6701841934454323},{"index":6,"count":5,"confidence":0.6871597019194692,"empirical_correctness":0.6,"signed_gap":0.0871597019194692,"absolute_gap":0.0871597019194692,"min_confidence":0.672031996652621,"max_confidence":0.6984970499888784},{"index":7,"count":4,"confidence":0.7511789510189117,"empirical_correctness":0.25,"signed_gap":0.5011789510189117,"absolute_gap":0.5011789510189117,"min_confidence":0.7053830863897288,"max_confidence":0.7861768503197518},{"index":8,"count":4,"confidence":0.8033278809380464,"empirical_correctness":0.75,"signed_gap":0.05332788093804641,"absolute_gap":0.05332788093804641,"min_confidence":0.7894700024335193,"max_confidence":0.8329896733611852},{"index":9,"count":5,"confidence":0.8523043995989594,"empirical_correctness":0.6,"signed_gap":0.25230439959895945,"absolute_gap":0.25230439959895945,"min_confidence":0.8352374127429925,"max_confidence":0.863844684513026}]}},"calibration_singleton_labels":{"equal_width":{"n":38,"bin_count":10,"ece":0.16762140597781822,"signed_mean_gap":0.12316834768912259,"absolute_mean_gap":0.12316834768912259,"bins":[{"index":0,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.0,"upper":0.1,"upper_inclusive":false},{"index":1,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.1,"upper":0.2,"upper_inclusive":false},{"index":2,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.2,"upper":0.3,"upper_inclusive":false},{"index":3,"count":3,"confidence":0.3496320392204479,"empirical_correctness":0.0,"signed_gap":0.3496320392204479,"absolute_gap":0.3496320392204479,"lower":0.3,"upper":0.4,"upper_inclusive":false},{"index":4,"count":5,"confidence":0.4434524815787898,"empirical_correctness":0.6,"signed_gap":-0.15654751842121017,"absolute_gap":0.15654751842121017,"lower":0.4,"upper":0.5,"upper_inclusive":false},{"index":5,"count":9,"confidence":0.5486810538467594,"empirical_correctness":0.5555555555555556,"signed_gap":-0.006874501708796221,"absolute_gap":0.006874501708796221,"lower":0.5,"upper":0.6,"upper_inclusive":false},{"index":6,"count":10,"confidence":0.6620983937888638,"empirical_correctness":0.6,"signed_gap":0.06209839378886384,"absolute_gap":0.06209839378886384,"lower":0.6,"upper":0.7,"upper_inclusive":false},{"index":7,"count":6,"confidence":0.7690082156677103,"empirical_correctness":0.5,"signed_gap":0.26900821566771027,"absolute_gap":0.26900821566771027,"lower":0.7,"upper":0.8,"upper_inclusive":false},{"index":8,"count":5,"confidence":0.8482151940231264,"empirical_correctness":0.4,"signed_gap":0.4482151940231264,"absolute_gap":0.4482151940231264,"lower":0.8,"upper":0.9,"upper_inclusive":false},{"index":9,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.9,"upper":1.0,"upper_inclusive":true}]},"equal_mass":{"n":38,"bin_count":10,"ece":0.2623490158089549,"signed_mean_gap":0.12316834768912258,"absolute_mean_gap":0.12316834768912258,"bins":[{"index":0,"count":3,"confidence":0.3496320392204479,"empirical_correctness":0.0,"signed_gap":0.3496320392204479,"absolute_gap":0.3496320392204479,"min_confidence":0.30979111044980195,"max_confidence":0.3726757590818348},{"index":1,"count":4,"confidence":0.429339150618825,"empirical_correctness":0.5,"signed_gap":-0.070660849381175,"absolute_gap":0.070660849381175,"min_confidence":0.40693873951074616,"max_confidence":0.45716165793806285},{"index":2,"count":4,"confidence":0.50300592948369,"empirical_correctness":0.25,"signed_gap":0.25300592948369005,"absolute_gap":0.25300592948369005,"min_confidence":0.49990580541864904,"max_confidence":0.5073628039154153},{"index":3,"count":4,"confidence":0.5594197938047004,"empirical_correctness":0.75,"signed_gap":-0.19058020619529958,"absolute_gap":0.19058020619529958,"min_confidence":0.5515672546164125,"max_confidence":0.5779711228509653},{"index":4,"count":4,"confidence":0.6106094996474293,"empirical_correctness":0.75,"signed_gap":-0.13939050035257072,"absolute_gap":0.13939050035257072,"min_confidence":0.5907667343625321,"max_confidence":0.6425922727671144},{"index":5,"count":3,"confidence":0.6526978431464555,"empirical_correctness":1.0,"signed_gap":-0.3473021568535445,"absolute_gap":0.3473021568535445,"min_confidence":0.6461457492725078,"max_confidence":0.6632635172932289},{"index":6,"count":4,"confidence":0.6775719391891492,"empirical_correctness":0.25,"signed_gap":0.42757193918914915,"absolute_gap":0.42757193918914915,"min_confidence":0.6701841934454323,"max_confidence":0.6905071825689125},{"index":7,"count":4,"confidence":0.7330561234010349,"empirical_correctness":0.25,"signed_gap":0.4830561234010349,"absolute_gap":0.4830561234010349,"min_confidence":0.6984970499888784,"max_confidence":0.7861768503197518},{"index":8,"count":4,"confidence":0.8033278809380464,"empirical_correctness":0.75,"signed_gap":0.05332788093804641,"absolute_gap":0.05332788093804641,"min_confidence":0.7894700024335193,"max_confidence":0.8329896733611852},{"index":9,"count":4,"confidence":0.8520215741886117,"empirical_correctness":0.5,"signed_gap":0.3520215741886117,"absolute_gap":0.3520215741886117,"min_confidence":0.8352374127429925,"max_confidence":0.863844684513026}]}},"latency_ms":{"n":43,"median":46.398849051911384,"p95":47.09449891233817},"per_action":{"keep":{"n":12,"raw":{"n":12,"acceptable":0,"unsafe":0,"review":0,"automated_count":12,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":2.7755575615628914e-17,"upper":0.24249400665524085,"confidence_level":0.95}},"effective":{"n":12,"acceptable":0,"unsafe":0,"review":4,"automated_count":8,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":4,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":0.3333333333333333,"automated_coverage":0.6666666666666666,"conditional_correctness":0.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.3333333333333333,"accuracy_wilson_95":{"lower":2.7755575615628914e-17,"upper":0.24249400665524085,"confidence_level":0.95}}},"drop":{"n":16,"raw":{"n":16,"acceptable":12,"unsafe":0,"review":0,"automated_count":16,"automated_acceptable":12,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":0.75,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.75,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.5050168346449122,"upper":0.8981793250878696,"confidence_level":0.95}},"effective":{"n":16,"acceptable":12,"unsafe":0,"review":0,"automated_count":16,"automated_acceptable":12,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":0.75,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.75,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.5050168346449122,"upper":0.8981793250878696,"confidence_level":0.95}}},"mask":{"n":2,"raw":{"n":2,"acceptable":2,"unsafe":0,"review":0,"automated_count":2,"automated_acceptable":2,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":1.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.34238022750665303,"upper":1.0,"confidence_level":0.95}},"effective":{"n":2,"acceptable":2,"unsafe":0,"review":0,"automated_count":2,"automated_acceptable":2,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":1.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.34238022750665303,"upper":1.0,"confidence_level":0.95}}},"pseudonymize":{"n":2,"raw":{"n":2,"acceptable":0,"unsafe":0,"review":0,"automated_count":2,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.6576197724933469,"confidence_level":0.95}},"effective":{"n":2,"acceptable":0,"unsafe":0,"review":0,"automated_count":2,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.6576197724933469,"confidence_level":0.95}}},"generalize":{"n":6,"raw":{"n":6,"acceptable":5,"unsafe":0,"review":0,"automated_count":6,"automated_acceptable":5,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":0.8333333333333334,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.8333333333333334,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.43649717781352987,"upper":0.9699466302516935,"confidence_level":0.95}},"effective":{"n":6,"acceptable":3,"unsafe":0,"review":2,"automated_count":4,"automated_acceptable":3,"automated_unsafe":0,"unnecessary_reviews":2,"accuracy":0.5,"unsafe_rate":0.0,"review_rate":0.3333333333333333,"automated_coverage":0.6666666666666666,"conditional_correctness":0.75,"conditional_unsafe":0.0,"unnecessary_review_rate":0.3333333333333333,"accuracy_wilson_95":{"lower":0.1876163064826506,"upper":0.8123836935173494,"confidence_level":0.95}}},"review":{"n":0,"raw":{"n":0,"acceptable":0,"unsafe":0,"review":0,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":null,"unsafe_rate":null,"review_rate":null,"automated_coverage":null,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":null},"effective":{"n":0,"acceptable":0,"unsafe":0,"review":0,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":null,"unsafe_rate":null,"review_rate":null,"automated_coverage":null,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":null}}}},"diagnostic43-compact_no_samples":{"n":43,"raw":{"n":43,"acceptable":27,"unsafe":0,"review":0,"automated_count":43,"automated_acceptable":27,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":0.627906976744186,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.627906976744186,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.47859809159688477,"upper":0.7562366099034943,"confidence_level":0.95}},"effective":{"n":43,"acceptable":26,"unsafe":0,"review":5,"automated_count":38,"automated_acceptable":26,"automated_unsafe":0,"unnecessary_reviews":5,"accuracy":0.6046511627906976,"unsafe_rate":0.0,"review_rate":0.11627906976744186,"automated_coverage":0.8837209302325582,"conditional_correctness":0.6842105263157895,"conditional_unsafe":0.0,"unnecessary_review_rate":0.11627906976744186,"accuracy_wilson_95":{"lower":0.45579063062873904,"upper":0.7363468524170255,"confidence_level":0.95}},"confusion":{"raw":{"keep":{"keep":0,"drop":3,"mask":9,"pseudonymize":0,"generalize":0,"review":0},"drop":{"keep":0,"drop":13,"mask":3,"pseudonymize":0,"generalize":0,"review":0},"mask":{"keep":0,"drop":0,"mask":2,"pseudonymize":0,"generalize":0,"review":0},"pseudonymize":{"keep":0,"drop":1,"mask":0,"pseudonymize":1,"generalize":0,"review":0},"generalize":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":6,"review":0},"review":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0}},"effective":{"keep":{"keep":0,"drop":3,"mask":6,"pseudonymize":0,"generalize":0,"review":3},"drop":{"keep":0,"drop":13,"mask":3,"pseudonymize":0,"generalize":0,"review":0},"mask":{"keep":0,"drop":0,"mask":2,"pseudonymize":0,"generalize":0,"review":0},"pseudonymize":{"keep":0,"drop":0,"mask":0,"pseudonymize":1,"generalize":0,"review":1},"generalize":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":5,"review":1},"review":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0}}},"probability_metrics":{"singleton_n":38,"multiacceptable_n":5,"multiacceptable_coverage":0.11627906976744186,"nll":1.203084477232392,"nll_zero_mass_n":0,"brier":0.618326235309567,"acceptable_set_n":43,"acceptable_set_nll":1.071788513286675,"acceptable_set_zero_mass_n":0},"calibration_all_labels":{"equal_width":{"n":43,"bin_count":10,"ece":0.1681472639711185,"signed_mean_gap":0.03400463991907784,"absolute_mean_gap":0.03400463991907784,"bins":[{"index":0,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.0,"upper":0.1,"upper_inclusive":false},{"index":1,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.1,"upper":0.2,"upper_inclusive":false},{"index":2,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.2,"upper":0.3,"upper_inclusive":false},{"index":3,"count":2,"confidence":0.37152950559461606,"empirical_correctness":0.0,"signed_gap":0.37152950559461606,"absolute_gap":0.37152950559461606,"lower":0.3,"upper":0.4,"upper_inclusive":false},{"index":4,"count":6,"confidence":0.4533217108391728,"empirical_correctness":0.6666666666666666,"signed_gap":-0.21334495582749385,"absolute_gap":0.21334495582749385,"lower":0.4,"upper":0.5,"upper_inclusive":false},{"index":5,"count":7,"confidence":0.5507989023967176,"empirical_correctness":0.42857142857142855,"signed_gap":0.12222747382528903,"absolute_gap":0.12222747382528903,"lower":0.5,"upper":0.6,"upper_inclusive":false},{"index":6,"count":7,"confidence":0.6280004739780126,"empirical_correctness":0.8571428571428571,"signed_gap":-0.22914238316484448,"absolute_gap":0.22914238316484448,"lower":0.6,"upper":0.7,"upper_inclusive":false},{"index":7,"count":11,"confidence":0.7513212255728767,"empirical_correctness":0.5454545454545454,"signed_gap":0.2058666801183313,"absolute_gap":0.2058666801183313,"lower":0.7,"upper":0.8,"upper_inclusive":false},{"index":8,"count":10,"confidence":0.8483081124371322,"empirical_correctness":0.8,"signed_gap":0.04830811243713218,"absolute_gap":0.04830811243713218,"lower":0.8,"upper":0.9,"upper_inclusive":false},{"index":9,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.9,"upper":1.0,"upper_inclusive":true}]},"equal_mass":{"n":43,"bin_count":10,"ece":0.2368842667189391,"signed_mean_gap":0.034004639919077845,"absolute_mean_gap":0.034004639919077845,"bins":[{"index":0,"count":4,"confidence":0.4001616776325231,"empirical_correctness":0.0,"signed_gap":0.4001616776325231,"absolute_gap":0.4001616776325231,"min_confidence":0.3653171046828056,"max_confidence":0.4293817936187827},{"index":1,"count":4,"confidence":0.4655856414235441,"empirical_correctness":1.0,"signed_gap":-0.5344143585764559,"absolute_gap":0.5344143585764559,"min_confidence":0.4461674522289714,"max_confidence":0.4874714801838859},{"index":2,"count":4,"confidence":0.5319149519496563,"empirical_correctness":0.5,"signed_gap":0.03191495194965632,"absolute_gap":0.03191495194965632,"min_confidence":0.5001116317270546,"max_confidence":0.5498120209333324},{"index":3,"count":5,"confidence":0.5883270364097349,"empirical_correctness":0.6,"signed_gap":-0.01167296359026504,"absolute_gap":0.01167296359026504,"min_confidence":0.5656376477520246,"max_confidence":0.6075413451993252},{"index":4,"count":4,"confidence":0.6241170694087483,"empirical_correctness":1.0,"signed_gap":-0.37588293059125166,"absolute_gap":0.37588293059125166,"min_confidence":0.6173005488194772,"max_confidence":0.6340051141924266},{"index":5,"count":4,"confidence":0.7058708083630298,"empirical_correctness":0.25,"signed_gap":0.45587080836302984,"absolute_gap":0.45587080836302984,"min_confidence":0.6858323671408184,"max_confidence":0.7157650898173704},{"index":6,"count":5,"confidence":0.7496496189058091,"empirical_correctness":0.6,"signed_gap":0.14964961890580908,"absolute_gap":0.14964961890580908,"min_confidence":0.7299086301238091,"max_confidence":0.7659729328726353},{"index":7,"count":4,"confidence":0.7960743917542363,"empirical_correctness":0.75,"signed_gap":0.04607439175423633,"absolute_gap":0.04607439175423633,"min_confidence":0.7865018559298684,"max_confidence":0.8056630465556469},{"index":8,"count":4,"confidence":0.8349440198476341,"empirical_correctness":0.5,"signed_gap":0.3349440198476341,"absolute_gap":0.3349440198476341,"min_confidence":0.8126402875921411,"max_confidence":0.8536371705424759},{"index":9,"count":5,"confidence":0.8675283996850277,"empirical_correctness":1.0,"signed_gap":-0.1324716003149723,"absolute_gap":0.1324716003149723,"min_confidence":0.8567697352245653,"max_confidence":0.8830172942310321}]}},"calibration_singleton_labels":{"equal_width":{"n":38,"bin_count":10,"ece":0.19112995715840028,"signed_mean_gap":0.07874195294099087,"absolute_mean_gap":0.07874195294099087,"bins":[{"index":0,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.0,"upper":0.1,"upper_inclusive":false},{"index":1,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.1,"upper":0.2,"upper_inclusive":false},{"index":2,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.2,"upper":0.3,"upper_inclusive":false},{"index":3,"count":2,"confidence":0.37152950559461606,"empirical_correctness":0.0,"signed_gap":0.37152950559461606,"absolute_gap":0.37152950559461606,"lower":0.3,"upper":0.4,"upper_inclusive":false},{"index":4,"count":6,"confidence":0.4533217108391728,"empirical_correctness":0.6666666666666666,"signed_gap":-0.21334495582749385,"absolute_gap":0.21334495582749385,"lower":0.4,"upper":0.5,"upper_inclusive":false},{"index":5,"count":6,"confidence":0.5483257781708331,"empirical_correctness":0.3333333333333333,"signed_gap":0.21499244483749974,"absolute_gap":0.21499244483749974,"lower":0.5,"upper":0.6,"upper_inclusive":false},{"index":6,"count":5,"confidence":0.628939530966837,"empirical_correctness":0.8,"signed_gap":-0.17106046903316308,"absolute_gap":0.17106046903316308,"lower":0.6,"upper":0.7,"upper_inclusive":false},{"index":7,"count":10,"confidence":0.7465108657845356,"empirical_correctness":0.5,"signed_gap":0.24651086578453563,"absolute_gap":0.24651086578453563,"lower":0.7,"upper":0.8,"upper_inclusive":false},{"index":8,"count":9,"confidence":0.8477159948698717,"empirical_correctness":0.7777777777777778,"signed_gap":0.06993821709209391,"absolute_gap":0.06993821709209391,"lower":0.8,"upper":0.9,"upper_inclusive":false},{"index":9,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.9,"upper":1.0,"upper_inclusive":true}]},"equal_mass":{"n":38,"bin_count":10,"ece":0.24247980892298782,"signed_mean_gap":0.07874195294099091,"absolute_mean_gap":0.07874195294099091,"bins":[{"index":0,"count":3,"confidence":0.3904216389704365,"empirical_correctness":0.0,"signed_gap":0.3904216389704365,"absolute_gap":0.3904216389704365,"min_confidence":0.3653171046828056,"max_confidence":0.42820590572207756},{"index":1,"count":4,"confidence":0.45106321978226827,"empirical_correctness":0.75,"signed_gap":-0.29893678021773173,"absolute_gap":0.29893678021773173,"min_confidence":0.4293817936187827,"max_confidence":0.465874920422543},{"index":2,"count":4,"confidence":0.5163298167622947,"empirical_correctness":0.75,"signed_gap":-0.23367018323770528,"absolute_gap":0.23367018323770528,"min_confidence":0.4874714801838859,"max_confidence":0.5390658880288898},{"index":3,"count":4,"confidence":0.5795670525076644,"empirical_correctness":0.25,"signed_gap":0.3295670525076644,"absolute_gap":0.3295670525076644,"min_confidence":0.5498120209333324,"max_confidence":0.6061613278709519},{"index":4,"count":4,"confidence":0.6346340817408083,"empirical_correctness":0.75,"signed_gap":-0.11536591825919174,"absolute_gap":0.11536591825919174,"min_confidence":0.6075413451993252,"max_confidence":0.6858323671408184},{"index":5,"count":3,"confidence":0.7125502887704336,"empirical_correctness":0.3333333333333333,"signed_gap":0.37921695543710027,"absolute_gap":0.37921695543710027,"min_confidence":0.7065463957632587,"max_confidence":0.7157650898173704},{"index":6,"count":4,"confidence":0.7455687904141025,"empirical_correctness":0.5,"signed_gap":0.2455687904141025,"absolute_gap":0.2455687904141025,"min_confidence":0.7299086301238091,"max_confidence":0.7645431146947376},{"index":7,"count":4,"confidence":0.7877114191083231,"empirical_correctness":0.75,"signed_gap":0.037711419108323097,"absolute_gap":0.037711419108323097,"min_confidence":0.7659729328726353,"max_confidence":0.8056630465556469},{"index":8,"count":4,"confidence":0.8357271610181565,"empirical_correctness":0.5,"signed_gap":0.3357271610181565,"absolute_gap":0.3357271610181565,"min_confidence":0.8126402875921411,"max_confidence":0.8567697352245653},{"index":9,"count":4,"confidence":0.8702180658001433,"empirical_correctness":1.0,"signed_gap":-0.12978193419985673,"absolute_gap":0.12978193419985673,"min_confidence":0.8605614475096778,"max_confidence":0.8830172942310321}]}},"latency_ms":{"n":43,"median":50.36803201073781,"p95":51.19301509694196},"per_action":{"keep":{"n":12,"raw":{"n":12,"acceptable":0,"unsafe":0,"review":0,"automated_count":12,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":2.7755575615628914e-17,"upper":0.24249400665524085,"confidence_level":0.95}},"effective":{"n":12,"acceptable":0,"unsafe":0,"review":3,"automated_count":9,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":3,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":0.25,"automated_coverage":0.75,"conditional_correctness":0.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.25,"accuracy_wilson_95":{"lower":2.7755575615628914e-17,"upper":0.24249400665524085,"confidence_level":0.95}}},"drop":{"n":16,"raw":{"n":16,"acceptable":13,"unsafe":0,"review":0,"automated_count":16,"automated_acceptable":13,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":0.8125,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.8125,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.5699111903802586,"upper":0.9340840092857187,"confidence_level":0.95}},"effective":{"n":16,"acceptable":13,"unsafe":0,"review":0,"automated_count":16,"automated_acceptable":13,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":0.8125,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.8125,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.5699111903802586,"upper":0.9340840092857187,"confidence_level":0.95}}},"mask":{"n":2,"raw":{"n":2,"acceptable":2,"unsafe":0,"review":0,"automated_count":2,"automated_acceptable":2,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":1.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.34238022750665303,"upper":1.0,"confidence_level":0.95}},"effective":{"n":2,"acceptable":2,"unsafe":0,"review":0,"automated_count":2,"automated_acceptable":2,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":1.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.34238022750665303,"upper":1.0,"confidence_level":0.95}}},"pseudonymize":{"n":2,"raw":{"n":2,"acceptable":1,"unsafe":0,"review":0,"automated_count":2,"automated_acceptable":1,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":0.5,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.5,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.09453120573423074,"upper":0.9054687942657693,"confidence_level":0.95}},"effective":{"n":2,"acceptable":1,"unsafe":0,"review":1,"automated_count":1,"automated_acceptable":1,"automated_unsafe":0,"unnecessary_reviews":1,"accuracy":0.5,"unsafe_rate":0.0,"review_rate":0.5,"automated_coverage":0.5,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.5,"accuracy_wilson_95":{"lower":0.09453120573423074,"upper":0.9054687942657693,"confidence_level":0.95}}},"generalize":{"n":6,"raw":{"n":6,"acceptable":6,"unsafe":0,"review":0,"automated_count":6,"automated_acceptable":6,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":1.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.6096657120978346,"upper":1.0,"confidence_level":0.95}},"effective":{"n":6,"acceptable":5,"unsafe":0,"review":1,"automated_count":5,"automated_acceptable":5,"automated_unsafe":0,"unnecessary_reviews":1,"accuracy":0.8333333333333334,"unsafe_rate":0.0,"review_rate":0.16666666666666666,"automated_coverage":0.8333333333333334,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.16666666666666666,"accuracy_wilson_95":{"lower":0.43649717781352987,"upper":0.9699466302516935,"confidence_level":0.95}}},"review":{"n":0,"raw":{"n":0,"acceptable":0,"unsafe":0,"review":0,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":null,"unsafe_rate":null,"review_rate":null,"automated_coverage":null,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":null},"effective":{"n":0,"acceptable":0,"unsafe":0,"review":0,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":null,"unsafe_rate":null,"review_rate":null,"automated_coverage":null,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":null}}}},"challenge56-no_samples":{"n":56,"raw":{"n":56,"acceptable":56,"unsafe":0,"review":2,"automated_count":54,"automated_acceptable":54,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":1.0,"unsafe_rate":0.0,"review_rate":0.03571428571428571,"automated_coverage":0.9642857142857143,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.9358060632812366,"upper":1.0,"confidence_level":0.95}},"effective":{"n":56,"acceptable":46,"unsafe":0,"review":12,"automated_count":44,"automated_acceptable":44,"automated_unsafe":0,"unnecessary_reviews":10,"accuracy":0.8214285714285714,"unsafe_rate":0.0,"review_rate":0.21428571428571427,"automated_coverage":0.7857142857142857,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.17857142857142858,"accuracy_wilson_95":{"lower":0.7015881796334894,"upper":0.900001432475877,"confidence_level":0.95}},"confusion":{"raw":{"keep":{"keep":8,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0},"drop":{"keep":0,"drop":22,"mask":0,"pseudonymize":0,"generalize":0,"review":0},"mask":{"keep":0,"drop":0,"mask":8,"pseudonymize":0,"generalize":0,"review":0},"pseudonymize":{"keep":0,"drop":0,"mask":0,"pseudonymize":8,"generalize":0,"review":0},"generalize":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":8,"review":0},"review":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":2}},"effective":{"keep":{"keep":5,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":3},"drop":{"keep":0,"drop":22,"mask":0,"pseudonymize":0,"generalize":0,"review":0},"mask":{"keep":0,"drop":0,"mask":8,"pseudonymize":0,"generalize":0,"review":0},"pseudonymize":{"keep":0,"drop":0,"mask":0,"pseudonymize":1,"generalize":0,"review":7},"generalize":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":8,"review":0},"review":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":2}}},"probability_metrics":{"singleton_n":56,"multiacceptable_n":0,"multiacceptable_coverage":0.0,"nll":0.23957129506882385,"nll_zero_mass_n":0,"brier":0.08203202378402306,"acceptable_set_n":56,"acceptable_set_nll":0.23957129506882385,"acceptable_set_zero_mass_n":0},"calibration_all_labels":{"equal_width":{"n":56,"bin_count":10,"ece":0.20207698479788094,"signed_mean_gap":-0.20207698479788094,"absolute_mean_gap":0.20207698479788094,"bins":[{"index":0,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.0,"upper":0.1,"upper_inclusive":false},{"index":1,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.1,"upper":0.2,"upper_inclusive":false},{"index":2,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.2,"upper":0.3,"upper_inclusive":false},{"index":3,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.3,"upper":0.4,"upper_inclusive":false},{"index":4,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.4,"upper":0.5,"upper_inclusive":false},{"index":5,"count":5,"confidence":0.5471652124076984,"empirical_correctness":1.0,"signed_gap":-0.45283478759230156,"absolute_gap":0.45283478759230156,"lower":0.5,"upper":0.6,"upper_inclusive":false},{"index":6,"count":9,"confidence":0.6431970828761592,"empirical_correctness":1.0,"signed_gap":-0.3568029171238408,"absolute_gap":0.3568029171238408,"lower":0.6,"upper":0.7,"upper_inclusive":false},{"index":7,"count":10,"confidence":0.7508895233168352,"empirical_correctness":1.0,"signed_gap":-0.24911047668316477,"absolute_gap":0.24911047668316477,"lower":0.7,"upper":0.8,"upper_inclusive":false},{"index":8,"count":15,"confidence":0.877083236499295,"empirical_correctness":1.0,"signed_gap":-0.12291676350070502,"absolute_gap":0.12291676350070502,"lower":0.8,"upper":0.9,"upper_inclusive":false},{"index":9,"count":17,"confidence":0.9114085448668804,"empirical_correctness":1.0,"signed_gap":-0.08859145513311961,"absolute_gap":0.08859145513311961,"lower":0.9,"upper":1.0,"upper_inclusive":true}]},"equal_mass":{"n":56,"bin_count":10,"ece":0.20207698479788097,"signed_mean_gap":-0.20207698479788097,"absolute_mean_gap":0.20207698479788097,"bins":[{"index":0,"count":5,"confidence":0.5471652124076984,"empirical_correctness":1.0,"signed_gap":-0.45283478759230156,"absolute_gap":0.45283478759230156,"min_confidence":0.522423276599713,"max_confidence":0.5830980007967358},{"index":1,"count":6,"confidence":0.633024441179757,"empirical_correctness":1.0,"signed_gap":-0.366975558820243,"absolute_gap":0.366975558820243,"min_confidence":0.6178493193360814,"max_confidence":0.6484935237605073},{"index":2,"count":5,"confidence":0.6814736666674552,"empirical_correctness":1.0,"signed_gap":-0.31852633333254476,"absolute_gap":0.31852633333254476,"min_confidence":0.6500120208394039,"max_confidence":0.7110342122094307},{"index":3,"count":6,"confidence":0.751178101204764,"empirical_correctness":1.0,"signed_gap":-0.24882189879523597,"absolute_gap":0.24882189879523597,"min_confidence":0.7218839708820336,"max_confidence":0.7795093453153873},{"index":4,"count":6,"confidence":0.8358210483521696,"empirical_correctness":1.0,"signed_gap":-0.16417895164783036,"absolute_gap":0.16417895164783036,"min_confidence":0.7905820535163395,"max_confidence":0.8629536115175868},{"index":5,"count":5,"confidence":0.8768047598253996,"empirical_correctness":1.0,"signed_gap":-0.12319524017460037,"absolute_gap":0.12319524017460037,"min_confidence":0.8678135850917371,"max_confidence":0.8833660947635256},{"index":6,"count":6,"confidence":0.890397308276465,"empirical_correctness":1.0,"signed_gap":-0.10960269172353498,"absolute_gap":0.10960269172353498,"min_confidence":0.884073060927988,"max_confidence":0.8980273121032839},{"index":7,"count":5,"confidence":0.9025220240559992,"empirical_correctness":1.0,"signed_gap":-0.09747797594400076,"absolute_gap":0.09747797594400076,"min_confidence":0.9001580708556771,"max_confidence":0.9053493658225142},{"index":8,"count":6,"confidence":0.9098686983768692,"empirical_correctness":1.0,"signed_gap":-0.09013130162313077,"absolute_gap":0.09013130162313077,"min_confidence":0.9066650326211549,"max_confidence":0.9129490278506946},{"index":9,"count":6,"confidence":0.920353825365959,"empirical_correctness":1.0,"signed_gap":-0.079646174634041,"absolute_gap":0.079646174634041,"min_confidence":0.9138743925357061,"max_confidence":0.9284903650372127}]}},"calibration_singleton_labels":{"equal_width":{"n":56,"bin_count":10,"ece":0.20207698479788094,"signed_mean_gap":-0.20207698479788094,"absolute_mean_gap":0.20207698479788094,"bins":[{"index":0,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.0,"upper":0.1,"upper_inclusive":false},{"index":1,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.1,"upper":0.2,"upper_inclusive":false},{"index":2,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.2,"upper":0.3,"upper_inclusive":false},{"index":3,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.3,"upper":0.4,"upper_inclusive":false},{"index":4,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.4,"upper":0.5,"upper_inclusive":false},{"index":5,"count":5,"confidence":0.5471652124076984,"empirical_correctness":1.0,"signed_gap":-0.45283478759230156,"absolute_gap":0.45283478759230156,"lower":0.5,"upper":0.6,"upper_inclusive":false},{"index":6,"count":9,"confidence":0.6431970828761592,"empirical_correctness":1.0,"signed_gap":-0.3568029171238408,"absolute_gap":0.3568029171238408,"lower":0.6,"upper":0.7,"upper_inclusive":false},{"index":7,"count":10,"confidence":0.7508895233168352,"empirical_correctness":1.0,"signed_gap":-0.24911047668316477,"absolute_gap":0.24911047668316477,"lower":0.7,"upper":0.8,"upper_inclusive":false},{"index":8,"count":15,"confidence":0.877083236499295,"empirical_correctness":1.0,"signed_gap":-0.12291676350070502,"absolute_gap":0.12291676350070502,"lower":0.8,"upper":0.9,"upper_inclusive":false},{"index":9,"count":17,"confidence":0.9114085448668804,"empirical_correctness":1.0,"signed_gap":-0.08859145513311961,"absolute_gap":0.08859145513311961,"lower":0.9,"upper":1.0,"upper_inclusive":true}]},"equal_mass":{"n":56,"bin_count":10,"ece":0.20207698479788097,"signed_mean_gap":-0.20207698479788097,"absolute_mean_gap":0.20207698479788097,"bins":[{"index":0,"count":5,"confidence":0.5471652124076984,"empirical_correctness":1.0,"signed_gap":-0.45283478759230156,"absolute_gap":0.45283478759230156,"min_confidence":0.522423276599713,"max_confidence":0.5830980007967358},{"index":1,"count":6,"confidence":0.633024441179757,"empirical_correctness":1.0,"signed_gap":-0.366975558820243,"absolute_gap":0.366975558820243,"min_confidence":0.6178493193360814,"max_confidence":0.6484935237605073},{"index":2,"count":5,"confidence":0.6814736666674552,"empirical_correctness":1.0,"signed_gap":-0.31852633333254476,"absolute_gap":0.31852633333254476,"min_confidence":0.6500120208394039,"max_confidence":0.7110342122094307},{"index":3,"count":6,"confidence":0.751178101204764,"empirical_correctness":1.0,"signed_gap":-0.24882189879523597,"absolute_gap":0.24882189879523597,"min_confidence":0.7218839708820336,"max_confidence":0.7795093453153873},{"index":4,"count":6,"confidence":0.8358210483521696,"empirical_correctness":1.0,"signed_gap":-0.16417895164783036,"absolute_gap":0.16417895164783036,"min_confidence":0.7905820535163395,"max_confidence":0.8629536115175868},{"index":5,"count":5,"confidence":0.8768047598253996,"empirical_correctness":1.0,"signed_gap":-0.12319524017460037,"absolute_gap":0.12319524017460037,"min_confidence":0.8678135850917371,"max_confidence":0.8833660947635256},{"index":6,"count":6,"confidence":0.890397308276465,"empirical_correctness":1.0,"signed_gap":-0.10960269172353498,"absolute_gap":0.10960269172353498,"min_confidence":0.884073060927988,"max_confidence":0.8980273121032839},{"index":7,"count":5,"confidence":0.9025220240559992,"empirical_correctness":1.0,"signed_gap":-0.09747797594400076,"absolute_gap":0.09747797594400076,"min_confidence":0.9001580708556771,"max_confidence":0.9053493658225142},{"index":8,"count":6,"confidence":0.9098686983768692,"empirical_correctness":1.0,"signed_gap":-0.09013130162313077,"absolute_gap":0.09013130162313077,"min_confidence":0.9066650326211549,"max_confidence":0.9129490278506946},{"index":9,"count":6,"confidence":0.920353825365959,"empirical_correctness":1.0,"signed_gap":-0.079646174634041,"absolute_gap":0.079646174634041,"min_confidence":0.9138743925357061,"max_confidence":0.9284903650372127}]}},"latency_ms":{"n":56,"median":41.33738749078475,"p95":42.02843923121691},"per_action":{"keep":{"n":8,"raw":{"n":8,"acceptable":8,"unsafe":0,"review":0,"automated_count":8,"automated_acceptable":8,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":1.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.6755924351161198,"upper":1.0,"confidence_level":0.95}},"effective":{"n":8,"acceptable":5,"unsafe":0,"review":3,"automated_count":5,"automated_acceptable":5,"automated_unsafe":0,"unnecessary_reviews":3,"accuracy":0.625,"unsafe_rate":0.0,"review_rate":0.375,"automated_coverage":0.625,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.375,"accuracy_wilson_95":{"lower":0.30574239460262737,"upper":0.8631557141764026,"confidence_level":0.95}}},"drop":{"n":22,"raw":{"n":22,"acceptable":22,"unsafe":0,"review":0,"automated_count":22,"automated_acceptable":22,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":1.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.8513451253913791,"upper":1.0,"confidence_level":0.95}},"effective":{"n":22,"acceptable":22,"unsafe":0,"review":0,"automated_count":22,"automated_acceptable":22,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":1.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.8513451253913791,"upper":1.0,"confidence_level":0.95}}},"mask":{"n":8,"raw":{"n":8,"acceptable":8,"unsafe":0,"review":0,"automated_count":8,"automated_acceptable":8,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":1.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.6755924351161198,"upper":1.0,"confidence_level":0.95}},"effective":{"n":8,"acceptable":8,"unsafe":0,"review":0,"automated_count":8,"automated_acceptable":8,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":1.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.6755924351161198,"upper":1.0,"confidence_level":0.95}}},"pseudonymize":{"n":8,"raw":{"n":8,"acceptable":8,"unsafe":0,"review":0,"automated_count":8,"automated_acceptable":8,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":1.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.6755924351161198,"upper":1.0,"confidence_level":0.95}},"effective":{"n":8,"acceptable":1,"unsafe":0,"review":7,"automated_count":1,"automated_acceptable":1,"automated_unsafe":0,"unnecessary_reviews":7,"accuracy":0.125,"unsafe_rate":0.0,"review_rate":0.875,"automated_coverage":0.125,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.875,"accuracy_wilson_95":{"lower":0.02241749145005667,"upper":0.4708881822128535,"confidence_level":0.95}}},"generalize":{"n":8,"raw":{"n":8,"acceptable":8,"unsafe":0,"review":0,"automated_count":8,"automated_acceptable":8,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":1.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.6755924351161198,"upper":1.0,"confidence_level":0.95}},"effective":{"n":8,"acceptable":8,"unsafe":0,"review":0,"automated_count":8,"automated_acceptable":8,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":1.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.6755924351161198,"upper":1.0,"confidence_level":0.95}}},"review":{"n":2,"raw":{"n":2,"acceptable":2,"unsafe":0,"review":2,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":1.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.34238022750665303,"upper":1.0,"confidence_level":0.95}},"effective":{"n":2,"acceptable":2,"unsafe":0,"review":2,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":1.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.34238022750665303,"upper":1.0,"confidence_level":0.95}}}}},"challenge56-compact_no_samples":{"n":56,"raw":{"n":56,"acceptable":50,"unsafe":6,"review":2,"automated_count":54,"automated_acceptable":48,"automated_unsafe":6,"unnecessary_reviews":0,"accuracy":0.8928571428571429,"unsafe_rate":0.10714285714285714,"review_rate":0.03571428571428571,"automated_coverage":0.9642857142857143,"conditional_correctness":0.8888888888888888,"conditional_unsafe":0.1111111111111111,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.7853155482892853,"upper":0.9499606442888293,"confidence_level":0.95}},"effective":{"n":56,"acceptable":38,"unsafe":0,"review":20,"automated_count":36,"automated_acceptable":36,"automated_unsafe":0,"unnecessary_reviews":18,"accuracy":0.6785714285714286,"unsafe_rate":0.0,"review_rate":0.35714285714285715,"automated_coverage":0.6428571428571429,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.32142857142857145,"accuracy_wilson_95":{"lower":0.5482263990973855,"upper":0.7859900520744849,"confidence_level":0.95}},"confusion":{"raw":{"keep":{"keep":8,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0},"drop":{"keep":6,"drop":16,"mask":0,"pseudonymize":0,"generalize":0,"review":0},"mask":{"keep":0,"drop":0,"mask":8,"pseudonymize":0,"generalize":0,"review":0},"pseudonymize":{"keep":0,"drop":0,"mask":0,"pseudonymize":8,"generalize":0,"review":0},"generalize":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":8,"review":0},"review":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":2}},"effective":{"keep":{"keep":5,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":3},"drop":{"keep":0,"drop":14,"mask":0,"pseudonymize":0,"generalize":0,"review":8},"mask":{"keep":0,"drop":0,"mask":8,"pseudonymize":0,"generalize":0,"review":0},"pseudonymize":{"keep":0,"drop":0,"mask":0,"pseudonymize":1,"generalize":0,"review":7},"generalize":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":8,"review":0},"review":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":2}}},"probability_metrics":{"singleton_n":56,"multiacceptable_n":0,"multiacceptable_coverage":0.0,"nll":0.3197690695780499,"nll_zero_mass_n":0,"brier":0.1381489716615046,"acceptable_set_n":56,"acceptable_set_nll":0.3197690695780499,"acceptable_set_zero_mass_n":0},"calibration_all_labels":{"equal_width":{"n":56,"bin_count":10,"ece":0.16156021951827468,"signed_mean_gap":-0.11358147191633763,"absolute_mean_gap":0.11358147191633763,"bins":[{"index":0,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.0,"upper":0.1,"upper_inclusive":false},{"index":1,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.1,"upper":0.2,"upper_inclusive":false},{"index":2,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.2,"upper":0.3,"upper_inclusive":false},{"index":3,"count":4,"confidence":0.3818365827550737,"empirical_correctness":0.25,"signed_gap":0.1318365827550737,"absolute_gap":0.1318365827550737,"lower":0.3,"upper":0.4,"upper_inclusive":false},{"index":4,"count":3,"confidence":0.43001778244357874,"empirical_correctness":0.3333333333333333,"signed_gap":0.09668444911024543,"absolute_gap":0.09668444911024543,"lower":0.4,"upper":0.5,"upper_inclusive":false},{"index":5,"count":1,"confidence":0.526005254503206,"empirical_correctness":0.0,"signed_gap":0.526005254503206,"absolute_gap":0.526005254503206,"lower":0.5,"upper":0.6,"upper_inclusive":false},{"index":6,"count":8,"confidence":0.659835459717097,"empirical_correctness":1.0,"signed_gap":-0.340164540282903,"absolute_gap":0.340164540282903,"lower":0.6,"upper":0.7,"upper_inclusive":false},{"index":7,"count":2,"confidence":0.7190991782296913,"empirical_correctness":1.0,"signed_gap":-0.28090082177030873,"absolute_gap":0.28090082177030873,"lower":0.7,"upper":0.8,"upper_inclusive":false},{"index":8,"count":22,"confidence":0.864753105056948,"empirical_correctness":1.0,"signed_gap":-0.13524689494305198,"absolute_gap":0.13524689494305198,"lower":0.8,"upper":0.9,"upper_inclusive":false},{"index":9,"count":16,"confidence":0.909661393398865,"empirical_correctness":1.0,"signed_gap":-0.09033860660113502,"absolute_gap":0.09033860660113502,"lower":0.9,"upper":1.0,"upper_inclusive":true}]},"equal_mass":{"n":56,"bin_count":10,"ece":0.1474864714134007,"signed_mean_gap":-0.11358147191633765,"absolute_mean_gap":0.11358147191633765,"bins":[{"index":0,"count":5,"confidence":0.3898679971835532,"empirical_correctness":0.2,"signed_gap":0.18986799718355318,"absolute_gap":0.18986799718355318,"min_confidence":0.37272112312111616,"max_confidence":0.42199365489747115},{"index":1,"count":6,"confidence":0.5489049599345465,"empirical_correctness":0.6666666666666666,"signed_gap":-0.11776170673212016,"absolute_gap":0.11776170673212016,"min_confidence":0.4299567275129237,"max_confidence":0.6466668848041519},{"index":2,"count":5,"confidence":0.6758637730131936,"empirical_correctness":1.0,"signed_gap":-0.32413622698680644,"absolute_gap":0.32413622698680644,"min_confidence":0.6471006141069536,"max_confidence":0.6988392199039225},{"index":3,"count":6,"confidence":0.7849867427393639,"empirical_correctness":1.0,"signed_gap":-0.21501325726063614,"absolute_gap":0.21501325726063614,"min_confidence":0.7105134873168814,"max_confidence":0.825487274614318},{"index":4,"count":6,"confidence":0.8551110528300044,"empirical_correctness":1.0,"signed_gap":-0.14488894716999556,"absolute_gap":0.14488894716999556,"min_confidence":0.8257964429033852,"max_confidence":0.8711355965954442},{"index":5,"count":5,"confidence":0.8757975736611272,"empirical_correctness":1.0,"signed_gap":-0.1242024263388728,"absolute_gap":0.1242024263388728,"min_confidence":0.872740124964201,"max_confidence":0.8772656229576421},{"index":6,"count":6,"confidence":0.8905765317080448,"empirical_correctness":1.0,"signed_gap":-0.10942346829195515,"absolute_gap":0.10942346829195515,"min_confidence":0.8825073269701235,"max_confidence":0.898306580354322},{"index":7,"count":5,"confidence":0.9023640113007927,"empirical_correctness":1.0,"signed_gap":-0.09763598869920731,"absolute_gap":0.09763598869920731,"min_confidence":0.8997328357421243,"max_confidence":0.9044072268760606},{"index":8,"count":6,"confidence":0.9076545839148699,"empirical_correctness":1.0,"signed_gap":-0.09234541608513014,"absolute_gap":0.09234541608513014,"min_confidence":0.9052616723113738,"max_confidence":0.9093762967091679},{"index":9,"count":6,"confidence":0.9160945950217969,"empirical_correctness":1.0,"signed_gap":-0.08390540497820309,"absolute_gap":0.08390540497820309,"min_confidence":0.9094333212118199,"max_confidence":0.9261186624786242}]}},"calibration_singleton_labels":{"equal_width":{"n":56,"bin_count":10,"ece":0.16156021951827468,"signed_mean_gap":-0.11358147191633763,"absolute_mean_gap":0.11358147191633763,"bins":[{"index":0,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.0,"upper":0.1,"upper_inclusive":false},{"index":1,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.1,"upper":0.2,"upper_inclusive":false},{"index":2,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.2,"upper":0.3,"upper_inclusive":false},{"index":3,"count":4,"confidence":0.3818365827550737,"empirical_correctness":0.25,"signed_gap":0.1318365827550737,"absolute_gap":0.1318365827550737,"lower":0.3,"upper":0.4,"upper_inclusive":false},{"index":4,"count":3,"confidence":0.43001778244357874,"empirical_correctness":0.3333333333333333,"signed_gap":0.09668444911024543,"absolute_gap":0.09668444911024543,"lower":0.4,"upper":0.5,"upper_inclusive":false},{"index":5,"count":1,"confidence":0.526005254503206,"empirical_correctness":0.0,"signed_gap":0.526005254503206,"absolute_gap":0.526005254503206,"lower":0.5,"upper":0.6,"upper_inclusive":false},{"index":6,"count":8,"confidence":0.659835459717097,"empirical_correctness":1.0,"signed_gap":-0.340164540282903,"absolute_gap":0.340164540282903,"lower":0.6,"upper":0.7,"upper_inclusive":false},{"index":7,"count":2,"confidence":0.7190991782296913,"empirical_correctness":1.0,"signed_gap":-0.28090082177030873,"absolute_gap":0.28090082177030873,"lower":0.7,"upper":0.8,"upper_inclusive":false},{"index":8,"count":22,"confidence":0.864753105056948,"empirical_correctness":1.0,"signed_gap":-0.13524689494305198,"absolute_gap":0.13524689494305198,"lower":0.8,"upper":0.9,"upper_inclusive":false},{"index":9,"count":16,"confidence":0.909661393398865,"empirical_correctness":1.0,"signed_gap":-0.09033860660113502,"absolute_gap":0.09033860660113502,"lower":0.9,"upper":1.0,"upper_inclusive":true}]},"equal_mass":{"n":56,"bin_count":10,"ece":0.1474864714134007,"signed_mean_gap":-0.11358147191633765,"absolute_mean_gap":0.11358147191633765,"bins":[{"index":0,"count":5,"confidence":0.3898679971835532,"empirical_correctness":0.2,"signed_gap":0.18986799718355318,"absolute_gap":0.18986799718355318,"min_confidence":0.37272112312111616,"max_confidence":0.42199365489747115},{"index":1,"count":6,"confidence":0.5489049599345465,"empirical_correctness":0.6666666666666666,"signed_gap":-0.11776170673212016,"absolute_gap":0.11776170673212016,"min_confidence":0.4299567275129237,"max_confidence":0.6466668848041519},{"index":2,"count":5,"confidence":0.6758637730131936,"empirical_correctness":1.0,"signed_gap":-0.32413622698680644,"absolute_gap":0.32413622698680644,"min_confidence":0.6471006141069536,"max_confidence":0.6988392199039225},{"index":3,"count":6,"confidence":0.7849867427393639,"empirical_correctness":1.0,"signed_gap":-0.21501325726063614,"absolute_gap":0.21501325726063614,"min_confidence":0.7105134873168814,"max_confidence":0.825487274614318},{"index":4,"count":6,"confidence":0.8551110528300044,"empirical_correctness":1.0,"signed_gap":-0.14488894716999556,"absolute_gap":0.14488894716999556,"min_confidence":0.8257964429033852,"max_confidence":0.8711355965954442},{"index":5,"count":5,"confidence":0.8757975736611272,"empirical_correctness":1.0,"signed_gap":-0.1242024263388728,"absolute_gap":0.1242024263388728,"min_confidence":0.872740124964201,"max_confidence":0.8772656229576421},{"index":6,"count":6,"confidence":0.8905765317080448,"empirical_correctness":1.0,"signed_gap":-0.10942346829195515,"absolute_gap":0.10942346829195515,"min_confidence":0.8825073269701235,"max_confidence":0.898306580354322},{"index":7,"count":5,"confidence":0.9023640113007927,"empirical_correctness":1.0,"signed_gap":-0.09763598869920731,"absolute_gap":0.09763598869920731,"min_confidence":0.8997328357421243,"max_confidence":0.9044072268760606},{"index":8,"count":6,"confidence":0.9076545839148699,"empirical_correctness":1.0,"signed_gap":-0.09234541608513014,"absolute_gap":0.09234541608513014,"min_confidence":0.9052616723113738,"max_confidence":0.9093762967091679},{"index":9,"count":6,"confidence":0.9160945950217969,"empirical_correctness":1.0,"signed_gap":-0.08390540497820309,"absolute_gap":0.08390540497820309,"min_confidence":0.9094333212118199,"max_confidence":0.9261186624786242}]}},"latency_ms":{"n":56,"median":46.79040299379267,"p95":47.35120198165532},"per_action":{"keep":{"n":8,"raw":{"n":8,"acceptable":8,"unsafe":0,"review":0,"automated_count":8,"automated_acceptable":8,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":1.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.6755924351161198,"upper":1.0,"confidence_level":0.95}},"effective":{"n":8,"acceptable":5,"unsafe":0,"review":3,"automated_count":5,"automated_acceptable":5,"automated_unsafe":0,"unnecessary_reviews":3,"accuracy":0.625,"unsafe_rate":0.0,"review_rate":0.375,"automated_coverage":0.625,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.375,"accuracy_wilson_95":{"lower":0.30574239460262737,"upper":0.8631557141764026,"confidence_level":0.95}}},"drop":{"n":22,"raw":{"n":22,"acceptable":16,"unsafe":6,"review":0,"automated_count":22,"automated_acceptable":16,"automated_unsafe":6,"unnecessary_reviews":0,"accuracy":0.7272727272727273,"unsafe_rate":0.2727272727272727,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.7272727272727273,"conditional_unsafe":0.2727272727272727,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.5184826598759762,"upper":0.8684923971201053,"confidence_level":0.95}},"effective":{"n":22,"acceptable":14,"unsafe":0,"review":8,"automated_count":14,"automated_acceptable":14,"automated_unsafe":0,"unnecessary_reviews":8,"accuracy":0.6363636363636364,"unsafe_rate":0.0,"review_rate":0.36363636363636365,"automated_coverage":0.6363636363636364,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.36363636363636365,"accuracy_wilson_95":{"lower":0.42951708090131013,"upper":0.8026679532963387,"confidence_level":0.95}}},"mask":{"n":8,"raw":{"n":8,"acceptable":8,"unsafe":0,"review":0,"automated_count":8,"automated_acceptable":8,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":1.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.6755924351161198,"upper":1.0,"confidence_level":0.95}},"effective":{"n":8,"acceptable":8,"unsafe":0,"review":0,"automated_count":8,"automated_acceptable":8,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":1.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.6755924351161198,"upper":1.0,"confidence_level":0.95}}},"pseudonymize":{"n":8,"raw":{"n":8,"acceptable":8,"unsafe":0,"review":0,"automated_count":8,"automated_acceptable":8,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":1.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.6755924351161198,"upper":1.0,"confidence_level":0.95}},"effective":{"n":8,"acceptable":1,"unsafe":0,"review":7,"automated_count":1,"automated_acceptable":1,"automated_unsafe":0,"unnecessary_reviews":7,"accuracy":0.125,"unsafe_rate":0.0,"review_rate":0.875,"automated_coverage":0.125,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.875,"accuracy_wilson_95":{"lower":0.02241749145005667,"upper":0.4708881822128535,"confidence_level":0.95}}},"generalize":{"n":8,"raw":{"n":8,"acceptable":8,"unsafe":0,"review":0,"automated_count":8,"automated_acceptable":8,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":1.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.6755924351161198,"upper":1.0,"confidence_level":0.95}},"effective":{"n":8,"acceptable":8,"unsafe":0,"review":0,"automated_count":8,"automated_acceptable":8,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":1.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.6755924351161198,"upper":1.0,"confidence_level":0.95}}},"review":{"n":2,"raw":{"n":2,"acceptable":2,"unsafe":0,"review":2,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":1.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.34238022750665303,"upper":1.0,"confidence_level":0.95}},"effective":{"n":2,"acceptable":2,"unsafe":0,"review":2,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":1.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.34238022750665303,"upper":1.0,"confidence_level":0.95}}}}}},"composite":{"n":198,"scope":"descriptive sum across four correlated panels","raw":{"n":198,"acceptable":157,"unsafe":6,"review":4,"automated_count":194,"automated_acceptable":153,"automated_unsafe":6,"unnecessary_reviews":0,"accuracy":0.7929292929292929,"unsafe_rate":0.030303030303030304,"review_rate":0.020202020202020204,"automated_coverage":0.9797979797979798,"conditional_correctness":0.788659793814433,"conditional_unsafe":0.030927835051546393,"unnecessary_review_rate":0.0},"effective":{"n":198,"acceptable":132,"unsafe":0,"review":43,"automated_count":155,"automated_acceptable":128,"automated_unsafe":0,"unnecessary_reviews":39,"accuracy":0.6666666666666666,"unsafe_rate":0.0,"review_rate":0.21717171717171718,"automated_coverage":0.7828282828282829,"conditional_correctness":0.8258064516129032,"conditional_unsafe":0.0,"unnecessary_review_rate":0.19696969696969696}}},{"id":"nli-deberta-large-fp32","platform":"Local","model_id":"MoritzLaurer/deberta-v3-large-zeroshot-v2.0","revision":"cf44676c28ba7312e5c5f8f8d2c22b3e0c9cdae2","loading":"float32 / none","scoring_kind":"Adapted NLI","settings":{"dtype":"float32","quantization":null,"max_tokens":512,"effective_max_tokens":512,"temperature":1.0,"scoring_batch_size":1,"trust_remote_code":false,"local_files_only":true,"use_safetensors":true,"guard_threshold":0.45,"seed":0,"tf32":false},"scoring_metadata":{"backend":"nli","protocol":"hf-local-sweep-backends-v1","probability_source":"adapted_entailment_softmax_not_calibrated_sixway","latency_scope":"synchronous_forward_scoring_including_transfer_excluding_tokenization","precision":"float32","equal_mass_convention":"rank bins with floor boundaries at floor(i*n/k), k=min(10,n//2); minimum two rows"},"load_ms":1855.5277209961787,"peak_allocated_bytes":1868677120,"peak_reserved_bytes":1881145344,"panels":{"diagnostic43-no_samples":{"n":43,"raw":{"n":43,"acceptable":17,"unsafe":6,"review":0,"automated_count":43,"automated_acceptable":17,"automated_unsafe":6,"unnecessary_reviews":0,"accuracy":0.3953488372093023,"unsafe_rate":0.13953488372093023,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.3953488372093023,"conditional_unsafe":0.13953488372093023,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.2636531475829744,"upper":0.5442093693712609,"confidence_level":0.95}},"effective":{"n":43,"acceptable":5,"unsafe":0,"review":43,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":38,"accuracy":0.11627906976744186,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":0.8837209302325582,"accuracy_wilson_95":{"lower":0.05070427739804166,"upper":0.24479161810082128,"confidence_level":0.95}},"confusion":{"raw":{"keep":{"keep":0,"drop":9,"mask":0,"pseudonymize":3,"generalize":0,"review":0},"drop":{"keep":0,"drop":11,"mask":0,"pseudonymize":5,"generalize":0,"review":0},"mask":{"keep":0,"drop":2,"mask":0,"pseudonymize":0,"generalize":0,"review":0},"pseudonymize":{"keep":0,"drop":0,"mask":0,"pseudonymize":2,"generalize":0,"review":0},"generalize":{"keep":0,"drop":3,"mask":0,"pseudonymize":3,"generalize":0,"review":0},"review":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0}},"effective":{"keep":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":12},"drop":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":16},"mask":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":2},"pseudonymize":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":2},"generalize":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":6},"review":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0}}},"probability_metrics":{"singleton_n":38,"multiacceptable_n":5,"multiacceptable_coverage":0.11627906976744186,"nll":1.7242054166191916,"nll_zero_mass_n":0,"brier":0.8078886665193189,"acceptable_set_n":43,"acceptable_set_nll":1.6005209848332653,"acceptable_set_zero_mass_n":0},"calibration_all_labels":{"equal_width":{"n":43,"bin_count":10,"ece":0.18443988679453385,"signed_mean_gap":-0.18443988679453385,"absolute_mean_gap":0.18443988679453385,"bins":[{"index":0,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.0,"upper":0.1,"upper_inclusive":false},{"index":1,"count":9,"confidence":0.19705750048160553,"empirical_correctness":0.4444444444444444,"signed_gap":-0.2473869439628389,"absolute_gap":0.2473869439628389,"lower":0.1,"upper":0.2,"upper_inclusive":false},{"index":2,"count":34,"confidence":0.21457551069119396,"empirical_correctness":0.38235294117647056,"signed_gap":-0.1677774304852766,"absolute_gap":0.1677774304852766,"lower":0.2,"upper":0.3,"upper_inclusive":false},{"index":3,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.3,"upper":0.4,"upper_inclusive":false},{"index":4,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.4,"upper":0.5,"upper_inclusive":false},{"index":5,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.5,"upper":0.6,"upper_inclusive":false},{"index":6,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.6,"upper":0.7,"upper_inclusive":false},{"index":7,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.7,"upper":0.8,"upper_inclusive":false},{"index":8,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.8,"upper":0.9,"upper_inclusive":false},{"index":9,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.9,"upper":1.0,"upper_inclusive":true}]},"equal_mass":{"n":43,"bin_count":10,"ece":0.22181524682876674,"signed_mean_gap":-0.18443988679453385,"absolute_mean_gap":0.18443988679453385,"bins":[{"index":0,"count":4,"confidence":0.19525083526968956,"empirical_correctness":0.5,"signed_gap":-0.30474916473031044,"absolute_gap":0.30474916473031044,"min_confidence":0.19214393198490143,"max_confidence":0.19646236300468445},{"index":1,"count":4,"confidence":0.19820093363523483,"empirical_correctness":0.5,"signed_gap":-0.30179906636476517,"absolute_gap":0.30179906636476517,"min_confidence":0.19727793335914612,"max_confidence":0.19959552586078644},{"index":2,"count":4,"confidence":0.20089256018400192,"empirical_correctness":0.0,"signed_gap":0.20089256018400192,"absolute_gap":0.20089256018400192,"min_confidence":0.1997104287147522,"max_confidence":0.20190514624118805},{"index":3,"count":5,"confidence":0.2057353675365448,"empirical_correctness":0.4,"signed_gap":-0.1942646324634552,"absolute_gap":0.1942646324634552,"min_confidence":0.203049436211586,"max_confidence":0.20811042189598083},{"index":4,"count":4,"confidence":0.20844995975494385,"empirical_correctness":0.25,"signed_gap":-0.04155004024505615,"absolute_gap":0.04155004024505615,"min_confidence":0.20829206705093384,"max_confidence":0.2086997777223587},{"index":5,"count":4,"confidence":0.20986366271972656,"empirical_correctness":0.25,"signed_gap":-0.04013633728027344,"absolute_gap":0.04013633728027344,"min_confidence":0.20906749367713928,"max_confidence":0.210786834359169},{"index":6,"count":5,"confidence":0.21332733631134032,"empirical_correctness":0.4,"signed_gap":-0.1866726636886597,"absolute_gap":0.1866726636886597,"min_confidence":0.21097099781036377,"max_confidence":0.21581710875034332},{"index":7,"count":4,"confidence":0.21717127412557602,"empirical_correctness":0.5,"signed_gap":-0.282828725874424,"absolute_gap":0.282828725874424,"min_confidence":0.216593936085701,"max_confidence":0.21770787239074707},{"index":8,"count":4,"confidence":0.2207079380750656,"empirical_correctness":0.25,"signed_gap":-0.029292061924934387,"absolute_gap":0.029292061924934387,"min_confidence":0.2194172888994217,"max_confidence":0.22252939641475677},{"index":9,"count":5,"confidence":0.23432453870773315,"empirical_correctness":0.8,"signed_gap":-0.5656754612922669,"absolute_gap":0.5656754612922669,"min_confidence":0.22411371767520905,"max_confidence":0.2585716247558594}]}},"calibration_singleton_labels":{"equal_width":{"n":38,"bin_count":10,"ece":0.13118264510443334,"signed_mean_gap":-0.13118264510443334,"absolute_mean_gap":0.13118264510443334,"bins":[{"index":0,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.0,"upper":0.1,"upper_inclusive":false},{"index":1,"count":8,"confidence":0.19702994637191296,"empirical_correctness":0.375,"signed_gap":-0.17797005362808704,"absolute_gap":0.17797005362808704,"lower":0.1,"upper":0.2,"upper_inclusive":false},{"index":2,"count":30,"confidence":0.2146273305018743,"empirical_correctness":0.3333333333333333,"signed_gap":-0.11870600283145902,"absolute_gap":0.11870600283145902,"lower":0.2,"upper":0.3,"upper_inclusive":false},{"index":3,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.3,"upper":0.4,"upper_inclusive":false},{"index":4,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.4,"upper":0.5,"upper_inclusive":false},{"index":5,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.5,"upper":0.6,"upper_inclusive":false},{"index":6,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.6,"upper":0.7,"upper_inclusive":false},{"index":7,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.7,"upper":0.8,"upper_inclusive":false},{"index":8,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.8,"upper":0.9,"upper_inclusive":false},{"index":9,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.9,"upper":1.0,"upper_inclusive":true}]},"equal_mass":{"n":38,"bin_count":10,"ece":0.1734758156694864,"signed_mean_gap":-0.13118264510443337,"absolute_mean_gap":0.13118264510443337,"bins":[{"index":0,"count":3,"confidence":0.19484699269135794,"empirical_correctness":0.3333333333333333,"signed_gap":-0.13848634064197538,"absolute_gap":0.13848634064197538,"min_confidence":0.19214393198490143,"max_confidence":0.1964578926563263},{"index":1,"count":4,"confidence":0.19799704104661942,"empirical_correctness":0.5,"signed_gap":-0.3020029589533806,"absolute_gap":0.3020029589533806,"min_confidence":0.19646236300468445,"max_confidence":0.19959552586078644},{"index":2,"count":4,"confidence":0.20089256018400192,"empirical_correctness":0.0,"signed_gap":0.20089256018400192,"absolute_gap":0.20089256018400192,"min_confidence":0.1997104287147522,"max_confidence":0.20190514624118805},{"index":3,"count":4,"confidence":0.2055930644273758,"empirical_correctness":0.25,"signed_gap":-0.04440693557262421,"absolute_gap":0.04440693557262421,"min_confidence":0.203049436211586,"max_confidence":0.20811042189598083},{"index":4,"count":4,"confidence":0.20844995975494385,"empirical_correctness":0.25,"signed_gap":-0.04155004024505615,"absolute_gap":0.04155004024505615,"min_confidence":0.20829206705093384,"max_confidence":0.2086997777223587},{"index":5,"count":3,"confidence":0.210129052400589,"empirical_correctness":0.3333333333333333,"signed_gap":-0.12320428093274433,"absolute_gap":0.12320428093274433,"min_confidence":0.2090926468372345,"max_confidence":0.210786834359169},{"index":6,"count":4,"confidence":0.21270489320158958,"empirical_correctness":0.5,"signed_gap":-0.2872951067984104,"absolute_gap":0.2872951067984104,"min_confidence":0.21097099781036377,"max_confidence":0.21519511938095093},{"index":7,"count":4,"confidence":0.21681013703346252,"empirical_correctness":0.25,"signed_gap":-0.033189862966537476,"absolute_gap":0.033189862966537476,"min_confidence":0.21581710875034332,"max_confidence":0.21770787239074707},{"index":8,"count":4,"confidence":0.2207079380750656,"empirical_correctness":0.25,"signed_gap":-0.029292061924934387,"absolute_gap":0.029292061924934387,"min_confidence":0.2194172888994217,"max_confidence":0.22252939641475677},{"index":9,"count":4,"confidence":0.23687724396586418,"empirical_correctness":0.75,"signed_gap":-0.5131227560341358,"absolute_gap":0.5131227560341358,"min_confidence":0.22766076028347015,"max_confidence":0.2585716247558594}]}},"latency_ms":{"n":43,"median":134.9250889616087,"p95":137.89301386568695},"per_action":{"keep":{"n":12,"raw":{"n":12,"acceptable":0,"unsafe":0,"review":0,"automated_count":12,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":2.7755575615628914e-17,"upper":0.24249400665524085,"confidence_level":0.95}},"effective":{"n":12,"acceptable":0,"unsafe":0,"review":12,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":12,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":2.7755575615628914e-17,"upper":0.24249400665524085,"confidence_level":0.95}}},"drop":{"n":16,"raw":{"n":16,"acceptable":11,"unsafe":5,"review":0,"automated_count":16,"automated_acceptable":11,"automated_unsafe":5,"unnecessary_reviews":0,"accuracy":0.6875,"unsafe_rate":0.3125,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.6875,"conditional_unsafe":0.3125,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.44404355834740666,"upper":0.8583535614521796,"confidence_level":0.95}},"effective":{"n":16,"acceptable":0,"unsafe":0,"review":16,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":16,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.1936076805344365,"confidence_level":0.95}}},"mask":{"n":2,"raw":{"n":2,"acceptable":0,"unsafe":0,"review":0,"automated_count":2,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.6576197724933469,"confidence_level":0.95}},"effective":{"n":2,"acceptable":0,"unsafe":0,"review":2,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":2,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.6576197724933469,"confidence_level":0.95}}},"pseudonymize":{"n":2,"raw":{"n":2,"acceptable":2,"unsafe":0,"review":0,"automated_count":2,"automated_acceptable":2,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":1.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.34238022750665303,"upper":1.0,"confidence_level":0.95}},"effective":{"n":2,"acceptable":0,"unsafe":0,"review":2,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":2,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.6576197724933469,"confidence_level":0.95}}},"generalize":{"n":6,"raw":{"n":6,"acceptable":0,"unsafe":0,"review":0,"automated_count":6,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":2.7755575615628914e-17,"upper":0.3903342879021653,"confidence_level":0.95}},"effective":{"n":6,"acceptable":0,"unsafe":0,"review":6,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":6,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":2.7755575615628914e-17,"upper":0.3903342879021653,"confidence_level":0.95}}},"review":{"n":0,"raw":{"n":0,"acceptable":0,"unsafe":0,"review":0,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":null,"unsafe_rate":null,"review_rate":null,"automated_coverage":null,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":null},"effective":{"n":0,"acceptable":0,"unsafe":0,"review":0,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":null,"unsafe_rate":null,"review_rate":null,"automated_coverage":null,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":null}}}},"diagnostic43-compact_no_samples":{"n":43,"raw":{"n":43,"acceptable":12,"unsafe":4,"review":0,"automated_count":43,"automated_acceptable":12,"automated_unsafe":4,"unnecessary_reviews":0,"accuracy":0.27906976744186046,"unsafe_rate":0.09302325581395349,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.27906976744186046,"conditional_unsafe":0.09302325581395349,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.16746613786237488,"upper":0.42691028681878873,"confidence_level":0.95}},"effective":{"n":43,"acceptable":5,"unsafe":0,"review":43,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":38,"accuracy":0.11627906976744186,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":0.8837209302325582,"accuracy_wilson_95":{"lower":0.05070427739804166,"upper":0.24479161810082128,"confidence_level":0.95}},"confusion":{"raw":{"keep":{"keep":0,"drop":9,"mask":2,"pseudonymize":1,"generalize":0,"review":0},"drop":{"keep":1,"drop":8,"mask":6,"pseudonymize":1,"generalize":0,"review":0},"mask":{"keep":0,"drop":2,"mask":0,"pseudonymize":0,"generalize":0,"review":0},"pseudonymize":{"keep":0,"drop":1,"mask":0,"pseudonymize":1,"generalize":0,"review":0},"generalize":{"keep":0,"drop":3,"mask":1,"pseudonymize":2,"generalize":0,"review":0},"review":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0}},"effective":{"keep":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":12},"drop":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":16},"mask":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":2},"pseudonymize":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":2},"generalize":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":6},"review":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0}}},"probability_metrics":{"singleton_n":38,"multiacceptable_n":5,"multiacceptable_coverage":0.11627906976744186,"nll":1.7725752158177184,"nll_zero_mass_n":0,"brier":0.8267311575517811,"acceptable_set_n":43,"acceptable_set_nll":1.6425750475764305,"acceptable_set_zero_mass_n":0},"calibration_all_labels":{"equal_width":{"n":43,"bin_count":10,"ece":0.09177713200103405,"signed_mean_gap":-0.09177713200103405,"absolute_mean_gap":0.09177713200103405,"bins":[{"index":0,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.0,"upper":0.1,"upper_inclusive":false},{"index":1,"count":40,"confidence":0.18610229082405566,"empirical_correctness":0.25,"signed_gap":-0.06389770917594434,"absolute_gap":0.06389770917594434,"lower":0.1,"upper":0.2,"upper_inclusive":false},{"index":2,"count":3,"confidence":0.20316389699776968,"empirical_correctness":0.6666666666666666,"signed_gap":-0.4635027696688969,"absolute_gap":0.4635027696688969,"lower":0.2,"upper":0.3,"upper_inclusive":false},{"index":3,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.3,"upper":0.4,"upper_inclusive":false},{"index":4,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.4,"upper":0.5,"upper_inclusive":false},{"index":5,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.5,"upper":0.6,"upper_inclusive":false},{"index":6,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.6,"upper":0.7,"upper_inclusive":false},{"index":7,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.7,"upper":0.8,"upper_inclusive":false},{"index":8,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.8,"upper":0.9,"upper_inclusive":false},{"index":9,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.9,"upper":1.0,"upper_inclusive":true}]},"equal_mass":{"n":43,"bin_count":10,"ece":0.22844345902287683,"signed_mean_gap":-0.09177713200103405,"absolute_mean_gap":0.09177713200103405,"bins":[{"index":0,"count":4,"confidence":0.17572534456849098,"empirical_correctness":0.0,"signed_gap":0.17572534456849098,"absolute_gap":0.17572534456849098,"min_confidence":0.17293839156627655,"max_confidence":0.17698664963245392},{"index":1,"count":4,"confidence":0.17907942831516266,"empirical_correctness":0.5,"signed_gap":-0.32092057168483734,"absolute_gap":0.32092057168483734,"min_confidence":0.1775376945734024,"max_confidence":0.18092358112335205},{"index":2,"count":4,"confidence":0.1814609244465828,"empirical_correctness":0.0,"signed_gap":0.1814609244465828,"absolute_gap":0.1814609244465828,"min_confidence":0.18099594116210938,"max_confidence":0.18193714320659637},{"index":3,"count":5,"confidence":0.183246910572052,"empirical_correctness":0.4,"signed_gap":-0.21675308942794802,"absolute_gap":0.21675308942794802,"min_confidence":0.18208570778369904,"max_confidence":0.18379369378089905},{"index":4,"count":4,"confidence":0.18461284041404724,"empirical_correctness":0.0,"signed_gap":0.18461284041404724,"absolute_gap":0.18461284041404724,"min_confidence":0.18399351835250854,"max_confidence":0.18523350358009338},{"index":5,"count":4,"confidence":0.18646208941936493,"empirical_correctness":0.25,"signed_gap":-0.06353791058063507,"absolute_gap":0.06353791058063507,"min_confidence":0.1854892075061798,"max_confidence":0.18699972331523895},{"index":6,"count":5,"confidence":0.18939274549484253,"empirical_correctness":0.2,"signed_gap":-0.010607254505157482,"absolute_gap":0.010607254505157482,"min_confidence":0.18727940320968628,"max_confidence":0.1909312605857849},{"index":7,"count":4,"confidence":0.19278239831328392,"empirical_correctness":0.0,"signed_gap":0.19278239831328392,"absolute_gap":0.19278239831328392,"min_confidence":0.19105422496795654,"max_confidence":0.19398099184036255},{"index":8,"count":4,"confidence":0.1960558220744133,"empirical_correctness":0.75,"signed_gap":-0.5539441779255867,"absolute_gap":0.5539441779255867,"min_confidence":0.19506195187568665,"max_confidence":0.1968483328819275},{"index":9,"count":5,"confidence":0.20113393068313598,"empirical_correctness":0.6,"signed_gap":-0.39886606931686397,"absolute_gap":0.39886606931686397,"min_confidence":0.1974502056837082,"max_confidence":0.2063887119293213}]}},"calibration_singleton_labels":{"equal_width":{"n":38,"bin_count":10,"ece":0.049403955277643685,"signed_mean_gap":-0.049403955277643685,"absolute_mean_gap":0.049403955277643685,"bins":[{"index":0,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.0,"upper":0.1,"upper_inclusive":false},{"index":1,"count":36,"confidence":0.18652360679374802,"empirical_correctness":0.2222222222222222,"signed_gap":-0.03569861542847419,"absolute_gap":0.03569861542847419,"lower":0.1,"upper":0.2,"upper_inclusive":false},{"index":2,"count":2,"confidence":0.20389992743730545,"empirical_correctness":0.5,"signed_gap":-0.29610007256269455,"absolute_gap":0.29610007256269455,"lower":0.2,"upper":0.3,"upper_inclusive":false},{"index":3,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.3,"upper":0.4,"upper_inclusive":false},{"index":4,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.4,"upper":0.5,"upper_inclusive":false},{"index":5,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.5,"upper":0.6,"upper_inclusive":false},{"index":6,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.6,"upper":0.7,"upper_inclusive":false},{"index":7,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.7,"upper":0.8,"upper_inclusive":false},{"index":8,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.8,"upper":0.9,"upper_inclusive":false},{"index":9,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.9,"upper":1.0,"upper_inclusive":true}]},"equal_mass":{"n":38,"bin_count":10,"ece":0.16678041847128616,"signed_mean_gap":-0.0494039552776437,"absolute_mean_gap":0.0494039552776437,"bins":[{"index":0,"count":3,"confidence":0.17625277241071066,"empirical_correctness":0.3333333333333333,"signed_gap":-0.15708056092262265,"absolute_gap":0.15708056092262265,"min_confidence":0.17293839156627655,"max_confidence":0.1788332760334015},{"index":1,"count":4,"confidence":0.18053147196769714,"empirical_correctness":0.0,"signed_gap":0.18053147196769714,"absolute_gap":0.18053147196769714,"min_confidence":0.1790231615304947,"max_confidence":0.18118320405483246},{"index":2,"count":4,"confidence":0.18223229050636292,"empirical_correctness":0.25,"signed_gap":-0.06776770949363708,"absolute_gap":0.06776770949363708,"min_confidence":0.18172740936279297,"max_confidence":0.18317890167236328},{"index":3,"count":4,"confidence":0.18374086543917656,"empirical_correctness":0.25,"signed_gap":-0.06625913456082344,"absolute_gap":0.06625913456082344,"min_confidence":0.18340986967086792,"max_confidence":0.18399351835250854},{"index":4,"count":4,"confidence":0.18498676270246506,"empirical_correctness":0.0,"signed_gap":0.18498676270246506,"absolute_gap":0.18498676270246506,"min_confidence":0.18451036512851715,"max_confidence":0.1854892075061798},{"index":5,"count":3,"confidence":0.18678638339042664,"empirical_correctness":0.3333333333333333,"signed_gap":-0.14654694994290668,"absolute_gap":0.14654694994290668,"min_confidence":0.18643297255039215,"max_confidence":0.18699972331523895},{"index":6,"count":4,"confidence":0.18900811672210693,"empirical_correctness":0.25,"signed_gap":-0.060991883277893066,"absolute_gap":0.060991883277893066,"min_confidence":0.18727940320968628,"max_confidence":0.1906326711177826},{"index":7,"count":4,"confidence":0.1920199654996395,"empirical_correctness":0.0,"signed_gap":0.1920199654996395,"absolute_gap":0.1920199654996395,"min_confidence":0.1909312605857849,"max_confidence":0.19371484220027924},{"index":8,"count":4,"confidence":0.19533898681402206,"empirical_correctness":0.75,"signed_gap":-0.5546610131859779,"absolute_gap":0.5546610131859779,"min_confidence":0.19398099184036255,"max_confidence":0.1962844431400299},{"index":9,"count":4,"confidence":0.20052459836006165,"empirical_correctness":0.25,"signed_gap":-0.049475401639938354,"absolute_gap":0.049475401639938354,"min_confidence":0.1968483328819275,"max_confidence":0.2063887119293213}]}},"latency_ms":{"n":43,"median":134.9739280412905,"p95":137.63727302430198},"per_action":{"keep":{"n":12,"raw":{"n":12,"acceptable":0,"unsafe":0,"review":0,"automated_count":12,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":2.7755575615628914e-17,"upper":0.24249400665524085,"confidence_level":0.95}},"effective":{"n":12,"acceptable":0,"unsafe":0,"review":12,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":12,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":2.7755575615628914e-17,"upper":0.24249400665524085,"confidence_level":0.95}}},"drop":{"n":16,"raw":{"n":16,"acceptable":8,"unsafe":2,"review":0,"automated_count":16,"automated_acceptable":8,"automated_unsafe":2,"unnecessary_reviews":0,"accuracy":0.5,"unsafe_rate":0.125,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.5,"conditional_unsafe":0.125,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.27999563610326017,"upper":0.7200043638967398,"confidence_level":0.95}},"effective":{"n":16,"acceptable":0,"unsafe":0,"review":16,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":16,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.1936076805344365,"confidence_level":0.95}}},"mask":{"n":2,"raw":{"n":2,"acceptable":0,"unsafe":0,"review":0,"automated_count":2,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.6576197724933469,"confidence_level":0.95}},"effective":{"n":2,"acceptable":0,"unsafe":0,"review":2,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":2,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.6576197724933469,"confidence_level":0.95}}},"pseudonymize":{"n":2,"raw":{"n":2,"acceptable":1,"unsafe":0,"review":0,"automated_count":2,"automated_acceptable":1,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":0.5,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.5,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.09453120573423074,"upper":0.9054687942657693,"confidence_level":0.95}},"effective":{"n":2,"acceptable":0,"unsafe":0,"review":2,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":2,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.6576197724933469,"confidence_level":0.95}}},"generalize":{"n":6,"raw":{"n":6,"acceptable":0,"unsafe":0,"review":0,"automated_count":6,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":2.7755575615628914e-17,"upper":0.3903342879021653,"confidence_level":0.95}},"effective":{"n":6,"acceptable":0,"unsafe":0,"review":6,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":6,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":2.7755575615628914e-17,"upper":0.3903342879021653,"confidence_level":0.95}}},"review":{"n":0,"raw":{"n":0,"acceptable":0,"unsafe":0,"review":0,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":null,"unsafe_rate":null,"review_rate":null,"automated_coverage":null,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":null},"effective":{"n":0,"acceptable":0,"unsafe":0,"review":0,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":null,"unsafe_rate":null,"review_rate":null,"automated_coverage":null,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":null}}}},"challenge56-no_samples":{"n":56,"raw":{"n":56,"acceptable":15,"unsafe":38,"review":0,"automated_count":56,"automated_acceptable":15,"automated_unsafe":38,"unnecessary_reviews":0,"accuracy":0.26785714285714285,"unsafe_rate":0.6785714285714286,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.26785714285714285,"conditional_unsafe":0.6785714285714286,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.16957305418501728,"upper":0.39594555929155156,"confidence_level":0.95}},"effective":{"n":56,"acceptable":2,"unsafe":0,"review":56,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":54,"accuracy":0.03571428571428571,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":0.9642857142857143,"accuracy_wilson_95":{"lower":0.00984942514823639,"upper":0.12118780180490107,"confidence_level":0.95}},"confusion":{"raw":{"keep":{"keep":7,"drop":0,"mask":0,"pseudonymize":1,"generalize":0,"review":0},"drop":{"keep":21,"drop":0,"mask":0,"pseudonymize":1,"generalize":0,"review":0},"mask":{"keep":8,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0},"pseudonymize":{"keep":0,"drop":0,"mask":0,"pseudonymize":8,"generalize":0,"review":0},"generalize":{"keep":6,"drop":0,"mask":0,"pseudonymize":2,"generalize":0,"review":0},"review":{"keep":1,"drop":0,"mask":0,"pseudonymize":1,"generalize":0,"review":0}},"effective":{"keep":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":8},"drop":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":22},"mask":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":8},"pseudonymize":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":8},"generalize":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":8},"review":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":2}}},"probability_metrics":{"singleton_n":56,"multiacceptable_n":0,"multiacceptable_coverage":0.0,"nll":1.7139123902864413,"nll_zero_mass_n":0,"brier":0.8075014794551688,"acceptable_set_n":56,"acceptable_set_nll":1.7139123902864413,"acceptable_set_zero_mass_n":0},"calibration_all_labels":{"equal_width":{"n":56,"bin_count":10,"ece":0.14326872383909564,"signed_mean_gap":-0.05919009739799156,"absolute_mean_gap":0.05919009739799156,"bins":[{"index":0,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.0,"upper":0.1,"upper_inclusive":false},{"index":1,"count":12,"confidence":0.1961834616959095,"empirical_correctness":0.0,"signed_gap":0.1961834616959095,"absolute_gap":0.1961834616959095,"lower":0.1,"upper":0.2,"upper_inclusive":false},{"index":2,"count":44,"confidence":0.21207165921276266,"empirical_correctness":0.3409090909090909,"signed_gap":-0.12883743169632822,"absolute_gap":0.12883743169632822,"lower":0.2,"upper":0.3,"upper_inclusive":false},{"index":3,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.3,"upper":0.4,"upper_inclusive":false},{"index":4,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.4,"upper":0.5,"upper_inclusive":false},{"index":5,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.5,"upper":0.6,"upper_inclusive":false},{"index":6,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.6,"upper":0.7,"upper_inclusive":false},{"index":7,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.7,"upper":0.8,"upper_inclusive":false},{"index":8,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.8,"upper":0.9,"upper_inclusive":false},{"index":9,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.9,"upper":1.0,"upper_inclusive":true}]},"equal_mass":{"n":56,"bin_count":10,"ece":0.3418510809008564,"signed_mean_gap":-0.05919009739799159,"absolute_mean_gap":0.05919009739799159,"bins":[{"index":0,"count":5,"confidence":0.1936587393283844,"empirical_correctness":0.0,"signed_gap":0.1936587393283844,"absolute_gap":0.1936587393283844,"min_confidence":0.19102691113948822,"max_confidence":0.1960998922586441},{"index":1,"count":6,"confidence":0.19779677192370096,"empirical_correctness":0.0,"signed_gap":0.19779677192370096,"absolute_gap":0.19779677192370096,"min_confidence":0.196617990732193,"max_confidence":0.1988435834646225},{"index":2,"count":5,"confidence":0.2009166568517685,"empirical_correctness":0.0,"signed_gap":0.2009166568517685,"absolute_gap":0.2009166568517685,"min_confidence":0.1991272121667862,"max_confidence":0.2018769085407257},{"index":3,"count":6,"confidence":0.20308426519234976,"empirical_correctness":0.0,"signed_gap":0.20308426519234976,"absolute_gap":0.20308426519234976,"min_confidence":0.20248518884181976,"max_confidence":0.20411553978919983},{"index":4,"count":6,"confidence":0.20534716298182806,"empirical_correctness":0.0,"signed_gap":0.20534716298182806,"absolute_gap":0.20534716298182806,"min_confidence":0.20425407588481903,"max_confidence":0.2063746154308319},{"index":5,"count":5,"confidence":0.20764748454093934,"empirical_correctness":0.0,"signed_gap":0.20764748454093934,"absolute_gap":0.20764748454093934,"min_confidence":0.20720665156841278,"max_confidence":0.20804433524608612},{"index":6,"count":6,"confidence":0.21100398898124695,"empirical_correctness":0.0,"signed_gap":0.21100398898124695,"absolute_gap":0.21100398898124695,"min_confidence":0.20841377973556519,"max_confidence":0.2128940224647522},{"index":7,"count":5,"confidence":0.21385634541511536,"empirical_correctness":1.0,"signed_gap":-0.7861436545848847,"absolute_gap":0.7861436545848847,"min_confidence":0.21296876668930054,"max_confidence":0.21546335518360138},{"index":8,"count":6,"confidence":0.21995059897502264,"empirical_correctness":0.8333333333333334,"signed_gap":-0.6133827343583107,"absolute_gap":0.6133827343583107,"min_confidence":0.21688318252563477,"max_confidence":0.22329741716384888},{"index":9,"count":6,"confidence":0.23031028111775717,"empirical_correctness":0.8333333333333334,"signed_gap":-0.6030230522155762,"absolute_gap":0.6030230522155762,"min_confidence":0.22379611432552338,"max_confidence":0.24530670046806335}]}},"calibration_singleton_labels":{"equal_width":{"n":56,"bin_count":10,"ece":0.14326872383909564,"signed_mean_gap":-0.05919009739799156,"absolute_mean_gap":0.05919009739799156,"bins":[{"index":0,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.0,"upper":0.1,"upper_inclusive":false},{"index":1,"count":12,"confidence":0.1961834616959095,"empirical_correctness":0.0,"signed_gap":0.1961834616959095,"absolute_gap":0.1961834616959095,"lower":0.1,"upper":0.2,"upper_inclusive":false},{"index":2,"count":44,"confidence":0.21207165921276266,"empirical_correctness":0.3409090909090909,"signed_gap":-0.12883743169632822,"absolute_gap":0.12883743169632822,"lower":0.2,"upper":0.3,"upper_inclusive":false},{"index":3,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.3,"upper":0.4,"upper_inclusive":false},{"index":4,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.4,"upper":0.5,"upper_inclusive":false},{"index":5,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.5,"upper":0.6,"upper_inclusive":false},{"index":6,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.6,"upper":0.7,"upper_inclusive":false},{"index":7,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.7,"upper":0.8,"upper_inclusive":false},{"index":8,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.8,"upper":0.9,"upper_inclusive":false},{"index":9,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.9,"upper":1.0,"upper_inclusive":true}]},"equal_mass":{"n":56,"bin_count":10,"ece":0.3418510809008564,"signed_mean_gap":-0.05919009739799159,"absolute_mean_gap":0.05919009739799159,"bins":[{"index":0,"count":5,"confidence":0.1936587393283844,"empirical_correctness":0.0,"signed_gap":0.1936587393283844,"absolute_gap":0.1936587393283844,"min_confidence":0.19102691113948822,"max_confidence":0.1960998922586441},{"index":1,"count":6,"confidence":0.19779677192370096,"empirical_correctness":0.0,"signed_gap":0.19779677192370096,"absolute_gap":0.19779677192370096,"min_confidence":0.196617990732193,"max_confidence":0.1988435834646225},{"index":2,"count":5,"confidence":0.2009166568517685,"empirical_correctness":0.0,"signed_gap":0.2009166568517685,"absolute_gap":0.2009166568517685,"min_confidence":0.1991272121667862,"max_confidence":0.2018769085407257},{"index":3,"count":6,"confidence":0.20308426519234976,"empirical_correctness":0.0,"signed_gap":0.20308426519234976,"absolute_gap":0.20308426519234976,"min_confidence":0.20248518884181976,"max_confidence":0.20411553978919983},{"index":4,"count":6,"confidence":0.20534716298182806,"empirical_correctness":0.0,"signed_gap":0.20534716298182806,"absolute_gap":0.20534716298182806,"min_confidence":0.20425407588481903,"max_confidence":0.2063746154308319},{"index":5,"count":5,"confidence":0.20764748454093934,"empirical_correctness":0.0,"signed_gap":0.20764748454093934,"absolute_gap":0.20764748454093934,"min_confidence":0.20720665156841278,"max_confidence":0.20804433524608612},{"index":6,"count":6,"confidence":0.21100398898124695,"empirical_correctness":0.0,"signed_gap":0.21100398898124695,"absolute_gap":0.21100398898124695,"min_confidence":0.20841377973556519,"max_confidence":0.2128940224647522},{"index":7,"count":5,"confidence":0.21385634541511536,"empirical_correctness":1.0,"signed_gap":-0.7861436545848847,"absolute_gap":0.7861436545848847,"min_confidence":0.21296876668930054,"max_confidence":0.21546335518360138},{"index":8,"count":6,"confidence":0.21995059897502264,"empirical_correctness":0.8333333333333334,"signed_gap":-0.6133827343583107,"absolute_gap":0.6133827343583107,"min_confidence":0.21688318252563477,"max_confidence":0.22329741716384888},{"index":9,"count":6,"confidence":0.23031028111775717,"empirical_correctness":0.8333333333333334,"signed_gap":-0.6030230522155762,"absolute_gap":0.6030230522155762,"min_confidence":0.22379611432552338,"max_confidence":0.24530670046806335}]}},"latency_ms":{"n":56,"median":118.71233701822348,"p95":121.89309849054553},"per_action":{"keep":{"n":8,"raw":{"n":8,"acceptable":7,"unsafe":0,"review":0,"automated_count":8,"automated_acceptable":7,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":0.875,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.875,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.5291118177871464,"upper":0.9775825085499433,"confidence_level":0.95}},"effective":{"n":8,"acceptable":0,"unsafe":0,"review":8,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":8,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.32440756488388023,"confidence_level":0.95}}},"drop":{"n":22,"raw":{"n":22,"acceptable":0,"unsafe":22,"review":0,"automated_count":22,"automated_acceptable":0,"automated_unsafe":22,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":1.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":1.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.14865487460862087,"confidence_level":0.95}},"effective":{"n":22,"acceptable":0,"unsafe":0,"review":22,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":22,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.14865487460862087,"confidence_level":0.95}}},"mask":{"n":8,"raw":{"n":8,"acceptable":0,"unsafe":8,"review":0,"automated_count":8,"automated_acceptable":0,"automated_unsafe":8,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":1.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":1.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.32440756488388023,"confidence_level":0.95}},"effective":{"n":8,"acceptable":0,"unsafe":0,"review":8,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":8,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.32440756488388023,"confidence_level":0.95}}},"pseudonymize":{"n":8,"raw":{"n":8,"acceptable":8,"unsafe":0,"review":0,"automated_count":8,"automated_acceptable":8,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":1.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.6755924351161198,"upper":1.0,"confidence_level":0.95}},"effective":{"n":8,"acceptable":0,"unsafe":0,"review":8,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":8,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.32440756488388023,"confidence_level":0.95}}},"generalize":{"n":8,"raw":{"n":8,"acceptable":0,"unsafe":6,"review":0,"automated_count":8,"automated_acceptable":0,"automated_unsafe":6,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":0.75,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":0.75,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.32440756488388023,"confidence_level":0.95}},"effective":{"n":8,"acceptable":0,"unsafe":0,"review":8,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":8,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.32440756488388023,"confidence_level":0.95}}},"review":{"n":2,"raw":{"n":2,"acceptable":0,"unsafe":2,"review":0,"automated_count":2,"automated_acceptable":0,"automated_unsafe":2,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":1.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":1.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.6576197724933469,"confidence_level":0.95}},"effective":{"n":2,"acceptable":2,"unsafe":0,"review":2,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":1.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.34238022750665303,"upper":1.0,"confidence_level":0.95}}}}},"challenge56-compact_no_samples":{"n":56,"raw":{"n":56,"acceptable":28,"unsafe":16,"review":0,"automated_count":56,"automated_acceptable":28,"automated_unsafe":16,"unnecessary_reviews":0,"accuracy":0.5,"unsafe_rate":0.2857142857142857,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.5,"conditional_unsafe":0.2857142857142857,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.37331738801362335,"upper":0.6266826119863766,"confidence_level":0.95}},"effective":{"n":56,"acceptable":2,"unsafe":0,"review":56,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":54,"accuracy":0.03571428571428571,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":0.9642857142857143,"accuracy_wilson_95":{"lower":0.00984942514823639,"upper":0.12118780180490107,"confidence_level":0.95}},"confusion":{"raw":{"keep":{"keep":3,"drop":0,"mask":0,"pseudonymize":5,"generalize":0,"review":0},"drop":{"keep":13,"drop":3,"mask":3,"pseudonymize":1,"generalize":2,"review":0},"mask":{"keep":0,"drop":0,"mask":8,"pseudonymize":0,"generalize":0,"review":0},"pseudonymize":{"keep":0,"drop":0,"mask":0,"pseudonymize":8,"generalize":0,"review":0},"generalize":{"keep":0,"drop":0,"mask":0,"pseudonymize":2,"generalize":6,"review":0},"review":{"keep":0,"drop":0,"mask":0,"pseudonymize":2,"generalize":0,"review":0}},"effective":{"keep":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":8},"drop":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":22},"mask":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":8},"pseudonymize":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":8},"generalize":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":8},"review":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":2}}},"probability_metrics":{"singleton_n":56,"multiacceptable_n":0,"multiacceptable_coverage":0.0,"nll":1.7168809887223304,"nll_zero_mass_n":0,"brier":0.8077527156310704,"acceptable_set_n":56,"acceptable_set_nll":1.7168809887223304,"acceptable_set_zero_mass_n":0},"calibration_all_labels":{"equal_width":{"n":56,"bin_count":10,"ece":0.3092000976737056,"signed_mean_gap":-0.3092000976737056,"absolute_mean_gap":0.3092000976737056,"bins":[{"index":0,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.0,"upper":0.1,"upper_inclusive":false},{"index":1,"count":45,"confidence":0.18685828083091313,"empirical_correctness":0.4444444444444444,"signed_gap":-0.25758616361353126,"absolute_gap":0.25758616361353126,"lower":0.1,"upper":0.2,"upper_inclusive":false},{"index":2,"count":11,"confidence":0.20692471753467212,"empirical_correctness":0.7272727272727273,"signed_gap":-0.5203480097380552,"absolute_gap":0.5203480097380552,"lower":0.2,"upper":0.3,"upper_inclusive":false},{"index":3,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.3,"upper":0.4,"upper_inclusive":false},{"index":4,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.4,"upper":0.5,"upper_inclusive":false},{"index":5,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.5,"upper":0.6,"upper_inclusive":false},{"index":6,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.6,"upper":0.7,"upper_inclusive":false},{"index":7,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.7,"upper":0.8,"upper_inclusive":false},{"index":8,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.8,"upper":0.9,"upper_inclusive":false},{"index":9,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.9,"upper":1.0,"upper_inclusive":true}]},"equal_mass":{"n":56,"bin_count":10,"ece":0.345228962334139,"signed_mean_gap":-0.30920009767370565,"absolute_mean_gap":0.30920009767370565,"bins":[{"index":0,"count":5,"confidence":0.17415379881858825,"empirical_correctness":0.2,"signed_gap":-0.02584620118141176,"absolute_gap":0.02584620118141176,"min_confidence":0.17350734770298004,"max_confidence":0.17563796043395996},{"index":1,"count":6,"confidence":0.17911231766144434,"empirical_correctness":0.3333333333333333,"signed_gap":-0.15422101567188898,"absolute_gap":0.15422101567188898,"min_confidence":0.17592047154903412,"max_confidence":0.18098196387290955},{"index":2,"count":5,"confidence":0.18150971829891205,"empirical_correctness":0.0,"signed_gap":0.18150971829891205,"absolute_gap":0.18150971829891205,"min_confidence":0.18101300299167633,"max_confidence":0.18206720054149628},{"index":3,"count":6,"confidence":0.18354326983292898,"empirical_correctness":0.16666666666666666,"signed_gap":0.016876603166262327,"absolute_gap":0.016876603166262327,"min_confidence":0.18262524902820587,"max_confidence":0.18469613790512085},{"index":4,"count":6,"confidence":0.189310389260451,"empirical_correctness":0.8333333333333334,"signed_gap":-0.6440229440728824,"absolute_gap":0.6440229440728824,"min_confidence":0.18633924424648285,"max_confidence":0.19087259471416473},{"index":5,"count":5,"confidence":0.19238348305225372,"empirical_correctness":0.8,"signed_gap":-0.6076165169477463,"absolute_gap":0.6076165169477463,"min_confidence":0.1914699226617813,"max_confidence":0.19398488104343414},{"index":6,"count":6,"confidence":0.194966288904349,"empirical_correctness":0.6666666666666666,"signed_gap":-0.47170037776231766,"absolute_gap":0.47170037776231766,"min_confidence":0.19418378174304962,"max_confidence":0.19563673436641693},{"index":7,"count":5,"confidence":0.19747745394706726,"empirical_correctness":0.4,"signed_gap":-0.20252254605293277,"absolute_gap":0.20252254605293277,"min_confidence":0.1968897134065628,"max_confidence":0.19818495213985443},{"index":8,"count":6,"confidence":0.20208314309517542,"empirical_correctness":1.0,"signed_gap":-0.7979168569048246,"absolute_gap":0.7979168569048246,"min_confidence":0.19940677285194397,"max_confidence":0.20447391271591187},{"index":9,"count":6,"confidence":0.21051330119371414,"empirical_correctness":0.5,"signed_gap":-0.28948669880628586,"absolute_gap":0.28948669880628586,"min_confidence":0.20572710037231445,"max_confidence":0.215020552277565}]}},"calibration_singleton_labels":{"equal_width":{"n":56,"bin_count":10,"ece":0.3092000976737056,"signed_mean_gap":-0.3092000976737056,"absolute_mean_gap":0.3092000976737056,"bins":[{"index":0,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.0,"upper":0.1,"upper_inclusive":false},{"index":1,"count":45,"confidence":0.18685828083091313,"empirical_correctness":0.4444444444444444,"signed_gap":-0.25758616361353126,"absolute_gap":0.25758616361353126,"lower":0.1,"upper":0.2,"upper_inclusive":false},{"index":2,"count":11,"confidence":0.20692471753467212,"empirical_correctness":0.7272727272727273,"signed_gap":-0.5203480097380552,"absolute_gap":0.5203480097380552,"lower":0.2,"upper":0.3,"upper_inclusive":false},{"index":3,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.3,"upper":0.4,"upper_inclusive":false},{"index":4,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.4,"upper":0.5,"upper_inclusive":false},{"index":5,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.5,"upper":0.6,"upper_inclusive":false},{"index":6,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.6,"upper":0.7,"upper_inclusive":false},{"index":7,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.7,"upper":0.8,"upper_inclusive":false},{"index":8,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.8,"upper":0.9,"upper_inclusive":false},{"index":9,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.9,"upper":1.0,"upper_inclusive":true}]},"equal_mass":{"n":56,"bin_count":10,"ece":0.345228962334139,"signed_mean_gap":-0.30920009767370565,"absolute_mean_gap":0.30920009767370565,"bins":[{"index":0,"count":5,"confidence":0.17415379881858825,"empirical_correctness":0.2,"signed_gap":-0.02584620118141176,"absolute_gap":0.02584620118141176,"min_confidence":0.17350734770298004,"max_confidence":0.17563796043395996},{"index":1,"count":6,"confidence":0.17911231766144434,"empirical_correctness":0.3333333333333333,"signed_gap":-0.15422101567188898,"absolute_gap":0.15422101567188898,"min_confidence":0.17592047154903412,"max_confidence":0.18098196387290955},{"index":2,"count":5,"confidence":0.18150971829891205,"empirical_correctness":0.0,"signed_gap":0.18150971829891205,"absolute_gap":0.18150971829891205,"min_confidence":0.18101300299167633,"max_confidence":0.18206720054149628},{"index":3,"count":6,"confidence":0.18354326983292898,"empirical_correctness":0.16666666666666666,"signed_gap":0.016876603166262327,"absolute_gap":0.016876603166262327,"min_confidence":0.18262524902820587,"max_confidence":0.18469613790512085},{"index":4,"count":6,"confidence":0.189310389260451,"empirical_correctness":0.8333333333333334,"signed_gap":-0.6440229440728824,"absolute_gap":0.6440229440728824,"min_confidence":0.18633924424648285,"max_confidence":0.19087259471416473},{"index":5,"count":5,"confidence":0.19238348305225372,"empirical_correctness":0.8,"signed_gap":-0.6076165169477463,"absolute_gap":0.6076165169477463,"min_confidence":0.1914699226617813,"max_confidence":0.19398488104343414},{"index":6,"count":6,"confidence":0.194966288904349,"empirical_correctness":0.6666666666666666,"signed_gap":-0.47170037776231766,"absolute_gap":0.47170037776231766,"min_confidence":0.19418378174304962,"max_confidence":0.19563673436641693},{"index":7,"count":5,"confidence":0.19747745394706726,"empirical_correctness":0.4,"signed_gap":-0.20252254605293277,"absolute_gap":0.20252254605293277,"min_confidence":0.1968897134065628,"max_confidence":0.19818495213985443},{"index":8,"count":6,"confidence":0.20208314309517542,"empirical_correctness":1.0,"signed_gap":-0.7979168569048246,"absolute_gap":0.7979168569048246,"min_confidence":0.19940677285194397,"max_confidence":0.20447391271591187},{"index":9,"count":6,"confidence":0.21051330119371414,"empirical_correctness":0.5,"signed_gap":-0.28948669880628586,"absolute_gap":0.28948669880628586,"min_confidence":0.20572710037231445,"max_confidence":0.215020552277565}]}},"latency_ms":{"n":56,"median":118.59639748581685,"p95":122.13269625499379},"per_action":{"keep":{"n":8,"raw":{"n":8,"acceptable":3,"unsafe":0,"review":0,"automated_count":8,"automated_acceptable":3,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":0.375,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.375,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.13684428582359742,"upper":0.6942576053973727,"confidence_level":0.95}},"effective":{"n":8,"acceptable":0,"unsafe":0,"review":8,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":8,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.32440756488388023,"confidence_level":0.95}}},"drop":{"n":22,"raw":{"n":22,"acceptable":3,"unsafe":14,"review":0,"automated_count":22,"automated_acceptable":3,"automated_unsafe":14,"unnecessary_reviews":0,"accuracy":0.13636363636363635,"unsafe_rate":0.6363636363636364,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.13636363636363635,"conditional_unsafe":0.6363636363636364,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.04749003714055253,"upper":0.33334987166571717,"confidence_level":0.95}},"effective":{"n":22,"acceptable":0,"unsafe":0,"review":22,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":22,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.14865487460862087,"confidence_level":0.95}}},"mask":{"n":8,"raw":{"n":8,"acceptable":8,"unsafe":0,"review":0,"automated_count":8,"automated_acceptable":8,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":1.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.6755924351161198,"upper":1.0,"confidence_level":0.95}},"effective":{"n":8,"acceptable":0,"unsafe":0,"review":8,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":8,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.32440756488388023,"confidence_level":0.95}}},"pseudonymize":{"n":8,"raw":{"n":8,"acceptable":8,"unsafe":0,"review":0,"automated_count":8,"automated_acceptable":8,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":1.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.6755924351161198,"upper":1.0,"confidence_level":0.95}},"effective":{"n":8,"acceptable":0,"unsafe":0,"review":8,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":8,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.32440756488388023,"confidence_level":0.95}}},"generalize":{"n":8,"raw":{"n":8,"acceptable":6,"unsafe":0,"review":0,"automated_count":8,"automated_acceptable":6,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":0.75,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.75,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.40927543031016883,"upper":0.9285207872478909,"confidence_level":0.95}},"effective":{"n":8,"acceptable":0,"unsafe":0,"review":8,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":8,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.32440756488388023,"confidence_level":0.95}}},"review":{"n":2,"raw":{"n":2,"acceptable":0,"unsafe":2,"review":0,"automated_count":2,"automated_acceptable":0,"automated_unsafe":2,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":1.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":1.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.6576197724933469,"confidence_level":0.95}},"effective":{"n":2,"acceptable":2,"unsafe":0,"review":2,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":1.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.34238022750665303,"upper":1.0,"confidence_level":0.95}}}}}},"composite":{"n":198,"scope":"descriptive sum across four correlated panels","raw":{"n":198,"acceptable":72,"unsafe":64,"review":0,"automated_count":198,"automated_acceptable":72,"automated_unsafe":64,"unnecessary_reviews":0,"accuracy":0.36363636363636365,"unsafe_rate":0.32323232323232326,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.36363636363636365,"conditional_unsafe":0.32323232323232326,"unnecessary_review_rate":0.0},"effective":{"n":198,"acceptable":14,"unsafe":0,"review":198,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":184,"accuracy":0.0707070707070707,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":0.9292929292929293}}},{"id":"nli-modernbert-large-fp32","platform":"Local","model_id":"MoritzLaurer/ModernBERT-large-zeroshot-v2.0","revision":"a51e07b524299e309dd2b88d48b0cfa2bd9ec598","loading":"float32 / none","scoring_kind":"Adapted NLI","settings":{"dtype":"float32","quantization":null,"max_tokens":512,"effective_max_tokens":512,"temperature":1.0,"scoring_batch_size":1,"trust_remote_code":false,"local_files_only":true,"use_safetensors":true,"guard_threshold":0.45,"seed":0,"tf32":false},"scoring_metadata":{"backend":"nli","protocol":"hf-local-sweep-backends-v1","probability_source":"adapted_entailment_softmax_not_calibrated_sixway","latency_scope":"synchronous_forward_scoring_including_transfer_excluding_tokenization","precision":"float32","equal_mass_convention":"rank bins with floor boundaries at floor(i*n/k), k=min(10,n//2); minimum two rows"},"load_ms":1562.5020930310711,"peak_allocated_bytes":1640247296,"peak_reserved_bytes":1740636160,"panels":{"diagnostic43-no_samples":{"n":43,"raw":{"n":43,"acceptable":7,"unsafe":0,"review":21,"automated_count":22,"automated_acceptable":5,"automated_unsafe":0,"unnecessary_reviews":19,"accuracy":0.16279069767441862,"unsafe_rate":0.0,"review_rate":0.4883720930232558,"automated_coverage":0.5116279069767442,"conditional_correctness":0.22727272727272727,"conditional_unsafe":0.0,"unnecessary_review_rate":0.4418604651162791,"accuracy_wilson_95":{"lower":0.08116627347062008,"upper":0.29972405893747167,"confidence_level":0.95}},"effective":{"n":43,"acceptable":5,"unsafe":0,"review":43,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":38,"accuracy":0.11627906976744186,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":0.8837209302325582,"accuracy_wilson_95":{"lower":0.05070427739804166,"upper":0.24479161810082128,"confidence_level":0.95}},"confusion":{"raw":{"keep":{"keep":0,"drop":0,"mask":6,"pseudonymize":0,"generalize":0,"review":6},"drop":{"keep":0,"drop":1,"mask":8,"pseudonymize":0,"generalize":0,"review":7},"mask":{"keep":0,"drop":0,"mask":1,"pseudonymize":0,"generalize":0,"review":1},"pseudonymize":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":2},"generalize":{"keep":0,"drop":1,"mask":2,"pseudonymize":0,"generalize":0,"review":3},"review":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0}},"effective":{"keep":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":12},"drop":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":16},"mask":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":2},"pseudonymize":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":2},"generalize":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":6},"review":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0}}},"probability_metrics":{"singleton_n":38,"multiacceptable_n":5,"multiacceptable_coverage":0.11627906976744186,"nll":1.8165111749059275,"nll_zero_mass_n":0,"brier":0.8422346800370055,"acceptable_set_n":43,"acceptable_set_nll":1.6789544361401347,"acceptable_set_zero_mass_n":0},"calibration_all_labels":{"equal_width":{"n":43,"bin_count":10,"ece":0.03298834070216779,"signed_mean_gap":0.03298834070216779,"absolute_mean_gap":0.03298834070216779,"bins":[{"index":0,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.0,"upper":0.1,"upper_inclusive":false},{"index":1,"count":38,"confidence":0.1946592966192647,"empirical_correctness":0.15789473684210525,"signed_gap":0.03676455977715945,"absolute_gap":0.03676455977715945,"lower":0.1,"upper":0.2,"upper_inclusive":false},{"index":2,"count":5,"confidence":0.20428907573223115,"empirical_correctness":0.2,"signed_gap":0.00428907573223114,"absolute_gap":0.00428907573223114,"lower":0.2,"upper":0.3,"upper_inclusive":false},{"index":3,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.3,"upper":0.4,"upper_inclusive":false},{"index":4,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.4,"upper":0.5,"upper_inclusive":false},{"index":5,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.5,"upper":0.6,"upper_inclusive":false},{"index":6,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.6,"upper":0.7,"upper_inclusive":false},{"index":7,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.7,"upper":0.8,"upper_inclusive":false},{"index":8,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.8,"upper":0.9,"upper_inclusive":false},{"index":9,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.9,"upper":1.0,"upper_inclusive":true}]},"equal_mass":{"n":43,"bin_count":10,"ece":0.11270103447659072,"signed_mean_gap":0.03298834070216778,"absolute_mean_gap":0.03298834070216778,"bins":[{"index":0,"count":4,"confidence":0.18747391551733017,"empirical_correctness":0.0,"signed_gap":0.18747391551733017,"absolute_gap":0.18747391551733017,"min_confidence":0.1862446665763855,"max_confidence":0.18849293887615204},{"index":1,"count":4,"confidence":0.1904144585132599,"empirical_correctness":0.5,"signed_gap":-0.3095855414867401,"absolute_gap":0.3095855414867401,"min_confidence":0.18908287584781647,"max_confidence":0.19171570241451263},{"index":2,"count":4,"confidence":0.19234123080968857,"empirical_correctness":0.25,"signed_gap":-0.05765876919031143,"absolute_gap":0.05765876919031143,"min_confidence":0.19195714592933655,"max_confidence":0.19272254407405853},{"index":3,"count":5,"confidence":0.1940990060567856,"empirical_correctness":0.2,"signed_gap":-0.0059009939432144165,"absolute_gap":0.0059009939432144165,"min_confidence":0.1928299367427826,"max_confidence":0.1950913816690445},{"index":4,"count":4,"confidence":0.19558970630168915,"empirical_correctness":0.0,"signed_gap":0.19558970630168915,"absolute_gap":0.19558970630168915,"min_confidence":0.19509243965148926,"max_confidence":0.19595269858837128},{"index":5,"count":4,"confidence":0.19630055129528046,"empirical_correctness":0.0,"signed_gap":0.19630055129528046,"absolute_gap":0.19630055129528046,"min_confidence":0.19619998335838318,"max_confidence":0.19641907513141632},{"index":6,"count":5,"confidence":0.19736917018890382,"empirical_correctness":0.2,"signed_gap":-0.0026308298110961914,"absolute_gap":0.0026308298110961914,"min_confidence":0.19654949009418488,"max_confidence":0.1979101300239563},{"index":7,"count":4,"confidence":0.19835487380623817,"empirical_correctness":0.0,"signed_gap":0.19835487380623817,"absolute_gap":0.19835487380623817,"min_confidence":0.19812311232089996,"max_confidence":0.19857817888259888},{"index":8,"count":4,"confidence":0.19945336133241653,"empirical_correctness":0.25,"signed_gap":-0.050546638667583466,"absolute_gap":0.050546638667583466,"min_confidence":0.19889238476753235,"max_confidence":0.1997864842414856},{"index":9,"count":5,"confidence":0.20428907573223115,"empirical_correctness":0.2,"signed_gap":0.00428907573223114,"absolute_gap":0.00428907573223114,"min_confidence":0.2002348154783249,"max_confidence":0.20614846050739288}]}},"calibration_singleton_labels":{"equal_width":{"n":38,"bin_count":10,"ece":0.14293553170404935,"signed_mean_gap":0.14293553170404935,"absolute_mean_gap":0.14293553170404935,"bins":[{"index":0,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.0,"upper":0.1,"upper_inclusive":false},{"index":1,"count":34,"confidence":0.19452760938335867,"empirical_correctness":0.058823529411764705,"signed_gap":0.13570407997159395,"absolute_gap":0.13570407997159395,"lower":0.1,"upper":0.2,"upper_inclusive":false},{"index":2,"count":4,"confidence":0.2044028714299202,"empirical_correctness":0.0,"signed_gap":0.2044028714299202,"absolute_gap":0.2044028714299202,"lower":0.2,"upper":0.3,"upper_inclusive":false},{"index":3,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.3,"upper":0.4,"upper_inclusive":false},{"index":4,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.4,"upper":0.5,"upper_inclusive":false},{"index":5,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.5,"upper":0.6,"upper_inclusive":false},{"index":6,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.6,"upper":0.7,"upper_inclusive":false},{"index":7,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.7,"upper":0.8,"upper_inclusive":false},{"index":8,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.8,"upper":0.9,"upper_inclusive":false},{"index":9,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.9,"upper":1.0,"upper_inclusive":true}]},"equal_mass":{"n":38,"bin_count":10,"ece":0.1678008711651752,"signed_mean_gap":0.14293553170404935,"absolute_mean_gap":0.14293553170404935,"bins":[{"index":0,"count":3,"confidence":0.18713424106438956,"empirical_correctness":0.0,"signed_gap":0.18713424106438956,"absolute_gap":0.18713424106438956,"min_confidence":0.1862446665763855,"max_confidence":0.18836069107055664},{"index":1,"count":4,"confidence":0.18960876762866974,"empirical_correctness":0.25,"signed_gap":-0.06039123237133026,"absolute_gap":0.06039123237133026,"min_confidence":0.18849293887615204,"max_confidence":0.19123958051204681},{"index":2,"count":4,"confidence":0.1922808699309826,"empirical_correctness":0.25,"signed_gap":-0.05771913006901741,"absolute_gap":0.05771913006901741,"min_confidence":0.19171570241451263,"max_confidence":0.19272254407405853},{"index":3,"count":4,"confidence":0.19393876940011978,"empirical_correctness":0.0,"signed_gap":0.19393876940011978,"absolute_gap":0.19393876940011978,"min_confidence":0.1928299367427826,"max_confidence":0.1950913816690445},{"index":4,"count":4,"confidence":0.19558970630168915,"empirical_correctness":0.0,"signed_gap":0.19558970630168915,"absolute_gap":0.19558970630168915,"min_confidence":0.19509243965148926,"max_confidence":0.19595269858837128},{"index":5,"count":3,"confidence":0.19626104334990183,"empirical_correctness":0.0,"signed_gap":0.19626104334990183,"absolute_gap":0.19626104334990183,"min_confidence":0.19619998335838318,"max_confidence":0.19632278382778168},{"index":6,"count":4,"confidence":0.19708722084760666,"empirical_correctness":0.0,"signed_gap":0.19708722084760666,"absolute_gap":0.19708722084760666,"min_confidence":0.19641907513141632,"max_confidence":0.19778558611869812},{"index":7,"count":4,"confidence":0.19818786159157753,"empirical_correctness":0.0,"signed_gap":0.19818786159157753,"absolute_gap":0.19818786159157753,"min_confidence":0.1979101300239563,"max_confidence":0.1983683854341507},{"index":8,"count":4,"confidence":0.19924502074718475,"empirical_correctness":0.0,"signed_gap":0.19924502074718475,"absolute_gap":0.19924502074718475,"min_confidence":0.19857817888259888,"max_confidence":0.1997864842414856},{"index":9,"count":4,"confidence":0.2044028714299202,"empirical_correctness":0.0,"signed_gap":0.2044028714299202,"absolute_gap":0.2044028714299202,"min_confidence":0.2002348154783249,"max_confidence":0.20614846050739288}]}},"latency_ms":{"n":43,"median":87.29153795866296,"p95":88.443145994097},"per_action":{"keep":{"n":12,"raw":{"n":12,"acceptable":0,"unsafe":0,"review":6,"automated_count":6,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":6,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":0.5,"automated_coverage":0.5,"conditional_correctness":0.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.5,"accuracy_wilson_95":{"lower":2.7755575615628914e-17,"upper":0.24249400665524085,"confidence_level":0.95}},"effective":{"n":12,"acceptable":0,"unsafe":0,"review":12,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":12,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":2.7755575615628914e-17,"upper":0.24249400665524085,"confidence_level":0.95}}},"drop":{"n":16,"raw":{"n":16,"acceptable":1,"unsafe":0,"review":7,"automated_count":9,"automated_acceptable":1,"automated_unsafe":0,"unnecessary_reviews":7,"accuracy":0.0625,"unsafe_rate":0.0,"review_rate":0.4375,"automated_coverage":0.5625,"conditional_correctness":0.1111111111111111,"conditional_unsafe":0.0,"unnecessary_review_rate":0.4375,"accuracy_wilson_95":{"lower":0.01111934476464252,"upper":0.28328737570298945,"confidence_level":0.95}},"effective":{"n":16,"acceptable":0,"unsafe":0,"review":16,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":16,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.1936076805344365,"confidence_level":0.95}}},"mask":{"n":2,"raw":{"n":2,"acceptable":1,"unsafe":0,"review":1,"automated_count":1,"automated_acceptable":1,"automated_unsafe":0,"unnecessary_reviews":1,"accuracy":0.5,"unsafe_rate":0.0,"review_rate":0.5,"automated_coverage":0.5,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.5,"accuracy_wilson_95":{"lower":0.09453120573423074,"upper":0.9054687942657693,"confidence_level":0.95}},"effective":{"n":2,"acceptable":0,"unsafe":0,"review":2,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":2,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.6576197724933469,"confidence_level":0.95}}},"pseudonymize":{"n":2,"raw":{"n":2,"acceptable":0,"unsafe":0,"review":2,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":2,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.6576197724933469,"confidence_level":0.95}},"effective":{"n":2,"acceptable":0,"unsafe":0,"review":2,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":2,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.6576197724933469,"confidence_level":0.95}}},"generalize":{"n":6,"raw":{"n":6,"acceptable":0,"unsafe":0,"review":3,"automated_count":3,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":3,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":0.5,"automated_coverage":0.5,"conditional_correctness":0.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.5,"accuracy_wilson_95":{"lower":2.7755575615628914e-17,"upper":0.3903342879021653,"confidence_level":0.95}},"effective":{"n":6,"acceptable":0,"unsafe":0,"review":6,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":6,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":2.7755575615628914e-17,"upper":0.3903342879021653,"confidence_level":0.95}}},"review":{"n":0,"raw":{"n":0,"acceptable":0,"unsafe":0,"review":0,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":null,"unsafe_rate":null,"review_rate":null,"automated_coverage":null,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":null},"effective":{"n":0,"acceptable":0,"unsafe":0,"review":0,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":null,"unsafe_rate":null,"review_rate":null,"automated_coverage":null,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":null}}}},"diagnostic43-compact_no_samples":{"n":43,"raw":{"n":43,"acceptable":7,"unsafe":0,"review":10,"automated_count":33,"automated_acceptable":7,"automated_unsafe":0,"unnecessary_reviews":10,"accuracy":0.16279069767441862,"unsafe_rate":0.0,"review_rate":0.23255813953488372,"automated_coverage":0.7674418604651163,"conditional_correctness":0.21212121212121213,"conditional_unsafe":0.0,"unnecessary_review_rate":0.23255813953488372,"accuracy_wilson_95":{"lower":0.08116627347062008,"upper":0.29972405893747167,"confidence_level":0.95}},"effective":{"n":43,"acceptable":5,"unsafe":0,"review":43,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":38,"accuracy":0.11627906976744186,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":0.8837209302325582,"accuracy_wilson_95":{"lower":0.05070427739804166,"upper":0.24479161810082128,"confidence_level":0.95}},"confusion":{"raw":{"keep":{"keep":0,"drop":0,"mask":9,"pseudonymize":0,"generalize":0,"review":3},"drop":{"keep":0,"drop":0,"mask":13,"pseudonymize":0,"generalize":0,"review":3},"mask":{"keep":0,"drop":0,"mask":2,"pseudonymize":0,"generalize":0,"review":0},"pseudonymize":{"keep":0,"drop":0,"mask":1,"pseudonymize":0,"generalize":0,"review":1},"generalize":{"keep":0,"drop":0,"mask":3,"pseudonymize":0,"generalize":0,"review":3},"review":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0}},"effective":{"keep":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":12},"drop":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":16},"mask":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":2},"pseudonymize":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":2},"generalize":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":6},"review":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0}}},"probability_metrics":{"singleton_n":38,"multiacceptable_n":5,"multiacceptable_coverage":0.11627906976744186,"nll":1.8211129999378002,"nll_zero_mass_n":0,"brier":0.8434448718555516,"acceptable_set_n":43,"acceptable_set_nll":1.6815004039070192,"acceptable_set_zero_mass_n":0},"calibration_all_labels":{"equal_width":{"n":43,"bin_count":10,"ece":0.06430100493652875,"signed_mean_gap":0.03008550821348679,"absolute_mean_gap":0.03008550821348679,"bins":[{"index":0,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.0,"upper":0.1,"upper_inclusive":false},{"index":1,"count":32,"confidence":0.1884159385226667,"empirical_correctness":0.125,"signed_gap":0.06341593852266669,"absolute_gap":0.06341593852266669,"lower":0.1,"upper":0.2,"upper_inclusive":false},{"index":2,"count":11,"confidence":0.20585152913223614,"empirical_correctness":0.2727272727272727,"signed_gap":-0.06687574359503656,"absolute_gap":0.06687574359503656,"lower":0.2,"upper":0.3,"upper_inclusive":false},{"index":3,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.3,"upper":0.4,"upper_inclusive":false},{"index":4,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.4,"upper":0.5,"upper_inclusive":false},{"index":5,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.5,"upper":0.6,"upper_inclusive":false},{"index":6,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.6,"upper":0.7,"upper_inclusive":false},{"index":7,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.7,"upper":0.8,"upper_inclusive":false},{"index":8,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.8,"upper":0.9,"upper_inclusive":false},{"index":9,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.9,"upper":1.0,"upper_inclusive":true}]},"equal_mass":{"n":43,"bin_count":10,"ece":0.08537535404050073,"signed_mean_gap":0.03008550821348678,"absolute_mean_gap":0.03008550821348678,"bins":[{"index":0,"count":4,"confidence":0.17999690026044846,"empirical_correctness":0.25,"signed_gap":-0.07000309973955154,"absolute_gap":0.07000309973955154,"min_confidence":0.17768803238868713,"max_confidence":0.18235917389392853},{"index":1,"count":4,"confidence":0.18315381929278374,"empirical_correctness":0.0,"signed_gap":0.18315381929278374,"absolute_gap":0.18315381929278374,"min_confidence":0.1828078031539917,"max_confidence":0.18351510167121887},{"index":2,"count":4,"confidence":0.18380951508879662,"empirical_correctness":0.0,"signed_gap":0.18380951508879662,"absolute_gap":0.18380951508879662,"min_confidence":0.18353118002414703,"max_confidence":0.18411974608898163},{"index":3,"count":5,"confidence":0.1873558223247528,"empirical_correctness":0.2,"signed_gap":-0.012644177675247198,"absolute_gap":0.012644177675247198,"min_confidence":0.18475157022476196,"max_confidence":0.18869507312774658},{"index":4,"count":4,"confidence":0.1892767809331417,"empirical_correctness":0.25,"signed_gap":-0.06072321906685829,"absolute_gap":0.06072321906685829,"min_confidence":0.18902385234832764,"max_confidence":0.18987323343753815},{"index":5,"count":4,"confidence":0.19295776262879372,"empirical_correctness":0.25,"signed_gap":-0.057042237371206284,"absolute_gap":0.057042237371206284,"min_confidence":0.19163984060287476,"max_confidence":0.1936628669500351},{"index":6,"count":5,"confidence":0.1954128384590149,"empirical_correctness":0.0,"signed_gap":0.1954128384590149,"absolute_gap":0.1954128384590149,"min_confidence":0.19388270378112793,"max_confidence":0.19779764115810394},{"index":7,"count":4,"confidence":0.20069942250847816,"empirical_correctness":0.25,"signed_gap":-0.049300577491521835,"absolute_gap":0.049300577491521835,"min_confidence":0.19877274334430695,"max_confidence":0.20326830446720123},{"index":8,"count":4,"confidence":0.20569143444299698,"empirical_correctness":0.25,"signed_gap":-0.04430856555700302,"absolute_gap":0.04430856555700302,"min_confidence":0.20523639023303986,"max_confidence":0.20616373419761658},{"index":9,"count":5,"confidence":0.20749820172786712,"empirical_correctness":0.2,"signed_gap":0.00749820172786711,"absolute_gap":0.00749820172786711,"min_confidence":0.20651929080486298,"max_confidence":0.20830558240413666}]}},"calibration_singleton_labels":{"equal_width":{"n":38,"bin_count":10,"ece":0.1403159589359635,"signed_mean_gap":0.1403159589359635,"absolute_mean_gap":0.1403159589359635,"bins":[{"index":0,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.0,"upper":0.1,"upper_inclusive":false},{"index":1,"count":29,"confidence":0.18875292919833084,"empirical_correctness":0.034482758620689655,"signed_gap":0.1542701705776412,"absolute_gap":0.1542701705776412,"lower":0.1,"upper":0.2,"upper_inclusive":false},{"index":2,"count":9,"confidence":0.20646349920166862,"empirical_correctness":0.1111111111111111,"signed_gap":0.09535238809055752,"absolute_gap":0.09535238809055752,"lower":0.2,"upper":0.3,"upper_inclusive":false},{"index":3,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.3,"upper":0.4,"upper_inclusive":false},{"index":4,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.4,"upper":0.5,"upper_inclusive":false},{"index":5,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.5,"upper":0.6,"upper_inclusive":false},{"index":6,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.6,"upper":0.7,"upper_inclusive":false},{"index":7,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.7,"upper":0.8,"upper_inclusive":false},{"index":8,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.8,"upper":0.9,"upper_inclusive":false},{"index":9,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.9,"upper":1.0,"upper_inclusive":true}]},"equal_mass":{"n":38,"bin_count":10,"ece":0.1619960199061193,"signed_mean_gap":0.1403159589359635,"absolute_mean_gap":0.1403159589359635,"bins":[{"index":0,"count":3,"confidence":0.18030130863189697,"empirical_correctness":0.0,"signed_gap":0.18030130863189697,"absolute_gap":0.18030130863189697,"min_confidence":0.17768803238868713,"max_confidence":0.18235917389392853},{"index":1,"count":4,"confidence":0.18315381929278374,"empirical_correctness":0.0,"signed_gap":0.18315381929278374,"absolute_gap":0.18315381929278374,"min_confidence":0.1828078031539917,"max_confidence":0.18351510167121887},{"index":2,"count":4,"confidence":0.18380951508879662,"empirical_correctness":0.0,"signed_gap":0.18380951508879662,"absolute_gap":0.18380951508879662,"min_confidence":0.18353118002414703,"max_confidence":0.18411974608898163},{"index":3,"count":4,"confidence":0.18800688534975052,"empirical_correctness":0.0,"signed_gap":0.18800688534975052,"absolute_gap":0.18800688534975052,"min_confidence":0.18749774992465973,"max_confidence":0.18869507312774658},{"index":4,"count":4,"confidence":0.1892767809331417,"empirical_correctness":0.25,"signed_gap":-0.06072321906685829,"absolute_gap":0.06072321906685829,"min_confidence":0.18902385234832764,"max_confidence":0.18987323343753815},{"index":5,"count":3,"confidence":0.1933970699707667,"empirical_correctness":0.0,"signed_gap":0.1933970699707667,"absolute_gap":0.1933970699707667,"min_confidence":0.1931605190038681,"max_confidence":0.1936628669500351},{"index":6,"count":4,"confidence":0.19481663778424263,"empirical_correctness":0.0,"signed_gap":0.19481663778424263,"absolute_gap":0.19481663778424263,"min_confidence":0.19388270378112793,"max_confidence":0.1963575780391693},{"index":7,"count":4,"confidence":0.19993839040398598,"empirical_correctness":0.0,"signed_gap":0.19993839040398598,"absolute_gap":0.19993839040398598,"min_confidence":0.19779764115810394,"max_confidence":0.20326830446720123},{"index":8,"count":4,"confidence":0.20598286762833595,"empirical_correctness":0.0,"signed_gap":0.20598286762833595,"absolute_gap":0.20598286762833595,"min_confidence":0.20523639023303986,"max_confidence":0.20651929080486298},{"index":9,"count":4,"confidence":0.20774292945861816,"empirical_correctness":0.25,"signed_gap":-0.042257070541381836,"absolute_gap":0.042257070541381836,"min_confidence":0.20747368037700653,"max_confidence":0.20830558240413666}]}},"latency_ms":{"n":43,"median":102.00510802678764,"p95":103.34868712234311},"per_action":{"keep":{"n":12,"raw":{"n":12,"acceptable":0,"unsafe":0,"review":3,"automated_count":9,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":3,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":0.25,"automated_coverage":0.75,"conditional_correctness":0.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.25,"accuracy_wilson_95":{"lower":2.7755575615628914e-17,"upper":0.24249400665524085,"confidence_level":0.95}},"effective":{"n":12,"acceptable":0,"unsafe":0,"review":12,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":12,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":2.7755575615628914e-17,"upper":0.24249400665524085,"confidence_level":0.95}}},"drop":{"n":16,"raw":{"n":16,"acceptable":0,"unsafe":0,"review":3,"automated_count":13,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":3,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":0.1875,"automated_coverage":0.8125,"conditional_correctness":0.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.1875,"accuracy_wilson_95":{"lower":0.0,"upper":0.1936076805344365,"confidence_level":0.95}},"effective":{"n":16,"acceptable":0,"unsafe":0,"review":16,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":16,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.1936076805344365,"confidence_level":0.95}}},"mask":{"n":2,"raw":{"n":2,"acceptable":2,"unsafe":0,"review":0,"automated_count":2,"automated_acceptable":2,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":1.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.34238022750665303,"upper":1.0,"confidence_level":0.95}},"effective":{"n":2,"acceptable":0,"unsafe":0,"review":2,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":2,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.6576197724933469,"confidence_level":0.95}}},"pseudonymize":{"n":2,"raw":{"n":2,"acceptable":0,"unsafe":0,"review":1,"automated_count":1,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":1,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":0.5,"automated_coverage":0.5,"conditional_correctness":0.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.5,"accuracy_wilson_95":{"lower":0.0,"upper":0.6576197724933469,"confidence_level":0.95}},"effective":{"n":2,"acceptable":0,"unsafe":0,"review":2,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":2,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.6576197724933469,"confidence_level":0.95}}},"generalize":{"n":6,"raw":{"n":6,"acceptable":0,"unsafe":0,"review":3,"automated_count":3,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":3,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":0.5,"automated_coverage":0.5,"conditional_correctness":0.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.5,"accuracy_wilson_95":{"lower":2.7755575615628914e-17,"upper":0.3903342879021653,"confidence_level":0.95}},"effective":{"n":6,"acceptable":0,"unsafe":0,"review":6,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":6,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":2.7755575615628914e-17,"upper":0.3903342879021653,"confidence_level":0.95}}},"review":{"n":0,"raw":{"n":0,"acceptable":0,"unsafe":0,"review":0,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":null,"unsafe_rate":null,"review_rate":null,"automated_coverage":null,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":null},"effective":{"n":0,"acceptable":0,"unsafe":0,"review":0,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":null,"unsafe_rate":null,"review_rate":null,"automated_coverage":null,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":null}}}},"challenge56-no_samples":{"n":56,"raw":{"n":56,"acceptable":16,"unsafe":5,"review":9,"automated_count":47,"automated_acceptable":16,"automated_unsafe":5,"unnecessary_reviews":9,"accuracy":0.2857142857142857,"unsafe_rate":0.08928571428571429,"review_rate":0.16071428571428573,"automated_coverage":0.8392857142857143,"conditional_correctness":0.3404255319148936,"conditional_unsafe":0.10638297872340426,"unnecessary_review_rate":0.16071428571428573,"accuracy_wilson_95":{"lower":0.18418775143858906,"upper":0.4147525071551667,"confidence_level":0.95}},"effective":{"n":56,"acceptable":2,"unsafe":0,"review":56,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":54,"accuracy":0.03571428571428571,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":0.9642857142857143,"accuracy_wilson_95":{"lower":0.00984942514823639,"upper":0.12118780180490107,"confidence_level":0.95}},"confusion":{"raw":{"keep":{"keep":3,"drop":0,"mask":2,"pseudonymize":0,"generalize":0,"review":3},"drop":{"keep":1,"drop":4,"mask":16,"pseudonymize":0,"generalize":0,"review":1},"mask":{"keep":0,"drop":0,"mask":8,"pseudonymize":0,"generalize":0,"review":0},"pseudonymize":{"keep":1,"drop":0,"mask":4,"pseudonymize":0,"generalize":0,"review":3},"generalize":{"keep":3,"drop":0,"mask":2,"pseudonymize":0,"generalize":1,"review":2},"review":{"keep":0,"drop":0,"mask":2,"pseudonymize":0,"generalize":0,"review":0}},"effective":{"keep":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":8},"drop":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":22},"mask":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":8},"pseudonymize":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":8},"generalize":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":8},"review":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":2}}},"probability_metrics":{"singleton_n":56,"multiacceptable_n":0,"multiacceptable_coverage":0.0,"nll":1.7485591866551975,"nll_zero_mass_n":0,"brier":0.8194475237493167,"acceptable_set_n":56,"acceptable_set_nll":1.7485591866551975,"acceptable_set_zero_mass_n":0},"calibration_all_labels":{"equal_width":{"n":56,"bin_count":10,"ece":0.12633000793201582,"signed_mean_gap":-0.0862248380269323,"absolute_mean_gap":0.0862248380269323,"bins":[{"index":0,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.0,"upper":0.1,"upper_inclusive":false},{"index":1,"count":32,"confidence":0.19134202366694808,"empirical_correctness":0.15625,"signed_gap":0.03509202366694808,"absolute_gap":0.03509202366694808,"lower":0.1,"upper":0.2,"upper_inclusive":false},{"index":2,"count":24,"confidence":0.2103526797145605,"empirical_correctness":0.4583333333333333,"signed_gap":-0.2479806536187728,"absolute_gap":0.2479806536187728,"lower":0.2,"upper":0.3,"upper_inclusive":false},{"index":3,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.3,"upper":0.4,"upper_inclusive":false},{"index":4,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.4,"upper":0.5,"upper_inclusive":false},{"index":5,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.5,"upper":0.6,"upper_inclusive":false},{"index":6,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.6,"upper":0.7,"upper_inclusive":false},{"index":7,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.7,"upper":0.8,"upper_inclusive":false},{"index":8,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.8,"upper":0.9,"upper_inclusive":false},{"index":9,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.9,"upper":1.0,"upper_inclusive":true}]},"equal_mass":{"n":56,"bin_count":10,"ece":0.1749216616153717,"signed_mean_gap":-0.0862248380269323,"absolute_mean_gap":0.0862248380269323,"bins":[{"index":0,"count":5,"confidence":0.18146961331367492,"empirical_correctness":0.2,"signed_gap":-0.01853038668632509,"absolute_gap":0.01853038668632509,"min_confidence":0.17483451962471008,"max_confidence":0.18477201461791992},{"index":1,"count":6,"confidence":0.18786143014828363,"empirical_correctness":0.0,"signed_gap":0.18786143014828363,"absolute_gap":0.18786143014828363,"min_confidence":0.18666398525238037,"max_confidence":0.18894006311893463},{"index":2,"count":5,"confidence":0.19068036377429962,"empirical_correctness":0.0,"signed_gap":0.19068036377429962,"absolute_gap":0.19068036377429962,"min_confidence":0.1892523169517517,"max_confidence":0.19222666323184967},{"index":3,"count":6,"confidence":0.19350172579288483,"empirical_correctness":0.3333333333333333,"signed_gap":-0.1398316075404485,"absolute_gap":0.1398316075404485,"min_confidence":0.1925859898328781,"max_confidence":0.19422823190689087},{"index":4,"count":6,"confidence":0.19661028683185577,"empirical_correctness":0.16666666666666666,"signed_gap":0.029943620165189117,"absolute_gap":0.029943620165189117,"min_confidence":0.19480285048484802,"max_confidence":0.19801102578639984},{"index":5,"count":5,"confidence":0.1989918053150177,"empirical_correctness":0.2,"signed_gap":-0.001008194684982311,"absolute_gap":0.001008194684982311,"min_confidence":0.19808952510356903,"max_confidence":0.20060481131076813},{"index":6,"count":6,"confidence":0.2038798232873281,"empirical_correctness":0.16666666666666666,"signed_gap":0.037213156620661436,"absolute_gap":0.037213156620661436,"min_confidence":0.20183740556240082,"max_confidence":0.2050705999135971},{"index":7,"count":5,"confidence":0.20841293036937714,"empirical_correctness":0.6,"signed_gap":-0.39158706963062284,"absolute_gap":0.39158706963062284,"min_confidence":0.20756103098392487,"max_confidence":0.20886902511119843},{"index":8,"count":6,"confidence":0.21090265611807504,"empirical_correctness":0.3333333333333333,"signed_gap":-0.12243067721525827,"absolute_gap":0.12243067721525827,"min_confidence":0.209284245967865,"max_confidence":0.21244150400161743},{"index":9,"count":6,"confidence":0.2195166622598966,"empirical_correctness":0.8333333333333334,"signed_gap":-0.6138166710734367,"absolute_gap":0.6138166710734367,"min_confidence":0.21347808837890625,"max_confidence":0.22573204338550568}]}},"calibration_singleton_labels":{"equal_width":{"n":56,"bin_count":10,"ece":0.12633000793201582,"signed_mean_gap":-0.0862248380269323,"absolute_mean_gap":0.0862248380269323,"bins":[{"index":0,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.0,"upper":0.1,"upper_inclusive":false},{"index":1,"count":32,"confidence":0.19134202366694808,"empirical_correctness":0.15625,"signed_gap":0.03509202366694808,"absolute_gap":0.03509202366694808,"lower":0.1,"upper":0.2,"upper_inclusive":false},{"index":2,"count":24,"confidence":0.2103526797145605,"empirical_correctness":0.4583333333333333,"signed_gap":-0.2479806536187728,"absolute_gap":0.2479806536187728,"lower":0.2,"upper":0.3,"upper_inclusive":false},{"index":3,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.3,"upper":0.4,"upper_inclusive":false},{"index":4,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.4,"upper":0.5,"upper_inclusive":false},{"index":5,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.5,"upper":0.6,"upper_inclusive":false},{"index":6,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.6,"upper":0.7,"upper_inclusive":false},{"index":7,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.7,"upper":0.8,"upper_inclusive":false},{"index":8,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.8,"upper":0.9,"upper_inclusive":false},{"index":9,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.9,"upper":1.0,"upper_inclusive":true}]},"equal_mass":{"n":56,"bin_count":10,"ece":0.1749216616153717,"signed_mean_gap":-0.0862248380269323,"absolute_mean_gap":0.0862248380269323,"bins":[{"index":0,"count":5,"confidence":0.18146961331367492,"empirical_correctness":0.2,"signed_gap":-0.01853038668632509,"absolute_gap":0.01853038668632509,"min_confidence":0.17483451962471008,"max_confidence":0.18477201461791992},{"index":1,"count":6,"confidence":0.18786143014828363,"empirical_correctness":0.0,"signed_gap":0.18786143014828363,"absolute_gap":0.18786143014828363,"min_confidence":0.18666398525238037,"max_confidence":0.18894006311893463},{"index":2,"count":5,"confidence":0.19068036377429962,"empirical_correctness":0.0,"signed_gap":0.19068036377429962,"absolute_gap":0.19068036377429962,"min_confidence":0.1892523169517517,"max_confidence":0.19222666323184967},{"index":3,"count":6,"confidence":0.19350172579288483,"empirical_correctness":0.3333333333333333,"signed_gap":-0.1398316075404485,"absolute_gap":0.1398316075404485,"min_confidence":0.1925859898328781,"max_confidence":0.19422823190689087},{"index":4,"count":6,"confidence":0.19661028683185577,"empirical_correctness":0.16666666666666666,"signed_gap":0.029943620165189117,"absolute_gap":0.029943620165189117,"min_confidence":0.19480285048484802,"max_confidence":0.19801102578639984},{"index":5,"count":5,"confidence":0.1989918053150177,"empirical_correctness":0.2,"signed_gap":-0.001008194684982311,"absolute_gap":0.001008194684982311,"min_confidence":0.19808952510356903,"max_confidence":0.20060481131076813},{"index":6,"count":6,"confidence":0.2038798232873281,"empirical_correctness":0.16666666666666666,"signed_gap":0.037213156620661436,"absolute_gap":0.037213156620661436,"min_confidence":0.20183740556240082,"max_confidence":0.2050705999135971},{"index":7,"count":5,"confidence":0.20841293036937714,"empirical_correctness":0.6,"signed_gap":-0.39158706963062284,"absolute_gap":0.39158706963062284,"min_confidence":0.20756103098392487,"max_confidence":0.20886902511119843},{"index":8,"count":6,"confidence":0.21090265611807504,"empirical_correctness":0.3333333333333333,"signed_gap":-0.12243067721525827,"absolute_gap":0.12243067721525827,"min_confidence":0.209284245967865,"max_confidence":0.21244150400161743},{"index":9,"count":6,"confidence":0.2195166622598966,"empirical_correctness":0.8333333333333334,"signed_gap":-0.6138166710734367,"absolute_gap":0.6138166710734367,"min_confidence":0.21347808837890625,"max_confidence":0.22573204338550568}]}},"latency_ms":{"n":56,"median":75.01832151319832,"p95":86.133220262127},"per_action":{"keep":{"n":8,"raw":{"n":8,"acceptable":3,"unsafe":0,"review":3,"automated_count":5,"automated_acceptable":3,"automated_unsafe":0,"unnecessary_reviews":3,"accuracy":0.375,"unsafe_rate":0.0,"review_rate":0.375,"automated_coverage":0.625,"conditional_correctness":0.6,"conditional_unsafe":0.0,"unnecessary_review_rate":0.375,"accuracy_wilson_95":{"lower":0.13684428582359742,"upper":0.6942576053973727,"confidence_level":0.95}},"effective":{"n":8,"acceptable":0,"unsafe":0,"review":8,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":8,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.32440756488388023,"confidence_level":0.95}}},"drop":{"n":22,"raw":{"n":22,"acceptable":4,"unsafe":1,"review":1,"automated_count":21,"automated_acceptable":4,"automated_unsafe":1,"unnecessary_reviews":1,"accuracy":0.18181818181818182,"unsafe_rate":0.045454545454545456,"review_rate":0.045454545454545456,"automated_coverage":0.9545454545454546,"conditional_correctness":0.19047619047619047,"conditional_unsafe":0.047619047619047616,"unnecessary_review_rate":0.045454545454545456,"accuracy_wilson_95":{"lower":0.07306884583422052,"upper":0.38516607437126543,"confidence_level":0.95}},"effective":{"n":22,"acceptable":0,"unsafe":0,"review":22,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":22,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.14865487460862087,"confidence_level":0.95}}},"mask":{"n":8,"raw":{"n":8,"acceptable":8,"unsafe":0,"review":0,"automated_count":8,"automated_acceptable":8,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":1.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.6755924351161198,"upper":1.0,"confidence_level":0.95}},"effective":{"n":8,"acceptable":0,"unsafe":0,"review":8,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":8,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.32440756488388023,"confidence_level":0.95}}},"pseudonymize":{"n":8,"raw":{"n":8,"acceptable":0,"unsafe":1,"review":3,"automated_count":5,"automated_acceptable":0,"automated_unsafe":1,"unnecessary_reviews":3,"accuracy":0.0,"unsafe_rate":0.125,"review_rate":0.375,"automated_coverage":0.625,"conditional_correctness":0.0,"conditional_unsafe":0.2,"unnecessary_review_rate":0.375,"accuracy_wilson_95":{"lower":0.0,"upper":0.32440756488388023,"confidence_level":0.95}},"effective":{"n":8,"acceptable":0,"unsafe":0,"review":8,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":8,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.32440756488388023,"confidence_level":0.95}}},"generalize":{"n":8,"raw":{"n":8,"acceptable":1,"unsafe":3,"review":2,"automated_count":6,"automated_acceptable":1,"automated_unsafe":3,"unnecessary_reviews":2,"accuracy":0.125,"unsafe_rate":0.375,"review_rate":0.25,"automated_coverage":0.75,"conditional_correctness":0.16666666666666666,"conditional_unsafe":0.5,"unnecessary_review_rate":0.25,"accuracy_wilson_95":{"lower":0.02241749145005667,"upper":0.4708881822128535,"confidence_level":0.95}},"effective":{"n":8,"acceptable":0,"unsafe":0,"review":8,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":8,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.32440756488388023,"confidence_level":0.95}}},"review":{"n":2,"raw":{"n":2,"acceptable":0,"unsafe":0,"review":0,"automated_count":2,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.6576197724933469,"confidence_level":0.95}},"effective":{"n":2,"acceptable":2,"unsafe":0,"review":2,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":1.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.34238022750665303,"upper":1.0,"confidence_level":0.95}}}}},"challenge56-compact_no_samples":{"n":56,"raw":{"n":56,"acceptable":8,"unsafe":5,"review":7,"automated_count":49,"automated_acceptable":8,"automated_unsafe":5,"unnecessary_reviews":7,"accuracy":0.14285714285714285,"unsafe_rate":0.08928571428571429,"review_rate":0.125,"automated_coverage":0.875,"conditional_correctness":0.16326530612244897,"conditional_unsafe":0.10204081632653061,"unnecessary_review_rate":0.125,"accuracy_wilson_95":{"lower":0.07420788536920624,"upper":0.25735921228705333,"confidence_level":0.95}},"effective":{"n":56,"acceptable":2,"unsafe":0,"review":56,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":54,"accuracy":0.03571428571428571,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":0.9642857142857143,"accuracy_wilson_95":{"lower":0.00984942514823639,"upper":0.12118780180490107,"confidence_level":0.95}},"confusion":{"raw":{"keep":{"keep":1,"drop":0,"mask":7,"pseudonymize":0,"generalize":0,"review":0},"drop":{"keep":1,"drop":0,"mask":21,"pseudonymize":0,"generalize":0,"review":0},"mask":{"keep":0,"drop":0,"mask":7,"pseudonymize":0,"generalize":0,"review":1},"pseudonymize":{"keep":4,"drop":0,"mask":4,"pseudonymize":0,"generalize":0,"review":0},"generalize":{"keep":0,"drop":0,"mask":2,"pseudonymize":0,"generalize":0,"review":6},"review":{"keep":0,"drop":0,"mask":2,"pseudonymize":0,"generalize":0,"review":0}},"effective":{"keep":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":8},"drop":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":22},"mask":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":8},"pseudonymize":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":8},"generalize":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":8},"review":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":2}}},"probability_metrics":{"singleton_n":56,"multiacceptable_n":0,"multiacceptable_coverage":0.0,"nll":1.7958217379184682,"nll_zero_mass_n":0,"brier":0.8357463768069141,"acceptable_set_n":56,"acceptable_set_nll":1.7958217379184682,"acceptable_set_zero_mass_n":0},"calibration_all_labels":{"equal_width":{"n":56,"bin_count":10,"ece":0.07275885104068688,"signed_mean_gap":0.05687689115958555,"absolute_mean_gap":0.05687689115958555,"bins":[{"index":0,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.0,"upper":0.1,"upper_inclusive":false},{"index":1,"count":29,"confidence":0.1941310614347458,"empirical_correctness":0.06896551724137931,"signed_gap":0.12516554419336648,"absolute_gap":0.12516554419336648,"lower":0.1,"upper":0.2,"upper_inclusive":false},{"index":2,"count":27,"confidence":0.2057520416047838,"empirical_correctness":0.2222222222222222,"signed_gap":-0.016470180617438407,"absolute_gap":0.016470180617438407,"lower":0.2,"upper":0.3,"upper_inclusive":false},{"index":3,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.3,"upper":0.4,"upper_inclusive":false},{"index":4,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.4,"upper":0.5,"upper_inclusive":false},{"index":5,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.5,"upper":0.6,"upper_inclusive":false},{"index":6,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.6,"upper":0.7,"upper_inclusive":false},{"index":7,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.7,"upper":0.8,"upper_inclusive":false},{"index":8,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.8,"upper":0.9,"upper_inclusive":false},{"index":9,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.9,"upper":1.0,"upper_inclusive":true}]},"equal_mass":{"n":56,"bin_count":10,"ece":0.1579114824001278,"signed_mean_gap":0.056876891159585545,"absolute_mean_gap":0.056876891159585545,"bins":[{"index":0,"count":5,"confidence":0.18694088757038116,"empirical_correctness":0.2,"signed_gap":-0.013059112429618852,"absolute_gap":0.013059112429618852,"min_confidence":0.1821914166212082,"max_confidence":0.19091489911079407},{"index":1,"count":6,"confidence":0.19266130030155182,"empirical_correctness":0.0,"signed_gap":0.19266130030155182,"absolute_gap":0.19266130030155182,"min_confidence":0.19108548760414124,"max_confidence":0.1940455138683319},{"index":2,"count":5,"confidence":0.19435388147830962,"empirical_correctness":0.2,"signed_gap":-0.005646118521690391,"absolute_gap":0.005646118521690391,"min_confidence":0.19405721127986908,"max_confidence":0.19493474066257477},{"index":3,"count":6,"confidence":0.1957210749387741,"empirical_correctness":0.0,"signed_gap":0.1957210749387741,"absolute_gap":0.1957210749387741,"min_confidence":0.1949441134929657,"max_confidence":0.19648762047290802},{"index":4,"count":6,"confidence":0.19886182993650436,"empirical_correctness":0.0,"signed_gap":0.19886182993650436,"absolute_gap":0.19886182993650436,"min_confidence":0.19748103618621826,"max_confidence":0.199832484126091},{"index":5,"count":5,"confidence":0.20044728815555574,"empirical_correctness":0.0,"signed_gap":0.20044728815555574,"absolute_gap":0.20044728815555574,"min_confidence":0.19986170530319214,"max_confidence":0.20069310069084167},{"index":6,"count":6,"confidence":0.20199626435836157,"empirical_correctness":0.0,"signed_gap":0.20199626435836157,"absolute_gap":0.20199626435836157,"min_confidence":0.20106181502342224,"max_confidence":0.2026475965976715},{"index":7,"count":5,"confidence":0.20435349345207215,"empirical_correctness":0.4,"signed_gap":-0.19564650654792787,"absolute_gap":0.19564650654792787,"min_confidence":0.20291967689990997,"max_confidence":0.20627711713314056},{"index":8,"count":6,"confidence":0.20713168879350027,"empirical_correctness":0.5,"signed_gap":-0.2928683112064997,"absolute_gap":0.2928683112064997,"min_confidence":0.20646126568317413,"max_confidence":0.20776712894439697},{"index":9,"count":6,"confidence":0.21273253361384073,"empirical_correctness":0.16666666666666666,"signed_gap":0.04606586694717407,"absolute_gap":0.04606586694717407,"min_confidence":0.210121750831604,"max_confidence":0.2162466198205948}]}},"calibration_singleton_labels":{"equal_width":{"n":56,"bin_count":10,"ece":0.07275885104068688,"signed_mean_gap":0.05687689115958555,"absolute_mean_gap":0.05687689115958555,"bins":[{"index":0,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.0,"upper":0.1,"upper_inclusive":false},{"index":1,"count":29,"confidence":0.1941310614347458,"empirical_correctness":0.06896551724137931,"signed_gap":0.12516554419336648,"absolute_gap":0.12516554419336648,"lower":0.1,"upper":0.2,"upper_inclusive":false},{"index":2,"count":27,"confidence":0.2057520416047838,"empirical_correctness":0.2222222222222222,"signed_gap":-0.016470180617438407,"absolute_gap":0.016470180617438407,"lower":0.2,"upper":0.3,"upper_inclusive":false},{"index":3,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.3,"upper":0.4,"upper_inclusive":false},{"index":4,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.4,"upper":0.5,"upper_inclusive":false},{"index":5,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.5,"upper":0.6,"upper_inclusive":false},{"index":6,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.6,"upper":0.7,"upper_inclusive":false},{"index":7,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.7,"upper":0.8,"upper_inclusive":false},{"index":8,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.8,"upper":0.9,"upper_inclusive":false},{"index":9,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.9,"upper":1.0,"upper_inclusive":true}]},"equal_mass":{"n":56,"bin_count":10,"ece":0.1579114824001278,"signed_mean_gap":0.056876891159585545,"absolute_mean_gap":0.056876891159585545,"bins":[{"index":0,"count":5,"confidence":0.18694088757038116,"empirical_correctness":0.2,"signed_gap":-0.013059112429618852,"absolute_gap":0.013059112429618852,"min_confidence":0.1821914166212082,"max_confidence":0.19091489911079407},{"index":1,"count":6,"confidence":0.19266130030155182,"empirical_correctness":0.0,"signed_gap":0.19266130030155182,"absolute_gap":0.19266130030155182,"min_confidence":0.19108548760414124,"max_confidence":0.1940455138683319},{"index":2,"count":5,"confidence":0.19435388147830962,"empirical_correctness":0.2,"signed_gap":-0.005646118521690391,"absolute_gap":0.005646118521690391,"min_confidence":0.19405721127986908,"max_confidence":0.19493474066257477},{"index":3,"count":6,"confidence":0.1957210749387741,"empirical_correctness":0.0,"signed_gap":0.1957210749387741,"absolute_gap":0.1957210749387741,"min_confidence":0.1949441134929657,"max_confidence":0.19648762047290802},{"index":4,"count":6,"confidence":0.19886182993650436,"empirical_correctness":0.0,"signed_gap":0.19886182993650436,"absolute_gap":0.19886182993650436,"min_confidence":0.19748103618621826,"max_confidence":0.199832484126091},{"index":5,"count":5,"confidence":0.20044728815555574,"empirical_correctness":0.0,"signed_gap":0.20044728815555574,"absolute_gap":0.20044728815555574,"min_confidence":0.19986170530319214,"max_confidence":0.20069310069084167},{"index":6,"count":6,"confidence":0.20199626435836157,"empirical_correctness":0.0,"signed_gap":0.20199626435836157,"absolute_gap":0.20199626435836157,"min_confidence":0.20106181502342224,"max_confidence":0.2026475965976715},{"index":7,"count":5,"confidence":0.20435349345207215,"empirical_correctness":0.4,"signed_gap":-0.19564650654792787,"absolute_gap":0.19564650654792787,"min_confidence":0.20291967689990997,"max_confidence":0.20627711713314056},{"index":8,"count":6,"confidence":0.20713168879350027,"empirical_correctness":0.5,"signed_gap":-0.2928683112064997,"absolute_gap":0.2928683112064997,"min_confidence":0.20646126568317413,"max_confidence":0.20776712894439697},{"index":9,"count":6,"confidence":0.21273253361384073,"empirical_correctness":0.16666666666666666,"signed_gap":0.04606586694717407,"absolute_gap":0.04606586694717407,"min_confidence":0.210121750831604,"max_confidence":0.2162466198205948}]}},"latency_ms":{"n":56,"median":87.60082349181175,"p95":88.62664924527053},"per_action":{"keep":{"n":8,"raw":{"n":8,"acceptable":1,"unsafe":0,"review":0,"automated_count":8,"automated_acceptable":1,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":0.125,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.125,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.02241749145005667,"upper":0.4708881822128535,"confidence_level":0.95}},"effective":{"n":8,"acceptable":0,"unsafe":0,"review":8,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":8,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.32440756488388023,"confidence_level":0.95}}},"drop":{"n":22,"raw":{"n":22,"acceptable":0,"unsafe":1,"review":0,"automated_count":22,"automated_acceptable":0,"automated_unsafe":1,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":0.045454545454545456,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":0.045454545454545456,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.14865487460862087,"confidence_level":0.95}},"effective":{"n":22,"acceptable":0,"unsafe":0,"review":22,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":22,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.14865487460862087,"confidence_level":0.95}}},"mask":{"n":8,"raw":{"n":8,"acceptable":7,"unsafe":0,"review":1,"automated_count":7,"automated_acceptable":7,"automated_unsafe":0,"unnecessary_reviews":1,"accuracy":0.875,"unsafe_rate":0.0,"review_rate":0.125,"automated_coverage":0.875,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.125,"accuracy_wilson_95":{"lower":0.5291118177871464,"upper":0.9775825085499433,"confidence_level":0.95}},"effective":{"n":8,"acceptable":0,"unsafe":0,"review":8,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":8,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.32440756488388023,"confidence_level":0.95}}},"pseudonymize":{"n":8,"raw":{"n":8,"acceptable":0,"unsafe":4,"review":0,"automated_count":8,"automated_acceptable":0,"automated_unsafe":4,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":0.5,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":0.5,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.32440756488388023,"confidence_level":0.95}},"effective":{"n":8,"acceptable":0,"unsafe":0,"review":8,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":8,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.32440756488388023,"confidence_level":0.95}}},"generalize":{"n":8,"raw":{"n":8,"acceptable":0,"unsafe":0,"review":6,"automated_count":2,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":6,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":0.75,"automated_coverage":0.25,"conditional_correctness":0.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.75,"accuracy_wilson_95":{"lower":0.0,"upper":0.32440756488388023,"confidence_level":0.95}},"effective":{"n":8,"acceptable":0,"unsafe":0,"review":8,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":8,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.32440756488388023,"confidence_level":0.95}}},"review":{"n":2,"raw":{"n":2,"acceptable":0,"unsafe":0,"review":0,"automated_count":2,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.6576197724933469,"confidence_level":0.95}},"effective":{"n":2,"acceptable":2,"unsafe":0,"review":2,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":1.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.34238022750665303,"upper":1.0,"confidence_level":0.95}}}}}},"composite":{"n":198,"scope":"descriptive sum across four correlated panels","raw":{"n":198,"acceptable":38,"unsafe":10,"review":47,"automated_count":151,"automated_acceptable":36,"automated_unsafe":10,"unnecessary_reviews":45,"accuracy":0.1919191919191919,"unsafe_rate":0.050505050505050504,"review_rate":0.23737373737373738,"automated_coverage":0.7626262626262627,"conditional_correctness":0.23841059602649006,"conditional_unsafe":0.06622516556291391,"unnecessary_review_rate":0.22727272727272727},"effective":{"n":198,"acceptable":14,"unsafe":0,"review":198,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":184,"accuracy":0.0707070707070707,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":0.9292929292929293}}},{"id":"nli-cross-encoder-base-fp32","platform":"Local","model_id":"cross-encoder/nli-deberta-v3-base","revision":"6c749ce3425cd33b46d187e45b92bbf96ee12ec7","loading":"float32 / none","scoring_kind":"Adapted NLI","settings":{"dtype":"float32","quantization":null,"max_tokens":512,"effective_max_tokens":512,"temperature":1.0,"scoring_batch_size":1,"trust_remote_code":false,"local_files_only":true,"use_safetensors":true,"guard_threshold":0.45,"seed":0,"tf32":false},"scoring_metadata":{"backend":"nli","protocol":"hf-local-sweep-backends-v1","probability_source":"adapted_entailment_softmax_not_calibrated_sixway","latency_scope":"synchronous_forward_scoring_including_transfer_excluding_tokenization","precision":"float32","equal_mass_convention":"rank bins with floor boundaries at floor(i*n/k), k=min(10,n//2); minimum two rows"},"load_ms":9467.603800119832,"peak_allocated_bytes":846681088,"peak_reserved_bytes":870318080,"panels":{"diagnostic43-no_samples":{"n":43,"raw":{"n":43,"acceptable":9,"unsafe":25,"review":0,"automated_count":43,"automated_acceptable":9,"automated_unsafe":25,"unnecessary_reviews":0,"accuracy":0.20930232558139536,"unsafe_rate":0.5813953488372093,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.20930232558139536,"conditional_unsafe":0.5813953488372093,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.11422834220995602,"upper":0.3520564271073644,"confidence_level":0.95}},"effective":{"n":43,"acceptable":6,"unsafe":0,"review":42,"automated_count":1,"automated_acceptable":1,"automated_unsafe":0,"unnecessary_reviews":37,"accuracy":0.13953488372093023,"unsafe_rate":0.0,"review_rate":0.9767441860465116,"automated_coverage":0.023255813953488372,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.8604651162790697,"accuracy_wilson_95":{"lower":0.06555723427964878,"upper":0.2726358796738286,"confidence_level":0.95}},"confusion":{"raw":{"keep":{"keep":6,"drop":0,"mask":0,"pseudonymize":5,"generalize":1,"review":0},"drop":{"keep":8,"drop":0,"mask":0,"pseudonymize":5,"generalize":3,"review":0},"mask":{"keep":1,"drop":0,"mask":0,"pseudonymize":1,"generalize":0,"review":0},"pseudonymize":{"keep":1,"drop":0,"mask":0,"pseudonymize":1,"generalize":0,"review":0},"generalize":{"keep":3,"drop":0,"mask":0,"pseudonymize":1,"generalize":2,"review":0},"review":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0}},"effective":{"keep":{"keep":1,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":11},"drop":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":16},"mask":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":2},"pseudonymize":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":2},"generalize":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":6},"review":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0}}},"probability_metrics":{"singleton_n":38,"multiacceptable_n":5,"multiacceptable_coverage":0.11627906976744186,"nll":2.396029155510367,"nll_zero_mass_n":0,"brier":1.016450629126867,"acceptable_set_n":43,"acceptable_set_nll":2.263343413440345,"acceptable_set_zero_mass_n":0},"calibration_all_labels":{"equal_width":{"n":43,"bin_count":10,"ece":0.26996593419895615,"signed_mean_gap":0.26996593419895615,"absolute_mean_gap":0.26996593419895615,"bins":[{"index":0,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.0,"upper":0.1,"upper_inclusive":false},{"index":1,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.1,"upper":0.2,"upper_inclusive":false},{"index":2,"count":4,"confidence":0.28269926458597183,"empirical_correctness":0.25,"signed_gap":0.03269926458597183,"absolute_gap":0.03269926458597183,"lower":0.2,"upper":0.3,"upper_inclusive":false},{"index":3,"count":14,"confidence":0.37630862423351835,"empirical_correctness":0.2857142857142857,"signed_gap":0.09059433851923265,"absolute_gap":0.09059433851923265,"lower":0.3,"upper":0.4,"upper_inclusive":false},{"index":4,"count":11,"confidence":0.4186133769425479,"empirical_correctness":0.2727272727272727,"signed_gap":0.1458861042152752,"absolute_gap":0.1458861042152752,"lower":0.4,"upper":0.5,"upper_inclusive":false},{"index":5,"count":3,"confidence":0.567760705947876,"empirical_correctness":0.0,"signed_gap":0.567760705947876,"absolute_gap":0.567760705947876,"lower":0.5,"upper":0.6,"upper_inclusive":false},{"index":6,"count":4,"confidence":0.6256669461727142,"empirical_correctness":0.25,"signed_gap":0.37566694617271423,"absolute_gap":0.37566694617271423,"lower":0.6,"upper":0.7,"upper_inclusive":false},{"index":7,"count":7,"confidence":0.7712457605770656,"empirical_correctness":0.0,"signed_gap":0.7712457605770656,"absolute_gap":0.7712457605770656,"lower":0.7,"upper":0.8,"upper_inclusive":false},{"index":8,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.8,"upper":0.9,"upper_inclusive":false},{"index":9,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.9,"upper":1.0,"upper_inclusive":true}]},"equal_mass":{"n":43,"bin_count":10,"ece":0.3087884159975274,"signed_mean_gap":0.26996593419895615,"absolute_mean_gap":0.26996593419895615,"bins":[{"index":0,"count":4,"confidence":0.28269926458597183,"empirical_correctness":0.25,"signed_gap":0.03269926458597183,"absolute_gap":0.03269926458597183,"min_confidence":0.2627226710319519,"max_confidence":0.29749882221221924},{"index":1,"count":4,"confidence":0.35396500676870346,"empirical_correctness":0.0,"signed_gap":0.35396500676870346,"absolute_gap":0.35396500676870346,"min_confidence":0.3383110463619232,"max_confidence":0.3644558787345886},{"index":2,"count":4,"confidence":0.37275516986846924,"empirical_correctness":0.5,"signed_gap":-0.12724483013153076,"absolute_gap":0.12724483013153076,"min_confidence":0.3651926517486572,"max_confidence":0.3799024522304535},{"index":3,"count":5,"confidence":0.39259599447250365,"empirical_correctness":0.2,"signed_gap":0.19259599447250364,"absolute_gap":0.19259599447250364,"min_confidence":0.38891348242759705,"max_confidence":0.3973107635974884},{"index":4,"count":4,"confidence":0.40334440022706985,"empirical_correctness":0.25,"signed_gap":0.15334440022706985,"absolute_gap":0.15334440022706985,"min_confidence":0.3984600603580475,"max_confidence":0.4068319499492645},{"index":5,"count":4,"confidence":0.4185739904642105,"empirical_correctness":0.5,"signed_gap":-0.08142600953578949,"absolute_gap":0.08142600953578949,"min_confidence":0.4088648557662964,"max_confidence":0.42635732889175415},{"index":6,"count":5,"confidence":0.4519091784954071,"empirical_correctness":0.2,"signed_gap":0.2519091784954071,"absolute_gap":0.2519091784954071,"min_confidence":0.4278479516506195,"max_confidence":0.5440122485160828},{"index":7,"count":4,"confidence":0.5930467545986176,"empirical_correctness":0.25,"signed_gap":0.34304675459861755,"absolute_gap":0.34304675459861755,"min_confidence":0.5697901844978333,"max_confidence":0.6070662140846252},{"index":8,"count":4,"confidence":0.7017382830381393,"empirical_correctness":0.0,"signed_gap":0.7017382830381393,"absolute_gap":0.7017382830381393,"min_confidence":0.641459047794342,"max_confidence":0.7593733668327332},{"index":9,"count":5,"confidence":0.7763035655021667,"empirical_correctness":0.0,"signed_gap":0.7763035655021667,"absolute_gap":0.7763035655021667,"min_confidence":0.7657516598701477,"max_confidence":0.7888846397399902}]}},"calibration_singleton_labels":{"equal_width":{"n":38,"bin_count":10,"ece":0.23655394190236143,"signed_mean_gap":0.23655394190236143,"absolute_mean_gap":0.23655394190236143,"bins":[{"index":0,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.0,"upper":0.1,"upper_inclusive":false},{"index":1,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.1,"upper":0.2,"upper_inclusive":false},{"index":2,"count":4,"confidence":0.28269926458597183,"empirical_correctness":0.25,"signed_gap":0.03269926458597183,"absolute_gap":0.03269926458597183,"lower":0.2,"upper":0.3,"upper_inclusive":false},{"index":3,"count":13,"confidence":0.3780415906355931,"empirical_correctness":0.3076923076923077,"signed_gap":0.07034928294328541,"absolute_gap":0.07034928294328541,"lower":0.3,"upper":0.4,"upper_inclusive":false},{"index":4,"count":9,"confidence":0.41880926820966935,"empirical_correctness":0.3333333333333333,"signed_gap":0.08547593487633603,"absolute_gap":0.08547593487633603,"lower":0.4,"upper":0.5,"upper_inclusive":false},{"index":5,"count":3,"confidence":0.567760705947876,"empirical_correctness":0.0,"signed_gap":0.567760705947876,"absolute_gap":0.567760705947876,"lower":0.5,"upper":0.6,"upper_inclusive":false},{"index":6,"count":3,"confidence":0.6181253989537557,"empirical_correctness":0.3333333333333333,"signed_gap":0.2847920656204224,"absolute_gap":0.2847920656204224,"lower":0.6,"upper":0.7,"upper_inclusive":false},{"index":7,"count":6,"confidence":0.7694617211818695,"empirical_correctness":0.0,"signed_gap":0.7694617211818695,"absolute_gap":0.7694617211818695,"lower":0.7,"upper":0.8,"upper_inclusive":false},{"index":8,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.8,"upper":0.9,"upper_inclusive":false},{"index":9,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.9,"upper":1.0,"upper_inclusive":true}]},"equal_mass":{"n":38,"bin_count":10,"ece":0.287207725800966,"signed_mean_gap":0.23655394190236143,"absolute_mean_gap":0.23655394190236143,"bins":[{"index":0,"count":3,"confidence":0.27776607871055603,"empirical_correctness":0.3333333333333333,"signed_gap":-0.055567254622777285,"absolute_gap":0.055567254622777285,"min_confidence":0.2627226710319519,"max_confidence":0.29667001962661743},{"index":1,"count":4,"confidence":0.33989469707012177,"empirical_correctness":0.0,"signed_gap":0.33989469707012177,"absolute_gap":0.33989469707012177,"min_confidence":0.29749882221221924,"max_confidence":0.3644558787345886},{"index":2,"count":4,"confidence":0.37275516986846924,"empirical_correctness":0.5,"signed_gap":-0.12724483013153076,"absolute_gap":0.12724483013153076,"min_confidence":0.3651926517486572,"max_confidence":0.3799024522304535},{"index":3,"count":4,"confidence":0.3914173021912575,"empirical_correctness":0.25,"signed_gap":0.14141730219125748,"absolute_gap":0.14141730219125748,"min_confidence":0.38891348242759705,"max_confidence":0.39415380358695984},{"index":4,"count":4,"confidence":0.4009641036391258,"empirical_correctness":0.25,"signed_gap":0.15096410363912582,"absolute_gap":0.15096410363912582,"min_confidence":0.3973107635974884,"max_confidence":0.4061334729194641},{"index":5,"count":3,"confidence":0.415979544321696,"empirical_correctness":0.3333333333333333,"signed_gap":0.08264621098836267,"absolute_gap":0.08264621098836267,"min_confidence":0.4088648557662964,"max_confidence":0.42135509848594666},{"index":6,"count":4,"confidence":0.42831479758024216,"empirical_correctness":0.5,"signed_gap":-0.07168520241975784,"absolute_gap":0.07168520241975784,"min_confidence":0.42635732889175415,"max_confidence":0.4310276210308075},{"index":7,"count":4,"confidence":0.5772832632064819,"empirical_correctness":0.0,"signed_gap":0.5772832632064819,"absolute_gap":0.5772832632064819,"min_confidence":0.5440122485160828,"max_confidence":0.6058509349822998},{"index":8,"count":4,"confidence":0.6914319396018982,"empirical_correctness":0.25,"signed_gap":0.4414319396018982,"absolute_gap":0.4414319396018982,"min_confidence":0.6070662140846252,"max_confidence":0.7593733668327332},{"index":9,"count":4,"confidence":0.7748919576406479,"empirical_correctness":0.0,"signed_gap":0.7748919576406479,"absolute_gap":0.7748919576406479,"min_confidence":0.7657516598701477,"max_confidence":0.7888846397399902}]}},"latency_ms":{"n":43,"median":167.82239987514913,"p95":171.12756986171007},"per_action":{"keep":{"n":12,"raw":{"n":12,"acceptable":6,"unsafe":0,"review":0,"automated_count":12,"automated_acceptable":6,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":0.5,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.5,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.2537815976337061,"upper":0.7462184023662939,"confidence_level":0.95}},"effective":{"n":12,"acceptable":1,"unsafe":0,"review":11,"automated_count":1,"automated_acceptable":1,"automated_unsafe":0,"unnecessary_reviews":11,"accuracy":0.08333333333333333,"unsafe_rate":0.0,"review_rate":0.9166666666666666,"automated_coverage":0.08333333333333333,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.9166666666666666,"accuracy_wilson_95":{"lower":0.014865094404917095,"upper":0.35387991114111694,"confidence_level":0.95}}},"drop":{"n":16,"raw":{"n":16,"acceptable":0,"unsafe":14,"review":0,"automated_count":16,"automated_acceptable":0,"automated_unsafe":14,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":0.875,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":0.875,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.1936076805344365,"confidence_level":0.95}},"effective":{"n":16,"acceptable":0,"unsafe":0,"review":16,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":16,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.1936076805344365,"confidence_level":0.95}}},"mask":{"n":2,"raw":{"n":2,"acceptable":0,"unsafe":2,"review":0,"automated_count":2,"automated_acceptable":0,"automated_unsafe":2,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":1.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":1.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.6576197724933469,"confidence_level":0.95}},"effective":{"n":2,"acceptable":0,"unsafe":0,"review":2,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":2,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.6576197724933469,"confidence_level":0.95}}},"pseudonymize":{"n":2,"raw":{"n":2,"acceptable":1,"unsafe":1,"review":0,"automated_count":2,"automated_acceptable":1,"automated_unsafe":1,"unnecessary_reviews":0,"accuracy":0.5,"unsafe_rate":0.5,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.5,"conditional_unsafe":0.5,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.09453120573423074,"upper":0.9054687942657693,"confidence_level":0.95}},"effective":{"n":2,"acceptable":0,"unsafe":0,"review":2,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":2,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.6576197724933469,"confidence_level":0.95}}},"generalize":{"n":6,"raw":{"n":6,"acceptable":2,"unsafe":3,"review":0,"automated_count":6,"automated_acceptable":2,"automated_unsafe":3,"unnecessary_reviews":0,"accuracy":0.3333333333333333,"unsafe_rate":0.5,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.3333333333333333,"conditional_unsafe":0.5,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.09677141110578041,"upper":0.700006684861608,"confidence_level":0.95}},"effective":{"n":6,"acceptable":0,"unsafe":0,"review":6,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":6,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":2.7755575615628914e-17,"upper":0.3903342879021653,"confidence_level":0.95}}},"review":{"n":0,"raw":{"n":0,"acceptable":0,"unsafe":0,"review":0,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":null,"unsafe_rate":null,"review_rate":null,"automated_coverage":null,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":null},"effective":{"n":0,"acceptable":0,"unsafe":0,"review":0,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":null,"unsafe_rate":null,"review_rate":null,"automated_coverage":null,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":null}}}},"diagnostic43-compact_no_samples":{"n":43,"raw":{"n":43,"acceptable":7,"unsafe":21,"review":0,"automated_count":43,"automated_acceptable":7,"automated_unsafe":21,"unnecessary_reviews":0,"accuracy":0.16279069767441862,"unsafe_rate":0.4883720930232558,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.16279069767441862,"conditional_unsafe":0.4883720930232558,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.08116627347062008,"upper":0.29972405893747167,"confidence_level":0.95}},"effective":{"n":43,"acceptable":5,"unsafe":0,"review":43,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":38,"accuracy":0.11627906976744186,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":0.8837209302325582,"accuracy_wilson_95":{"lower":0.05070427739804166,"upper":0.24479161810082128,"confidence_level":0.95}},"confusion":{"raw":{"keep":{"keep":3,"drop":1,"mask":0,"pseudonymize":8,"generalize":0,"review":0},"drop":{"keep":2,"drop":1,"mask":0,"pseudonymize":11,"generalize":2,"review":0},"mask":{"keep":1,"drop":0,"mask":0,"pseudonymize":1,"generalize":0,"review":0},"pseudonymize":{"keep":0,"drop":0,"mask":0,"pseudonymize":2,"generalize":0,"review":0},"generalize":{"keep":0,"drop":1,"mask":0,"pseudonymize":4,"generalize":1,"review":0},"review":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0}},"effective":{"keep":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":12},"drop":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":16},"mask":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":2},"pseudonymize":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":2},"generalize":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":6},"review":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0}}},"probability_metrics":{"singleton_n":38,"multiacceptable_n":5,"multiacceptable_coverage":0.11627906976744186,"nll":1.992656385975507,"nll_zero_mass_n":0,"brier":0.8863120058591302,"acceptable_set_n":43,"acceptable_set_nll":1.8688407444907067,"acceptable_set_zero_mass_n":0},"calibration_all_labels":{"equal_width":{"n":43,"bin_count":10,"ece":0.16106244852376542,"signed_mean_gap":0.16106244852376542,"absolute_mean_gap":0.16106244852376542,"bins":[{"index":0,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.0,"upper":0.1,"upper_inclusive":false},{"index":1,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.1,"upper":0.2,"upper_inclusive":false},{"index":2,"count":19,"confidence":0.25492725089976664,"empirical_correctness":0.15789473684210525,"signed_gap":0.09703251405766139,"absolute_gap":0.09703251405766139,"lower":0.2,"upper":0.3,"upper_inclusive":false},{"index":3,"count":17,"confidence":0.346468539798961,"empirical_correctness":0.23529411764705882,"signed_gap":0.11117442215190215,"absolute_gap":0.11117442215190215,"lower":0.3,"upper":0.4,"upper_inclusive":false},{"index":4,"count":6,"confidence":0.4475339700778325,"empirical_correctness":0.0,"signed_gap":0.4475339700778325,"absolute_gap":0.4475339700778325,"lower":0.4,"upper":0.5,"upper_inclusive":false},{"index":5,"count":1,"confidence":0.5068985223770142,"empirical_correctness":0.0,"signed_gap":0.5068985223770142,"absolute_gap":0.5068985223770142,"lower":0.5,"upper":0.6,"upper_inclusive":false},{"index":6,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.6,"upper":0.7,"upper_inclusive":false},{"index":7,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.7,"upper":0.8,"upper_inclusive":false},{"index":8,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.8,"upper":0.9,"upper_inclusive":false},{"index":9,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.9,"upper":1.0,"upper_inclusive":true}]},"equal_mass":{"n":43,"bin_count":10,"ece":0.2872522473335266,"signed_mean_gap":0.1610624485237654,"absolute_mean_gap":0.1610624485237654,"bins":[{"index":0,"count":4,"confidence":0.2341099977493286,"empirical_correctness":0.0,"signed_gap":0.2341099977493286,"absolute_gap":0.2341099977493286,"min_confidence":0.22321800887584686,"max_confidence":0.24244417250156403},{"index":1,"count":4,"confidence":0.24598624557256699,"empirical_correctness":0.0,"signed_gap":0.24598624557256699,"absolute_gap":0.24598624557256699,"min_confidence":0.2434571385383606,"max_confidence":0.2483418732881546},{"index":2,"count":4,"confidence":0.25594815611839294,"empirical_correctness":0.75,"signed_gap":-0.49405184388160706,"absolute_gap":0.49405184388160706,"min_confidence":0.2516706585884094,"max_confidence":0.2574814558029175},{"index":3,"count":5,"confidence":0.2642290234565735,"empirical_correctness":0.0,"signed_gap":0.2642290234565735,"absolute_gap":0.2642290234565735,"min_confidence":0.257890522480011,"max_confidence":0.2709386348724365},{"index":4,"count":4,"confidence":0.29607704281806946,"empirical_correctness":0.25,"signed_gap":0.04607704281806946,"absolute_gap":0.04607704281806946,"min_confidence":0.28551560640335083,"max_confidence":0.3043205440044403},{"index":5,"count":4,"confidence":0.3157816752791405,"empirical_correctness":0.5,"signed_gap":-0.18421832472085953,"absolute_gap":0.18421832472085953,"min_confidence":0.3047042489051819,"max_confidence":0.32292109727859497},{"index":6,"count":5,"confidence":0.34457799792289734,"empirical_correctness":0.0,"signed_gap":0.34457799792289734,"absolute_gap":0.34457799792289734,"min_confidence":0.3236517906188965,"max_confidence":0.3621060252189636},{"index":7,"count":4,"confidence":0.37674085050821304,"empirical_correctness":0.25,"signed_gap":0.12674085050821304,"absolute_gap":0.12674085050821304,"min_confidence":0.36406365036964417,"max_confidence":0.3875938355922699},{"index":8,"count":4,"confidence":0.41017279773950577,"empirical_correctness":0.0,"signed_gap":0.41017279773950577,"absolute_gap":0.41017279773950577,"min_confidence":0.3913716971874237,"max_confidence":0.4277888834476471},{"index":9,"count":5,"confidence":0.46847662329673767,"empirical_correctness":0.0,"signed_gap":0.46847662329673767,"absolute_gap":0.46847662329673767,"min_confidence":0.4489290714263916,"max_confidence":0.5068985223770142}]}},"calibration_singleton_labels":{"equal_width":{"n":38,"bin_count":10,"ece":0.1378681765575158,"signed_mean_gap":0.1378681765575158,"absolute_mean_gap":0.1378681765575158,"bins":[{"index":0,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.0,"upper":0.1,"upper_inclusive":false},{"index":1,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.1,"upper":0.2,"upper_inclusive":false},{"index":2,"count":17,"confidence":0.2545659007395015,"empirical_correctness":0.17647058823529413,"signed_gap":0.0780953125042074,"absolute_gap":0.0780953125042074,"lower":0.2,"upper":0.3,"upper_inclusive":false},{"index":3,"count":15,"confidence":0.34841110507647194,"empirical_correctness":0.26666666666666666,"signed_gap":0.08174443840980528,"absolute_gap":0.08174443840980528,"lower":0.3,"upper":0.4,"upper_inclusive":false},{"index":4,"count":6,"confidence":0.4475339700778325,"empirical_correctness":0.0,"signed_gap":0.4475339700778325,"absolute_gap":0.4475339700778325,"lower":0.4,"upper":0.5,"upper_inclusive":false},{"index":5,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.5,"upper":0.6,"upper_inclusive":false},{"index":6,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.6,"upper":0.7,"upper_inclusive":false},{"index":7,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.7,"upper":0.8,"upper_inclusive":false},{"index":8,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.8,"upper":0.9,"upper_inclusive":false},{"index":9,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.9,"upper":1.0,"upper_inclusive":true}]},"equal_mass":{"n":38,"bin_count":10,"ece":0.28798953914328623,"signed_mean_gap":0.1378681765575158,"absolute_mean_gap":0.1378681765575158,"bins":[{"index":0,"count":3,"confidence":0.23774066070715585,"empirical_correctness":0.0,"signed_gap":0.23774066070715585,"absolute_gap":0.23774066070715585,"min_confidence":0.22990499436855316,"max_confidence":0.24244417250156403},{"index":1,"count":4,"confidence":0.24598624557256699,"empirical_correctness":0.0,"signed_gap":0.24598624557256699,"absolute_gap":0.24598624557256699,"min_confidence":0.2434571385383606,"max_confidence":0.2483418732881546},{"index":2,"count":4,"confidence":0.25594815611839294,"empirical_correctness":0.75,"signed_gap":-0.49405184388160706,"absolute_gap":0.49405184388160706,"min_confidence":0.2516706585884094,"max_confidence":0.2574814558029175},{"index":3,"count":4,"confidence":0.2625516206026077,"empirical_correctness":0.0,"signed_gap":0.2625516206026077,"absolute_gap":0.2625516206026077,"min_confidence":0.257890522480011,"max_confidence":0.268028199672699},{"index":4,"count":4,"confidence":0.2913697585463524,"empirical_correctness":0.5,"signed_gap":-0.2086302414536476,"absolute_gap":0.2086302414536476,"min_confidence":0.2709386348724365,"max_confidence":0.3047042489051819},{"index":5,"count":3,"confidence":0.31947415073712665,"empirical_correctness":0.3333333333333333,"signed_gap":-0.013859182596206665,"absolute_gap":0.013859182596206665,"min_confidence":0.31631752848625183,"max_confidence":0.32292109727859497},{"index":6,"count":4,"confidence":0.34019599109888077,"empirical_correctness":0.0,"signed_gap":0.34019599109888077,"absolute_gap":0.34019599109888077,"min_confidence":0.3236517906188965,"max_confidence":0.35912418365478516},{"index":7,"count":4,"confidence":0.37674085050821304,"empirical_correctness":0.25,"signed_gap":0.12674085050821304,"absolute_gap":0.12674085050821304,"min_confidence":0.36406365036964417,"max_confidence":0.3875938355922699},{"index":8,"count":4,"confidence":0.41017279773950577,"empirical_correctness":0.0,"signed_gap":0.41017279773950577,"absolute_gap":0.41017279773950577,"min_confidence":0.3913716971874237,"max_confidence":0.4277888834476471},{"index":9,"count":4,"confidence":0.45887114852666855,"empirical_correctness":0.0,"signed_gap":0.45887114852666855,"absolute_gap":0.45887114852666855,"min_confidence":0.4489290714263916,"max_confidence":0.4735393226146698}]}},"latency_ms":{"n":43,"median":168.25350001454353,"p95":171.21108006685972},"per_action":{"keep":{"n":12,"raw":{"n":12,"acceptable":3,"unsafe":0,"review":0,"automated_count":12,"automated_acceptable":3,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":0.25,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.25,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.08894166839405471,"upper":0.5323053349335657,"confidence_level":0.95}},"effective":{"n":12,"acceptable":0,"unsafe":0,"review":12,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":12,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":2.7755575615628914e-17,"upper":0.24249400665524085,"confidence_level":0.95}}},"drop":{"n":16,"raw":{"n":16,"acceptable":1,"unsafe":14,"review":0,"automated_count":16,"automated_acceptable":1,"automated_unsafe":14,"unnecessary_reviews":0,"accuracy":0.0625,"unsafe_rate":0.875,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0625,"conditional_unsafe":0.875,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.01111934476464252,"upper":0.28328737570298945,"confidence_level":0.95}},"effective":{"n":16,"acceptable":0,"unsafe":0,"review":16,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":16,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.1936076805344365,"confidence_level":0.95}}},"mask":{"n":2,"raw":{"n":2,"acceptable":0,"unsafe":2,"review":0,"automated_count":2,"automated_acceptable":0,"automated_unsafe":2,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":1.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":1.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.6576197724933469,"confidence_level":0.95}},"effective":{"n":2,"acceptable":0,"unsafe":0,"review":2,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":2,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.6576197724933469,"confidence_level":0.95}}},"pseudonymize":{"n":2,"raw":{"n":2,"acceptable":2,"unsafe":0,"review":0,"automated_count":2,"automated_acceptable":2,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":1.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.34238022750665303,"upper":1.0,"confidence_level":0.95}},"effective":{"n":2,"acceptable":0,"unsafe":0,"review":2,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":2,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.6576197724933469,"confidence_level":0.95}}},"generalize":{"n":6,"raw":{"n":6,"acceptable":1,"unsafe":0,"review":0,"automated_count":6,"automated_acceptable":1,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":0.16666666666666666,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.16666666666666666,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.030053369748306635,"upper":0.5635028221864702,"confidence_level":0.95}},"effective":{"n":6,"acceptable":0,"unsafe":0,"review":6,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":6,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":2.7755575615628914e-17,"upper":0.3903342879021653,"confidence_level":0.95}}},"review":{"n":0,"raw":{"n":0,"acceptable":0,"unsafe":0,"review":0,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":null,"unsafe_rate":null,"review_rate":null,"automated_coverage":null,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":null},"effective":{"n":0,"acceptable":0,"unsafe":0,"review":0,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":null,"unsafe_rate":null,"review_rate":null,"automated_coverage":null,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":null}}}},"challenge56-no_samples":{"n":56,"raw":{"n":56,"acceptable":16,"unsafe":40,"review":0,"automated_count":56,"automated_acceptable":16,"automated_unsafe":40,"unnecessary_reviews":0,"accuracy":0.2857142857142857,"unsafe_rate":0.7142857142857143,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.2857142857142857,"conditional_unsafe":0.7142857142857143,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.18418775143858906,"upper":0.4147525071551667,"confidence_level":0.95}},"effective":{"n":56,"acceptable":7,"unsafe":0,"review":51,"automated_count":5,"automated_acceptable":5,"automated_unsafe":0,"unnecessary_reviews":49,"accuracy":0.125,"unsafe_rate":0.0,"review_rate":0.9107142857142857,"automated_coverage":0.08928571428571429,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.875,"accuracy_wilson_95":{"lower":0.06189068695717122,"upper":0.2362547655819014,"confidence_level":0.95}},"confusion":{"raw":{"keep":{"keep":8,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0},"drop":{"keep":22,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0},"mask":{"keep":0,"drop":0,"mask":8,"pseudonymize":0,"generalize":0,"review":0},"pseudonymize":{"keep":8,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0},"generalize":{"keep":8,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0},"review":{"keep":2,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0}},"effective":{"keep":{"keep":4,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":4},"drop":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":22},"mask":{"keep":0,"drop":0,"mask":1,"pseudonymize":0,"generalize":0,"review":7},"pseudonymize":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":8},"generalize":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":8},"review":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":2}}},"probability_metrics":{"singleton_n":56,"multiacceptable_n":0,"multiacceptable_coverage":0.0,"nll":2.1055352172834505,"nll_zero_mass_n":0,"brier":0.8853223142939568,"acceptable_set_n":56,"acceptable_set_nll":2.1055352172834505,"acceptable_set_zero_mass_n":0},"calibration_all_labels":{"equal_width":{"n":56,"bin_count":10,"ece":0.1922352228845869,"signed_mean_gap":0.1922352228845869,"absolute_mean_gap":0.1922352228845869,"bins":[{"index":0,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.0,"upper":0.1,"upper_inclusive":false},{"index":1,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.1,"upper":0.2,"upper_inclusive":false},{"index":2,"count":5,"confidence":0.27777854800224305,"empirical_correctness":0.0,"signed_gap":0.27777854800224305,"absolute_gap":0.27777854800224305,"lower":0.2,"upper":0.3,"upper_inclusive":false},{"index":3,"count":15,"confidence":0.34934516151746114,"empirical_correctness":0.26666666666666666,"signed_gap":0.08267849485079448,"absolute_gap":0.08267849485079448,"lower":0.3,"upper":0.4,"upper_inclusive":false},{"index":4,"count":13,"confidence":0.45524930495482224,"empirical_correctness":0.3076923076923077,"signed_gap":0.14755699726251453,"absolute_gap":0.14755699726251453,"lower":0.4,"upper":0.5,"upper_inclusive":false},{"index":5,"count":11,"confidence":0.5340102423321117,"empirical_correctness":0.18181818181818182,"signed_gap":0.35219206051392987,"absolute_gap":0.35219206051392987,"lower":0.5,"upper":0.6,"upper_inclusive":false},{"index":6,"count":8,"confidence":0.6525365188717842,"empirical_correctness":0.625,"signed_gap":0.02753651887178421,"absolute_gap":0.02753651887178421,"lower":0.6,"upper":0.7,"upper_inclusive":false},{"index":7,"count":2,"confidence":0.726069837808609,"empirical_correctness":0.5,"signed_gap":0.226069837808609,"absolute_gap":0.226069837808609,"lower":0.7,"upper":0.8,"upper_inclusive":false},{"index":8,"count":2,"confidence":0.8356584310531616,"empirical_correctness":0.0,"signed_gap":0.8356584310531616,"absolute_gap":0.8356584310531616,"lower":0.8,"upper":0.9,"upper_inclusive":false},{"index":9,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.9,"upper":1.0,"upper_inclusive":true}]},"equal_mass":{"n":56,"bin_count":10,"ece":0.26419789237635477,"signed_mean_gap":0.1922352228845869,"absolute_mean_gap":0.1922352228845869,"bins":[{"index":0,"count":5,"confidence":0.27777854800224305,"empirical_correctness":0.0,"signed_gap":0.27777854800224305,"absolute_gap":0.27777854800224305,"min_confidence":0.2604900300502777,"max_confidence":0.2933126986026764},{"index":1,"count":6,"confidence":0.32996755341688794,"empirical_correctness":0.0,"signed_gap":0.32996755341688794,"absolute_gap":0.32996755341688794,"min_confidence":0.3041587769985199,"max_confidence":0.3386088013648987},{"index":2,"count":5,"confidence":0.3502249300479889,"empirical_correctness":0.2,"signed_gap":0.15022493004798887,"absolute_gap":0.15022493004798887,"min_confidence":0.3393351435661316,"max_confidence":0.3659653961658478},{"index":3,"count":6,"confidence":0.38687221705913544,"empirical_correctness":0.6666666666666666,"signed_gap":-0.2797944496075312,"absolute_gap":0.2797944496075312,"min_confidence":0.3663260340690613,"max_confidence":0.41126781702041626},{"index":4,"count":6,"confidence":0.44396865864594776,"empirical_correctness":0.5,"signed_gap":-0.056031341354052244,"absolute_gap":0.056031341354052244,"min_confidence":0.4281836748123169,"max_confidence":0.470392644405365},{"index":5,"count":5,"confidence":0.488488632440567,"empirical_correctness":0.0,"signed_gap":0.488488632440567,"absolute_gap":0.488488632440567,"min_confidence":0.4851638376712799,"max_confidence":0.4911767840385437},{"index":6,"count":6,"confidence":0.5154514312744141,"empirical_correctness":0.16666666666666666,"signed_gap":0.34878476460774743,"absolute_gap":0.34878476460774743,"min_confidence":0.500543475151062,"max_confidence":0.5353272557258606},{"index":7,"count":5,"confidence":0.5562808156013489,"empirical_correctness":0.2,"signed_gap":0.3562808156013489,"absolute_gap":0.3562808156013489,"min_confidence":0.5354155898094177,"max_confidence":0.5946770906448364},{"index":8,"count":6,"confidence":0.6393581926822662,"empirical_correctness":0.5,"signed_gap":0.13935819268226624,"absolute_gap":0.13935819268226624,"min_confidence":0.6175512671470642,"max_confidence":0.6600439548492432},{"index":9,"count":6,"confidence":0.7512665887673696,"empirical_correctness":0.5,"signed_gap":0.2512665887673696,"absolute_gap":0.2512665887673696,"min_confidence":0.6880387663841248,"max_confidence":0.8394571542739868}]}},"calibration_singleton_labels":{"equal_width":{"n":56,"bin_count":10,"ece":0.1922352228845869,"signed_mean_gap":0.1922352228845869,"absolute_mean_gap":0.1922352228845869,"bins":[{"index":0,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.0,"upper":0.1,"upper_inclusive":false},{"index":1,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.1,"upper":0.2,"upper_inclusive":false},{"index":2,"count":5,"confidence":0.27777854800224305,"empirical_correctness":0.0,"signed_gap":0.27777854800224305,"absolute_gap":0.27777854800224305,"lower":0.2,"upper":0.3,"upper_inclusive":false},{"index":3,"count":15,"confidence":0.34934516151746114,"empirical_correctness":0.26666666666666666,"signed_gap":0.08267849485079448,"absolute_gap":0.08267849485079448,"lower":0.3,"upper":0.4,"upper_inclusive":false},{"index":4,"count":13,"confidence":0.45524930495482224,"empirical_correctness":0.3076923076923077,"signed_gap":0.14755699726251453,"absolute_gap":0.14755699726251453,"lower":0.4,"upper":0.5,"upper_inclusive":false},{"index":5,"count":11,"confidence":0.5340102423321117,"empirical_correctness":0.18181818181818182,"signed_gap":0.35219206051392987,"absolute_gap":0.35219206051392987,"lower":0.5,"upper":0.6,"upper_inclusive":false},{"index":6,"count":8,"confidence":0.6525365188717842,"empirical_correctness":0.625,"signed_gap":0.02753651887178421,"absolute_gap":0.02753651887178421,"lower":0.6,"upper":0.7,"upper_inclusive":false},{"index":7,"count":2,"confidence":0.726069837808609,"empirical_correctness":0.5,"signed_gap":0.226069837808609,"absolute_gap":0.226069837808609,"lower":0.7,"upper":0.8,"upper_inclusive":false},{"index":8,"count":2,"confidence":0.8356584310531616,"empirical_correctness":0.0,"signed_gap":0.8356584310531616,"absolute_gap":0.8356584310531616,"lower":0.8,"upper":0.9,"upper_inclusive":false},{"index":9,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.9,"upper":1.0,"upper_inclusive":true}]},"equal_mass":{"n":56,"bin_count":10,"ece":0.26419789237635477,"signed_mean_gap":0.1922352228845869,"absolute_mean_gap":0.1922352228845869,"bins":[{"index":0,"count":5,"confidence":0.27777854800224305,"empirical_correctness":0.0,"signed_gap":0.27777854800224305,"absolute_gap":0.27777854800224305,"min_confidence":0.2604900300502777,"max_confidence":0.2933126986026764},{"index":1,"count":6,"confidence":0.32996755341688794,"empirical_correctness":0.0,"signed_gap":0.32996755341688794,"absolute_gap":0.32996755341688794,"min_confidence":0.3041587769985199,"max_confidence":0.3386088013648987},{"index":2,"count":5,"confidence":0.3502249300479889,"empirical_correctness":0.2,"signed_gap":0.15022493004798887,"absolute_gap":0.15022493004798887,"min_confidence":0.3393351435661316,"max_confidence":0.3659653961658478},{"index":3,"count":6,"confidence":0.38687221705913544,"empirical_correctness":0.6666666666666666,"signed_gap":-0.2797944496075312,"absolute_gap":0.2797944496075312,"min_confidence":0.3663260340690613,"max_confidence":0.41126781702041626},{"index":4,"count":6,"confidence":0.44396865864594776,"empirical_correctness":0.5,"signed_gap":-0.056031341354052244,"absolute_gap":0.056031341354052244,"min_confidence":0.4281836748123169,"max_confidence":0.470392644405365},{"index":5,"count":5,"confidence":0.488488632440567,"empirical_correctness":0.0,"signed_gap":0.488488632440567,"absolute_gap":0.488488632440567,"min_confidence":0.4851638376712799,"max_confidence":0.4911767840385437},{"index":6,"count":6,"confidence":0.5154514312744141,"empirical_correctness":0.16666666666666666,"signed_gap":0.34878476460774743,"absolute_gap":0.34878476460774743,"min_confidence":0.500543475151062,"max_confidence":0.5353272557258606},{"index":7,"count":5,"confidence":0.5562808156013489,"empirical_correctness":0.2,"signed_gap":0.3562808156013489,"absolute_gap":0.3562808156013489,"min_confidence":0.5354155898094177,"max_confidence":0.5946770906448364},{"index":8,"count":6,"confidence":0.6393581926822662,"empirical_correctness":0.5,"signed_gap":0.13935819268226624,"absolute_gap":0.13935819268226624,"min_confidence":0.6175512671470642,"max_confidence":0.6600439548492432},{"index":9,"count":6,"confidence":0.7512665887673696,"empirical_correctness":0.5,"signed_gap":0.2512665887673696,"absolute_gap":0.2512665887673696,"min_confidence":0.6880387663841248,"max_confidence":0.8394571542739868}]}},"latency_ms":{"n":56,"median":144.3909500958398,"p95":149.7476499644108},"per_action":{"keep":{"n":8,"raw":{"n":8,"acceptable":8,"unsafe":0,"review":0,"automated_count":8,"automated_acceptable":8,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":1.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.6755924351161198,"upper":1.0,"confidence_level":0.95}},"effective":{"n":8,"acceptable":4,"unsafe":0,"review":4,"automated_count":4,"automated_acceptable":4,"automated_unsafe":0,"unnecessary_reviews":4,"accuracy":0.5,"unsafe_rate":0.0,"review_rate":0.5,"automated_coverage":0.5,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.5,"accuracy_wilson_95":{"lower":0.21521606221387757,"upper":0.7847839377861224,"confidence_level":0.95}}},"drop":{"n":22,"raw":{"n":22,"acceptable":0,"unsafe":22,"review":0,"automated_count":22,"automated_acceptable":0,"automated_unsafe":22,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":1.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":1.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.14865487460862087,"confidence_level":0.95}},"effective":{"n":22,"acceptable":0,"unsafe":0,"review":22,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":22,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.14865487460862087,"confidence_level":0.95}}},"mask":{"n":8,"raw":{"n":8,"acceptable":8,"unsafe":0,"review":0,"automated_count":8,"automated_acceptable":8,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":1.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.6755924351161198,"upper":1.0,"confidence_level":0.95}},"effective":{"n":8,"acceptable":1,"unsafe":0,"review":7,"automated_count":1,"automated_acceptable":1,"automated_unsafe":0,"unnecessary_reviews":7,"accuracy":0.125,"unsafe_rate":0.0,"review_rate":0.875,"automated_coverage":0.125,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.875,"accuracy_wilson_95":{"lower":0.02241749145005667,"upper":0.4708881822128535,"confidence_level":0.95}}},"pseudonymize":{"n":8,"raw":{"n":8,"acceptable":0,"unsafe":8,"review":0,"automated_count":8,"automated_acceptable":0,"automated_unsafe":8,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":1.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":1.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.32440756488388023,"confidence_level":0.95}},"effective":{"n":8,"acceptable":0,"unsafe":0,"review":8,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":8,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.32440756488388023,"confidence_level":0.95}}},"generalize":{"n":8,"raw":{"n":8,"acceptable":0,"unsafe":8,"review":0,"automated_count":8,"automated_acceptable":0,"automated_unsafe":8,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":1.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":1.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.32440756488388023,"confidence_level":0.95}},"effective":{"n":8,"acceptable":0,"unsafe":0,"review":8,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":8,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.32440756488388023,"confidence_level":0.95}}},"review":{"n":2,"raw":{"n":2,"acceptable":0,"unsafe":2,"review":0,"automated_count":2,"automated_acceptable":0,"automated_unsafe":2,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":1.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":1.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.6576197724933469,"confidence_level":0.95}},"effective":{"n":2,"acceptable":2,"unsafe":0,"review":2,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":1.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.34238022750665303,"upper":1.0,"confidence_level":0.95}}}}},"challenge56-compact_no_samples":{"n":56,"raw":{"n":56,"acceptable":16,"unsafe":40,"review":0,"automated_count":56,"automated_acceptable":16,"automated_unsafe":40,"unnecessary_reviews":0,"accuracy":0.2857142857142857,"unsafe_rate":0.7142857142857143,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.2857142857142857,"conditional_unsafe":0.7142857142857143,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.18418775143858906,"upper":0.4147525071551667,"confidence_level":0.95}},"effective":{"n":56,"acceptable":7,"unsafe":0,"review":51,"automated_count":5,"automated_acceptable":5,"automated_unsafe":0,"unnecessary_reviews":49,"accuracy":0.125,"unsafe_rate":0.0,"review_rate":0.9107142857142857,"automated_coverage":0.08928571428571429,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.875,"accuracy_wilson_95":{"lower":0.06189068695717122,"upper":0.2362547655819014,"confidence_level":0.95}},"confusion":{"raw":{"keep":{"keep":8,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0},"drop":{"keep":21,"drop":0,"mask":0,"pseudonymize":1,"generalize":0,"review":0},"mask":{"keep":0,"drop":0,"mask":0,"pseudonymize":8,"generalize":0,"review":0},"pseudonymize":{"keep":0,"drop":0,"mask":0,"pseudonymize":8,"generalize":0,"review":0},"generalize":{"keep":8,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0},"review":{"keep":2,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0}},"effective":{"keep":{"keep":5,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":3},"drop":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":22},"mask":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":8},"pseudonymize":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":8},"generalize":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":8},"review":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":2}}},"probability_metrics":{"singleton_n":56,"multiacceptable_n":0,"multiacceptable_coverage":0.0,"nll":1.8664081454025752,"nll_zero_mass_n":0,"brier":0.8248669285799485,"acceptable_set_n":56,"acceptable_set_nll":1.8664081454025752,"acceptable_set_zero_mass_n":0},"calibration_all_labels":{"equal_width":{"n":56,"bin_count":10,"ece":0.1145823278597423,"signed_mean_gap":0.07497042204652514,"absolute_mean_gap":0.07497042204652514,"bins":[{"index":0,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.0,"upper":0.1,"upper_inclusive":false},{"index":1,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.1,"upper":0.2,"upper_inclusive":false},{"index":2,"count":12,"confidence":0.2862061982353528,"empirical_correctness":0.0,"signed_gap":0.2862061982353528,"absolute_gap":0.2862061982353528,"lower":0.2,"upper":0.3,"upper_inclusive":false},{"index":3,"count":29,"confidence":0.3404483661569398,"empirical_correctness":0.27586206896551724,"signed_gap":0.06458629719142256,"absolute_gap":0.06458629719142256,"lower":0.3,"upper":0.4,"upper_inclusive":false},{"index":4,"count":15,"confidence":0.45939110914866127,"empirical_correctness":0.5333333333333333,"signed_gap":-0.07394222418467206,"absolute_gap":0.07394222418467206,"lower":0.4,"upper":0.5,"upper_inclusive":false},{"index":5,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.5,"upper":0.6,"upper_inclusive":false},{"index":6,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.6,"upper":0.7,"upper_inclusive":false},{"index":7,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.7,"upper":0.8,"upper_inclusive":false},{"index":8,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.8,"upper":0.9,"upper_inclusive":false},{"index":9,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.9,"upper":1.0,"upper_inclusive":true}]},"equal_mass":{"n":56,"bin_count":10,"ece":0.22537152575595035,"signed_mean_gap":0.07497042204652515,"absolute_mean_gap":0.07497042204652515,"bins":[{"index":0,"count":5,"confidence":0.27560398578643797,"empirical_correctness":0.0,"signed_gap":0.27560398578643797,"absolute_gap":0.27560398578643797,"min_confidence":0.26241156458854675,"max_confidence":0.287186861038208},{"index":1,"count":6,"confidence":0.2928162564833959,"empirical_correctness":0.0,"signed_gap":0.2928162564833959,"absolute_gap":0.2928162564833959,"min_confidence":0.28734445571899414,"max_confidence":0.2979046106338501},{"index":2,"count":5,"confidence":0.30520249605178834,"empirical_correctness":0.0,"signed_gap":0.30520249605178834,"absolute_gap":0.30520249605178834,"min_confidence":0.2995569109916687,"max_confidence":0.3145502507686615},{"index":3,"count":6,"confidence":0.32214564581712085,"empirical_correctness":0.0,"signed_gap":0.32214564581712085,"absolute_gap":0.32214564581712085,"min_confidence":0.3180494010448456,"max_confidence":0.3262813091278076},{"index":4,"count":6,"confidence":0.33489465216795605,"empirical_correctness":0.5,"signed_gap":-0.16510534783204395,"absolute_gap":0.16510534783204395,"min_confidence":0.3268413841724396,"max_confidence":0.3459439277648926},{"index":5,"count":5,"confidence":0.3537824213504791,"empirical_correctness":0.2,"signed_gap":0.1537824213504791,"absolute_gap":0.1537824213504791,"min_confidence":0.3472408056259155,"max_confidence":0.3567177653312683},{"index":6,"count":6,"confidence":0.3617786268393199,"empirical_correctness":0.6666666666666666,"signed_gap":-0.30488803982734675,"absolute_gap":0.30488803982734675,"min_confidence":0.35852620005607605,"max_confidence":0.3652629852294922},{"index":7,"count":5,"confidence":0.40937172174453734,"empirical_correctness":0.2,"signed_gap":0.20937172174453733,"absolute_gap":0.20937172174453733,"min_confidence":0.3676661550998688,"max_confidence":0.4373212456703186},{"index":8,"count":6,"confidence":0.4520268738269806,"empirical_correctness":0.5,"signed_gap":-0.04797312617301941,"absolute_gap":0.04797312617301941,"min_confidence":0.44142770767211914,"max_confidence":0.4701002538204193},{"index":9,"count":6,"confidence":0.4827613631884257,"empirical_correctness":0.6666666666666666,"signed_gap":-0.1839053034782409,"absolute_gap":0.1839053034782409,"min_confidence":0.47157537937164307,"max_confidence":0.4899725914001465}]}},"calibration_singleton_labels":{"equal_width":{"n":56,"bin_count":10,"ece":0.1145823278597423,"signed_mean_gap":0.07497042204652514,"absolute_mean_gap":0.07497042204652514,"bins":[{"index":0,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.0,"upper":0.1,"upper_inclusive":false},{"index":1,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.1,"upper":0.2,"upper_inclusive":false},{"index":2,"count":12,"confidence":0.2862061982353528,"empirical_correctness":0.0,"signed_gap":0.2862061982353528,"absolute_gap":0.2862061982353528,"lower":0.2,"upper":0.3,"upper_inclusive":false},{"index":3,"count":29,"confidence":0.3404483661569398,"empirical_correctness":0.27586206896551724,"signed_gap":0.06458629719142256,"absolute_gap":0.06458629719142256,"lower":0.3,"upper":0.4,"upper_inclusive":false},{"index":4,"count":15,"confidence":0.45939110914866127,"empirical_correctness":0.5333333333333333,"signed_gap":-0.07394222418467206,"absolute_gap":0.07394222418467206,"lower":0.4,"upper":0.5,"upper_inclusive":false},{"index":5,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.5,"upper":0.6,"upper_inclusive":false},{"index":6,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.6,"upper":0.7,"upper_inclusive":false},{"index":7,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.7,"upper":0.8,"upper_inclusive":false},{"index":8,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.8,"upper":0.9,"upper_inclusive":false},{"index":9,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.9,"upper":1.0,"upper_inclusive":true}]},"equal_mass":{"n":56,"bin_count":10,"ece":0.22537152575595035,"signed_mean_gap":0.07497042204652515,"absolute_mean_gap":0.07497042204652515,"bins":[{"index":0,"count":5,"confidence":0.27560398578643797,"empirical_correctness":0.0,"signed_gap":0.27560398578643797,"absolute_gap":0.27560398578643797,"min_confidence":0.26241156458854675,"max_confidence":0.287186861038208},{"index":1,"count":6,"confidence":0.2928162564833959,"empirical_correctness":0.0,"signed_gap":0.2928162564833959,"absolute_gap":0.2928162564833959,"min_confidence":0.28734445571899414,"max_confidence":0.2979046106338501},{"index":2,"count":5,"confidence":0.30520249605178834,"empirical_correctness":0.0,"signed_gap":0.30520249605178834,"absolute_gap":0.30520249605178834,"min_confidence":0.2995569109916687,"max_confidence":0.3145502507686615},{"index":3,"count":6,"confidence":0.32214564581712085,"empirical_correctness":0.0,"signed_gap":0.32214564581712085,"absolute_gap":0.32214564581712085,"min_confidence":0.3180494010448456,"max_confidence":0.3262813091278076},{"index":4,"count":6,"confidence":0.33489465216795605,"empirical_correctness":0.5,"signed_gap":-0.16510534783204395,"absolute_gap":0.16510534783204395,"min_confidence":0.3268413841724396,"max_confidence":0.3459439277648926},{"index":5,"count":5,"confidence":0.3537824213504791,"empirical_correctness":0.2,"signed_gap":0.1537824213504791,"absolute_gap":0.1537824213504791,"min_confidence":0.3472408056259155,"max_confidence":0.3567177653312683},{"index":6,"count":6,"confidence":0.3617786268393199,"empirical_correctness":0.6666666666666666,"signed_gap":-0.30488803982734675,"absolute_gap":0.30488803982734675,"min_confidence":0.35852620005607605,"max_confidence":0.3652629852294922},{"index":7,"count":5,"confidence":0.40937172174453734,"empirical_correctness":0.2,"signed_gap":0.20937172174453733,"absolute_gap":0.20937172174453733,"min_confidence":0.3676661550998688,"max_confidence":0.4373212456703186},{"index":8,"count":6,"confidence":0.4520268738269806,"empirical_correctness":0.5,"signed_gap":-0.04797312617301941,"absolute_gap":0.04797312617301941,"min_confidence":0.44142770767211914,"max_confidence":0.4701002538204193},{"index":9,"count":6,"confidence":0.4827613631884257,"empirical_correctness":0.6666666666666666,"signed_gap":-0.1839053034782409,"absolute_gap":0.1839053034782409,"min_confidence":0.47157537937164307,"max_confidence":0.4899725914001465}]}},"latency_ms":{"n":56,"median":145.2226999681443,"p95":150.39970015641302},"per_action":{"keep":{"n":8,"raw":{"n":8,"acceptable":8,"unsafe":0,"review":0,"automated_count":8,"automated_acceptable":8,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":1.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.6755924351161198,"upper":1.0,"confidence_level":0.95}},"effective":{"n":8,"acceptable":5,"unsafe":0,"review":3,"automated_count":5,"automated_acceptable":5,"automated_unsafe":0,"unnecessary_reviews":3,"accuracy":0.625,"unsafe_rate":0.0,"review_rate":0.375,"automated_coverage":0.625,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.375,"accuracy_wilson_95":{"lower":0.30574239460262737,"upper":0.8631557141764026,"confidence_level":0.95}}},"drop":{"n":22,"raw":{"n":22,"acceptable":0,"unsafe":22,"review":0,"automated_count":22,"automated_acceptable":0,"automated_unsafe":22,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":1.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":1.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.14865487460862087,"confidence_level":0.95}},"effective":{"n":22,"acceptable":0,"unsafe":0,"review":22,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":22,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.14865487460862087,"confidence_level":0.95}}},"mask":{"n":8,"raw":{"n":8,"acceptable":0,"unsafe":8,"review":0,"automated_count":8,"automated_acceptable":0,"automated_unsafe":8,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":1.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":1.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.32440756488388023,"confidence_level":0.95}},"effective":{"n":8,"acceptable":0,"unsafe":0,"review":8,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":8,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.32440756488388023,"confidence_level":0.95}}},"pseudonymize":{"n":8,"raw":{"n":8,"acceptable":8,"unsafe":0,"review":0,"automated_count":8,"automated_acceptable":8,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":1.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.6755924351161198,"upper":1.0,"confidence_level":0.95}},"effective":{"n":8,"acceptable":0,"unsafe":0,"review":8,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":8,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.32440756488388023,"confidence_level":0.95}}},"generalize":{"n":8,"raw":{"n":8,"acceptable":0,"unsafe":8,"review":0,"automated_count":8,"automated_acceptable":0,"automated_unsafe":8,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":1.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":1.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.32440756488388023,"confidence_level":0.95}},"effective":{"n":8,"acceptable":0,"unsafe":0,"review":8,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":8,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.32440756488388023,"confidence_level":0.95}}},"review":{"n":2,"raw":{"n":2,"acceptable":0,"unsafe":2,"review":0,"automated_count":2,"automated_acceptable":0,"automated_unsafe":2,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":1.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":1.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.6576197724933469,"confidence_level":0.95}},"effective":{"n":2,"acceptable":2,"unsafe":0,"review":2,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":1.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.34238022750665303,"upper":1.0,"confidence_level":0.95}}}}}},"composite":{"n":198,"scope":"descriptive sum across four correlated panels","raw":{"n":198,"acceptable":48,"unsafe":126,"review":0,"automated_count":198,"automated_acceptable":48,"automated_unsafe":126,"unnecessary_reviews":0,"accuracy":0.24242424242424243,"unsafe_rate":0.6363636363636364,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.24242424242424243,"conditional_unsafe":0.6363636363636364,"unnecessary_review_rate":0.0},"effective":{"n":198,"acceptable":25,"unsafe":0,"review":187,"automated_count":11,"automated_acceptable":11,"automated_unsafe":0,"unnecessary_reviews":173,"accuracy":0.12626262626262627,"unsafe_rate":0.0,"review_rate":0.9444444444444444,"automated_coverage":0.05555555555555555,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.8737373737373737}}},{"id":"exaone-native-bf16","platform":"Local","model_id":"carrtesy/EXAONE-4.0-1.2B-JEV","revision":"4341b28896ceac4e1c4ee9bf6a692b425adecefe","loading":"bfloat16 / none","scoring_kind":"Native decision readout","settings":{"dtype":"bfloat16","quantization":"none","max_tokens":4096,"temperature":1.2643285658964605,"truncation":false,"cpu_offload":false,"guard_threshold":0.45,"seed":0,"tf32":false},"scoring_metadata":{"backend":"native_exaone_choice","probability_source":"native published fixed-temperature label probabilities","latency_scope":"synchronous full forward including input transfer and score readout; tokenization excluded","equal_mass_convention":"rank bins with floor boundaries at floor(i*n/k), k=min(10,n//2); minimum two rows"},"load_ms":null,"peak_allocated_bytes":2714288128,"peak_reserved_bytes":3137339392,"panels":{"diagnostic43-no_samples":{"n":43,"raw":{"n":43,"acceptable":18,"unsafe":6,"review":0,"automated_count":43,"automated_acceptable":18,"automated_unsafe":6,"unnecessary_reviews":0,"accuracy":0.4186046511627907,"unsafe_rate":0.13953488372093023,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.4186046511627907,"conditional_unsafe":0.13953488372093023,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.28384550927849195,"upper":0.5667142261303578,"confidence_level":0.95}},"effective":{"n":43,"acceptable":18,"unsafe":3,"review":12,"automated_count":31,"automated_acceptable":17,"automated_unsafe":3,"unnecessary_reviews":11,"accuracy":0.4186046511627907,"unsafe_rate":0.06976744186046512,"review_rate":0.27906976744186046,"automated_coverage":0.7209302325581395,"conditional_correctness":0.5483870967741935,"conditional_unsafe":0.0967741935483871,"unnecessary_review_rate":0.2558139534883721,"accuracy_wilson_95":{"lower":0.28384550927849195,"upper":0.5667142261303578,"confidence_level":0.95}},"confusion":{"raw":{"keep":{"keep":0,"drop":6,"mask":3,"pseudonymize":3,"generalize":0,"review":0},"drop":{"keep":0,"drop":9,"mask":2,"pseudonymize":5,"generalize":0,"review":0},"mask":{"keep":0,"drop":0,"mask":2,"pseudonymize":0,"generalize":0,"review":0},"pseudonymize":{"keep":0,"drop":0,"mask":0,"pseudonymize":2,"generalize":0,"review":0},"generalize":{"keep":0,"drop":2,"mask":0,"pseudonymize":3,"generalize":1,"review":0},"review":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0}},"effective":{"keep":{"keep":0,"drop":6,"mask":3,"pseudonymize":0,"generalize":0,"review":3},"drop":{"keep":0,"drop":9,"mask":2,"pseudonymize":3,"generalize":0,"review":2},"mask":{"keep":0,"drop":0,"mask":2,"pseudonymize":0,"generalize":0,"review":0},"pseudonymize":{"keep":0,"drop":0,"mask":0,"pseudonymize":2,"generalize":0,"review":0},"generalize":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":6},"review":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0}}},"probability_metrics":{"singleton_n":38,"multiacceptable_n":5,"multiacceptable_coverage":0.11627906976744186,"nll":1.9984781863485865,"nll_zero_mass_n":0,"brier":0.8957207382221337,"acceptable_set_n":43,"acceptable_set_nll":1.7922332555360887,"acceptable_set_zero_mass_n":0},"calibration_all_labels":{"equal_width":{"n":43,"bin_count":10,"ece":0.2705726942350698,"signed_mean_gap":0.26617093003073405,"absolute_mean_gap":0.26617093003073405,"bins":[{"index":0,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.0,"upper":0.1,"upper_inclusive":false},{"index":1,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.1,"upper":0.2,"upper_inclusive":false},{"index":2,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.2,"upper":0.3,"upper_inclusive":false},{"index":3,"count":4,"confidence":0.3523872196674347,"empirical_correctness":0.25,"signed_gap":0.10238721966743469,"absolute_gap":0.10238721966743469,"lower":0.3,"upper":0.4,"upper_inclusive":false},{"index":4,"count":5,"confidence":0.4790050745010376,"empirical_correctness":0.2,"signed_gap":0.27900507450103756,"absolute_gap":0.27900507450103756,"lower":0.4,"upper":0.5,"upper_inclusive":false},{"index":5,"count":3,"confidence":0.5428890188535055,"empirical_correctness":0.3333333333333333,"signed_gap":0.20955568552017217,"absolute_gap":0.20955568552017217,"lower":0.5,"upper":0.6,"upper_inclusive":false},{"index":6,"count":7,"confidence":0.6455264176641192,"empirical_correctness":0.2857142857142857,"signed_gap":0.3598121319498335,"absolute_gap":0.3598121319498335,"lower":0.6,"upper":0.7,"upper_inclusive":false},{"index":7,"count":8,"confidence":0.7419809177517891,"empirical_correctness":0.5,"signed_gap":0.2419809177517891,"absolute_gap":0.2419809177517891,"lower":0.7,"upper":0.8,"upper_inclusive":false},{"index":8,"count":15,"confidence":0.8434809565544128,"empirical_correctness":0.5333333333333333,"signed_gap":0.31014762322107947,"absolute_gap":0.31014762322107947,"lower":0.8,"upper":0.9,"upper_inclusive":false},{"index":9,"count":1,"confidence":0.905362069606781,"empirical_correctness":1.0,"signed_gap":-0.094637930393219,"absolute_gap":0.094637930393219,"lower":0.9,"upper":1.0,"upper_inclusive":true}]},"equal_mass":{"n":43,"bin_count":10,"ece":0.27616351565649344,"signed_mean_gap":0.26617093003073405,"absolute_mean_gap":0.26617093003073405,"bins":[{"index":0,"count":4,"confidence":0.3523872196674347,"empirical_correctness":0.25,"signed_gap":0.10238721966743469,"absolute_gap":0.10238721966743469,"min_confidence":0.32169023156166077,"max_confidence":0.3903771638870239},{"index":1,"count":4,"confidence":0.4746949002146721,"empirical_correctness":0.25,"signed_gap":0.2246949002146721,"absolute_gap":0.2246949002146721,"min_confidence":0.44607916474342346,"max_confidence":0.4952767789363861},{"index":2,"count":4,"confidence":0.531228207051754,"empirical_correctness":0.25,"signed_gap":0.281228207051754,"absolute_gap":0.281228207051754,"min_confidence":0.49624577164649963,"max_confidence":0.5854812264442444},{"index":3,"count":5,"confidence":0.6281886219978332,"empirical_correctness":0.2,"signed_gap":0.4281886219978332,"absolute_gap":0.4281886219978332,"min_confidence":0.6107557415962219,"max_confidence":0.6559061408042908},{"index":4,"count":4,"confidence":0.6962898522615433,"empirical_correctness":0.75,"signed_gap":-0.053710147738456726,"absolute_gap":0.053710147738456726,"min_confidence":0.6837688088417053,"max_confidence":0.7053565979003906},{"index":5,"count":4,"confidence":0.7394324094057083,"empirical_correctness":0.25,"signed_gap":0.4894324094057083,"absolute_gap":0.4894324094057083,"min_confidence":0.7093722820281982,"max_confidence":0.7699125409126282},{"index":6,"count":5,"confidence":0.8003295063972473,"empirical_correctness":0.4,"signed_gap":0.4003295063972473,"absolute_gap":0.4003295063972473,"min_confidence":0.7768857479095459,"max_confidence":0.8158666491508484},{"index":7,"count":4,"confidence":0.8286560475826263,"empirical_correctness":0.5,"signed_gap":0.32865604758262634,"absolute_gap":0.32865604758262634,"min_confidence":0.8162364959716797,"max_confidence":0.8384348154067993},{"index":8,"count":4,"confidence":0.8563021123409271,"empirical_correctness":0.75,"signed_gap":0.10630211234092712,"absolute_gap":0.10630211234092712,"min_confidence":0.8503928184509277,"max_confidence":0.8635236620903015},{"index":9,"count":5,"confidence":0.8773592710494995,"empirical_correctness":0.6,"signed_gap":0.27735927104949953,"absolute_gap":0.27735927104949953,"min_confidence":0.865590512752533,"max_confidence":0.905362069606781}]}},"calibration_singleton_labels":{"equal_width":{"n":38,"bin_count":10,"ece":0.32952760865813807,"signed_mean_gap":0.3245466649532318,"absolute_mean_gap":0.3245466649532318,"bins":[{"index":0,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.0,"upper":0.1,"upper_inclusive":false},{"index":1,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.1,"upper":0.2,"upper_inclusive":false},{"index":2,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.2,"upper":0.3,"upper_inclusive":false},{"index":3,"count":4,"confidence":0.3523872196674347,"empirical_correctness":0.25,"signed_gap":0.10238721966743469,"absolute_gap":0.10238721966743469,"lower":0.3,"upper":0.4,"upper_inclusive":false},{"index":4,"count":3,"confidence":0.4769073526064555,"empirical_correctness":0.0,"signed_gap":0.4769073526064555,"absolute_gap":0.4769073526064555,"lower":0.4,"upper":0.5,"upper_inclusive":false},{"index":5,"count":2,"confidence":0.521592915058136,"empirical_correctness":0.0,"signed_gap":0.521592915058136,"absolute_gap":0.521592915058136,"lower":0.5,"upper":0.6,"upper_inclusive":false},{"index":6,"count":7,"confidence":0.6455264176641192,"empirical_correctness":0.2857142857142857,"signed_gap":0.3598121319498335,"absolute_gap":0.3598121319498335,"lower":0.6,"upper":0.7,"upper_inclusive":false},{"index":7,"count":7,"confidence":0.7476837635040283,"empirical_correctness":0.42857142857142855,"signed_gap":0.3191123349325998,"absolute_gap":0.3191123349325998,"lower":0.7,"upper":0.8,"upper_inclusive":false},{"index":8,"count":14,"confidence":0.842248797416687,"empirical_correctness":0.5,"signed_gap":0.342248797416687,"absolute_gap":0.342248797416687,"lower":0.8,"upper":0.9,"upper_inclusive":false},{"index":9,"count":1,"confidence":0.905362069606781,"empirical_correctness":1.0,"signed_gap":-0.094637930393219,"absolute_gap":0.094637930393219,"lower":0.9,"upper":1.0,"upper_inclusive":true}]},"equal_mass":{"n":38,"bin_count":10,"ece":0.3245466649532318,"signed_mean_gap":0.3245466649532318,"absolute_mean_gap":0.3245466649532318,"bins":[{"index":0,"count":3,"confidence":0.3397239049275716,"empirical_correctness":0.3333333333333333,"signed_gap":0.006390571594238281,"absolute_gap":0.006390571594238281,"min_confidence":0.32169023156166077,"max_confidence":0.3756331205368042},{"index":1,"count":4,"confidence":0.4552748054265976,"empirical_correctness":0.0,"signed_gap":0.4552748054265976,"absolute_gap":0.4552748054265976,"min_confidence":0.3903771638870239,"max_confidence":0.4952767789363861},{"index":2,"count":4,"confidence":0.568121463060379,"empirical_correctness":0.0,"signed_gap":0.568121463060379,"absolute_gap":0.568121463060379,"min_confidence":0.5124710202217102,"max_confidence":0.6185442805290222},{"index":3,"count":4,"confidence":0.6488529741764069,"empirical_correctness":0.25,"signed_gap":0.39885297417640686,"absolute_gap":0.39885297417640686,"min_confidence":0.6218635439872742,"max_confidence":0.6837688088417053},{"index":4,"count":4,"confidence":0.7063567787408829,"empirical_correctness":0.5,"signed_gap":0.20635677874088287,"absolute_gap":0.20635677874088287,"min_confidence":0.6939730048179626,"max_confidence":0.71672523021698},{"index":5,"count":3,"confidence":0.7695059577624003,"empirical_correctness":0.3333333333333333,"signed_gap":0.43617262442906696,"absolute_gap":0.43617262442906696,"min_confidence":0.7617195844650269,"max_confidence":0.7768857479095459},{"index":6,"count":4,"confidence":0.8061904460191727,"empirical_correctness":0.5,"signed_gap":0.30619044601917267,"absolute_gap":0.30619044601917267,"min_confidence":0.7938143610954285,"max_confidence":0.8158666491508484},{"index":7,"count":4,"confidence":0.8286560475826263,"empirical_correctness":0.5,"signed_gap":0.32865604758262634,"absolute_gap":0.32865604758262634,"min_confidence":0.8162364959716797,"max_confidence":0.8384348154067993},{"index":8,"count":4,"confidence":0.8575169444084167,"empirical_correctness":0.75,"signed_gap":0.10751694440841675,"absolute_gap":0.10751694440841675,"min_confidence":0.8503928184509277,"max_confidence":0.865590512752533},{"index":9,"count":4,"confidence":0.8803014606237411,"empirical_correctness":0.5,"signed_gap":0.38030146062374115,"absolute_gap":0.38030146062374115,"min_confidence":0.8673065304756165,"max_confidence":0.905362069606781}]}},"latency_ms":{"n":43,"median":28.56951195280999,"p95":29.25867610028945},"per_action":{"keep":{"n":12,"raw":{"n":12,"acceptable":0,"unsafe":0,"review":0,"automated_count":12,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":2.7755575615628914e-17,"upper":0.24249400665524085,"confidence_level":0.95}},"effective":{"n":12,"acceptable":0,"unsafe":0,"review":3,"automated_count":9,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":3,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":0.25,"automated_coverage":0.75,"conditional_correctness":0.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.25,"accuracy_wilson_95":{"lower":2.7755575615628914e-17,"upper":0.24249400665524085,"confidence_level":0.95}}},"drop":{"n":16,"raw":{"n":16,"acceptable":9,"unsafe":5,"review":0,"automated_count":16,"automated_acceptable":9,"automated_unsafe":5,"unnecessary_reviews":0,"accuracy":0.5625,"unsafe_rate":0.3125,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.5625,"conditional_unsafe":0.3125,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.331785563988119,"upper":0.7690134759450765,"confidence_level":0.95}},"effective":{"n":16,"acceptable":9,"unsafe":3,"review":2,"automated_count":14,"automated_acceptable":9,"automated_unsafe":3,"unnecessary_reviews":2,"accuracy":0.5625,"unsafe_rate":0.1875,"review_rate":0.125,"automated_coverage":0.875,"conditional_correctness":0.6428571428571429,"conditional_unsafe":0.21428571428571427,"unnecessary_review_rate":0.125,"accuracy_wilson_95":{"lower":0.331785563988119,"upper":0.7690134759450765,"confidence_level":0.95}}},"mask":{"n":2,"raw":{"n":2,"acceptable":2,"unsafe":0,"review":0,"automated_count":2,"automated_acceptable":2,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":1.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.34238022750665303,"upper":1.0,"confidence_level":0.95}},"effective":{"n":2,"acceptable":2,"unsafe":0,"review":0,"automated_count":2,"automated_acceptable":2,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":1.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.34238022750665303,"upper":1.0,"confidence_level":0.95}}},"pseudonymize":{"n":2,"raw":{"n":2,"acceptable":2,"unsafe":0,"review":0,"automated_count":2,"automated_acceptable":2,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":1.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.34238022750665303,"upper":1.0,"confidence_level":0.95}},"effective":{"n":2,"acceptable":2,"unsafe":0,"review":0,"automated_count":2,"automated_acceptable":2,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":1.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.34238022750665303,"upper":1.0,"confidence_level":0.95}}},"generalize":{"n":6,"raw":{"n":6,"acceptable":1,"unsafe":0,"review":0,"automated_count":6,"automated_acceptable":1,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":0.16666666666666666,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.16666666666666666,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.030053369748306635,"upper":0.5635028221864702,"confidence_level":0.95}},"effective":{"n":6,"acceptable":0,"unsafe":0,"review":6,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":6,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":2.7755575615628914e-17,"upper":0.3903342879021653,"confidence_level":0.95}}},"review":{"n":0,"raw":{"n":0,"acceptable":0,"unsafe":0,"review":0,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":null,"unsafe_rate":null,"review_rate":null,"automated_coverage":null,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":null},"effective":{"n":0,"acceptable":0,"unsafe":0,"review":0,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":null,"unsafe_rate":null,"review_rate":null,"automated_coverage":null,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":null}}}},"diagnostic43-compact_no_samples":{"n":43,"raw":{"n":43,"acceptable":29,"unsafe":9,"review":0,"automated_count":43,"automated_acceptable":29,"automated_unsafe":9,"unnecessary_reviews":0,"accuracy":0.6744186046511628,"unsafe_rate":0.20930232558139536,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.6744186046511628,"conditional_unsafe":0.20930232558139536,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.5251621096752948,"upper":0.7950670287343129,"confidence_level":0.95}},"effective":{"n":43,"acceptable":24,"unsafe":3,"review":15,"automated_count":28,"automated_acceptable":23,"automated_unsafe":3,"unnecessary_reviews":14,"accuracy":0.5581395348837209,"unsafe_rate":0.06976744186046512,"review_rate":0.3488372093023256,"automated_coverage":0.6511627906976745,"conditional_correctness":0.8214285714285714,"conditional_unsafe":0.10714285714285714,"unnecessary_review_rate":0.32558139534883723,"accuracy_wilson_95":{"lower":0.41107614452824104,"upper":0.6956669016082949,"confidence_level":0.95}},"confusion":{"raw":{"keep":{"keep":11,"drop":0,"mask":1,"pseudonymize":0,"generalize":0,"review":0},"drop":{"keep":0,"drop":10,"mask":2,"pseudonymize":4,"generalize":0,"review":0},"mask":{"keep":0,"drop":0,"mask":2,"pseudonymize":0,"generalize":0,"review":0},"pseudonymize":{"keep":0,"drop":0,"mask":0,"pseudonymize":2,"generalize":0,"review":0},"generalize":{"keep":4,"drop":0,"mask":0,"pseudonymize":2,"generalize":0,"review":0},"review":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0}},"effective":{"keep":{"keep":7,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":5},"drop":{"keep":0,"drop":8,"mask":2,"pseudonymize":3,"generalize":0,"review":3},"mask":{"keep":0,"drop":0,"mask":2,"pseudonymize":0,"generalize":0,"review":0},"pseudonymize":{"keep":0,"drop":0,"mask":0,"pseudonymize":2,"generalize":0,"review":0},"generalize":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":6},"review":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0}}},"probability_metrics":{"singleton_n":38,"multiacceptable_n":5,"multiacceptable_coverage":0.11627906976744186,"nll":1.018192499804879,"nll_zero_mass_n":0,"brier":0.5299928987116406,"acceptable_set_n":43,"acceptable_set_nll":0.9276017550278084,"acceptable_set_zero_mass_n":0},"calibration_all_labels":{"equal_width":{"n":43,"bin_count":10,"ece":0.13100595598997072,"signed_mean_gap":-0.07036576922549757,"absolute_mean_gap":0.07036576922549757,"bins":[{"index":0,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.0,"upper":0.1,"upper_inclusive":false},{"index":1,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.1,"upper":0.2,"upper_inclusive":false},{"index":2,"count":1,"confidence":0.2800944745540619,"empirical_correctness":1.0,"signed_gap":-0.7199055254459381,"absolute_gap":0.7199055254459381,"lower":0.2,"upper":0.3,"upper_inclusive":false},{"index":3,"count":6,"confidence":0.3886518329381943,"empirical_correctness":0.3333333333333333,"signed_gap":0.05531849960486096,"absolute_gap":0.05531849960486096,"lower":0.3,"upper":0.4,"upper_inclusive":false},{"index":4,"count":6,"confidence":0.44957103828589123,"empirical_correctness":0.5,"signed_gap":-0.050428961714108766,"absolute_gap":0.050428961714108766,"lower":0.4,"upper":0.5,"upper_inclusive":false},{"index":5,"count":9,"confidence":0.5500360131263733,"empirical_correctness":0.7777777777777778,"signed_gap":-0.2277417646514045,"absolute_gap":0.2277417646514045,"lower":0.5,"upper":0.6,"upper_inclusive":false},{"index":6,"count":7,"confidence":0.6573336890765599,"empirical_correctness":0.7142857142857143,"signed_gap":-0.05695202520915443,"absolute_gap":0.05695202520915443,"lower":0.6,"upper":0.7,"upper_inclusive":false},{"index":7,"count":8,"confidence":0.7464816272258759,"empirical_correctness":0.625,"signed_gap":0.12148162722587585,"absolute_gap":0.12148162722587585,"lower":0.7,"upper":0.8,"upper_inclusive":false},{"index":8,"count":5,"confidence":0.8457222819328308,"empirical_correctness":1.0,"signed_gap":-0.15427771806716917,"absolute_gap":0.15427771806716917,"lower":0.8,"upper":0.9,"upper_inclusive":false},{"index":9,"count":1,"confidence":0.9127158522605896,"empirical_correctness":1.0,"signed_gap":-0.0872841477394104,"absolute_gap":0.0872841477394104,"lower":0.9,"upper":1.0,"upper_inclusive":true}]},"equal_mass":{"n":43,"bin_count":10,"ece":0.1088326774364294,"signed_mean_gap":-0.07036576922549757,"absolute_mean_gap":0.07036576922549757,"bins":[{"index":0,"count":4,"confidence":0.3562111109495163,"empirical_correctness":0.5,"signed_gap":-0.1437888890504837,"absolute_gap":0.1437888890504837,"min_confidence":0.2800944745540619,"max_confidence":0.38739535212516785},{"index":1,"count":4,"confidence":0.4000243544578552,"empirical_correctness":0.5,"signed_gap":-0.09997564554214478,"absolute_gap":0.09997564554214478,"min_confidence":0.39157426357269287,"max_confidence":0.41293638944625854},{"index":2,"count":4,"confidence":0.4469941034913063,"empirical_correctness":0.5,"signed_gap":-0.053005896508693695,"absolute_gap":0.053005896508693695,"min_confidence":0.4201076924800873,"max_confidence":0.4740343689918518},{"index":3,"count":5,"confidence":0.5157408356666565,"empirical_correctness":0.8,"signed_gap":-0.28425916433334353,"absolute_gap":0.28425916433334353,"min_confidence":0.4965134263038635,"max_confidence":0.5404967665672302},{"index":4,"count":4,"confidence":0.5676110088825226,"empirical_correctness":0.5,"signed_gap":0.06761100888252258,"absolute_gap":0.06761100888252258,"min_confidence":0.5528150796890259,"max_confidence":0.5841831564903259},{"index":5,"count":4,"confidence":0.6218896210193634,"empirical_correctness":0.75,"signed_gap":-0.1281103789806366,"absolute_gap":0.1281103789806366,"min_confidence":0.5976893305778503,"max_confidence":0.6415003538131714},{"index":6,"count":5,"confidence":0.6829626679420471,"empirical_correctness":0.6,"signed_gap":0.0829626679420471,"absolute_gap":0.0829626679420471,"min_confidence":0.6626022458076477,"max_confidence":0.7033466696739197},{"index":7,"count":4,"confidence":0.7339103370904922,"empirical_correctness":0.75,"signed_gap":-0.01608966290950775,"absolute_gap":0.01608966290950775,"min_confidence":0.7148964405059814,"max_confidence":0.7642317414283752},{"index":8,"count":4,"confidence":0.7854452878236771,"empirical_correctness":0.75,"signed_gap":0.03544528782367706,"absolute_gap":0.03544528782367706,"min_confidence":0.7663069367408752,"max_confidence":0.8089161515235901},{"index":9,"count":5,"confidence":0.8664822220802307,"empirical_correctness":1.0,"signed_gap":-0.13351777791976926,"absolute_gap":0.13351777791976926,"min_confidence":0.8238611817359924,"max_confidence":0.9127158522605896}]}},"calibration_singleton_labels":{"equal_width":{"n":38,"bin_count":10,"ece":0.1533996345181214,"signed_mean_gap":-0.05747539040289425,"absolute_mean_gap":0.05747539040289425,"bins":[{"index":0,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.0,"upper":0.1,"upper_inclusive":false},{"index":1,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.1,"upper":0.2,"upper_inclusive":false},{"index":2,"count":1,"confidence":0.2800944745540619,"empirical_correctness":1.0,"signed_gap":-0.7199055254459381,"absolute_gap":0.7199055254459381,"lower":0.2,"upper":0.3,"upper_inclusive":false},{"index":3,"count":6,"confidence":0.3886518329381943,"empirical_correctness":0.3333333333333333,"signed_gap":0.05531849960486096,"absolute_gap":0.05531849960486096,"lower":0.3,"upper":0.4,"upper_inclusive":false},{"index":4,"count":5,"confidence":0.4446783721446991,"empirical_correctness":0.6,"signed_gap":-0.1553216278553009,"absolute_gap":0.1553216278553009,"lower":0.4,"upper":0.5,"upper_inclusive":false},{"index":5,"count":8,"confidence":0.5512284189462662,"empirical_correctness":0.75,"signed_gap":-0.19877158105373383,"absolute_gap":0.19877158105373383,"lower":0.5,"upper":0.6,"upper_inclusive":false},{"index":6,"count":6,"confidence":0.6564555962880453,"empirical_correctness":0.6666666666666666,"signed_gap":-0.010211070378621345,"absolute_gap":0.010211070378621345,"lower":0.6,"upper":0.7,"upper_inclusive":false},{"index":7,"count":6,"confidence":0.748441606760025,"empirical_correctness":0.5,"signed_gap":0.24844160676002502,"absolute_gap":0.24844160676002502,"lower":0.7,"upper":0.8,"upper_inclusive":false},{"index":8,"count":5,"confidence":0.8457222819328308,"empirical_correctness":1.0,"signed_gap":-0.15427771806716917,"absolute_gap":0.15427771806716917,"lower":0.8,"upper":0.9,"upper_inclusive":false},{"index":9,"count":1,"confidence":0.9127158522605896,"empirical_correctness":1.0,"signed_gap":-0.0872841477394104,"absolute_gap":0.0872841477394104,"lower":0.9,"upper":1.0,"upper_inclusive":true}]},"equal_mass":{"n":38,"bin_count":10,"ece":0.16107683197448128,"signed_mean_gap":-0.05747539040289427,"absolute_mean_gap":0.05747539040289427,"bins":[{"index":0,"count":3,"confidence":0.34581636389096576,"empirical_correctness":0.6666666666666666,"signed_gap":-0.32085030277570087,"absolute_gap":0.32085030277570087,"min_confidence":0.2800944745540619,"max_confidence":0.381788432598114},{"index":1,"count":4,"confidence":0.39363909512758255,"empirical_correctness":0.25,"signed_gap":0.14363909512758255,"absolute_gap":0.14363909512758255,"min_confidence":0.38739535212516785,"max_confidence":0.3985971510410309},{"index":2,"count":4,"confidence":0.431719608604908,"empirical_correctness":0.75,"signed_gap":-0.318280391395092,"absolute_gap":0.318280391395092,"min_confidence":0.41293638944625854,"max_confidence":0.4638851583003998},{"index":3,"count":4,"confidence":0.5095518529415131,"empirical_correctness":0.75,"signed_gap":-0.24044814705848694,"absolute_gap":0.24044814705848694,"min_confidence":0.4965134263038635,"max_confidence":0.5265154242515564},{"index":4,"count":4,"confidence":0.5676110088825226,"empirical_correctness":0.5,"signed_gap":0.06761100888252258,"absolute_gap":0.06761100888252258,"min_confidence":0.5528150796890259,"max_confidence":0.5841831564903259},{"index":5,"count":3,"confidence":0.6153527100880941,"empirical_correctness":0.6666666666666666,"signed_gap":-0.05131395657857252,"absolute_gap":0.05131395657857252,"min_confidence":0.5976893305778503,"max_confidence":0.6248860359191895},{"index":6,"count":4,"confidence":0.6725911945104599,"empirical_correctness":0.75,"signed_gap":-0.0774088054895401,"absolute_gap":0.0774088054895401,"min_confidence":0.6415003538131714,"max_confidence":0.6944898962974548},{"index":7,"count":4,"confidence":0.7310228943824768,"empirical_correctness":0.5,"signed_gap":0.2310228943824768,"absolute_gap":0.2310228943824768,"min_confidence":0.7033466696739197,"max_confidence":0.7642317414283752},{"index":8,"count":4,"confidence":0.7998338490724564,"empirical_correctness":0.75,"signed_gap":0.04983384907245636,"absolute_gap":0.04983384907245636,"min_confidence":0.7831655144691467,"max_confidence":0.8238611817359924},{"index":9,"count":4,"confidence":0.8771374821662903,"empirical_correctness":1.0,"signed_gap":-0.12286251783370972,"absolute_gap":0.12286251783370972,"min_confidence":0.8464104533195496,"max_confidence":0.9127158522605896}]}},"latency_ms":{"n":43,"median":28.783681977074593,"p95":28.84263193118386},"per_action":{"keep":{"n":12,"raw":{"n":12,"acceptable":11,"unsafe":0,"review":0,"automated_count":12,"automated_acceptable":11,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":0.9166666666666666,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.9166666666666666,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.646120088858883,"upper":0.9851349055950829,"confidence_level":0.95}},"effective":{"n":12,"acceptable":7,"unsafe":0,"review":5,"automated_count":7,"automated_acceptable":7,"automated_unsafe":0,"unnecessary_reviews":5,"accuracy":0.5833333333333334,"unsafe_rate":0.0,"review_rate":0.4166666666666667,"automated_coverage":0.5833333333333334,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.4166666666666667,"accuracy_wilson_95":{"lower":0.31951131254954973,"upper":0.8067396863412435,"confidence_level":0.95}}},"drop":{"n":16,"raw":{"n":16,"acceptable":10,"unsafe":4,"review":0,"automated_count":16,"automated_acceptable":10,"automated_unsafe":4,"unnecessary_reviews":0,"accuracy":0.625,"unsafe_rate":0.25,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.625,"conditional_unsafe":0.25,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.38641040545502714,"upper":0.8151876744113636,"confidence_level":0.95}},"effective":{"n":16,"acceptable":8,"unsafe":3,"review":3,"automated_count":13,"automated_acceptable":8,"automated_unsafe":3,"unnecessary_reviews":3,"accuracy":0.5,"unsafe_rate":0.1875,"review_rate":0.1875,"automated_coverage":0.8125,"conditional_correctness":0.6153846153846154,"conditional_unsafe":0.23076923076923078,"unnecessary_review_rate":0.1875,"accuracy_wilson_95":{"lower":0.27999563610326017,"upper":0.7200043638967398,"confidence_level":0.95}}},"mask":{"n":2,"raw":{"n":2,"acceptable":2,"unsafe":0,"review":0,"automated_count":2,"automated_acceptable":2,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":1.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.34238022750665303,"upper":1.0,"confidence_level":0.95}},"effective":{"n":2,"acceptable":2,"unsafe":0,"review":0,"automated_count":2,"automated_acceptable":2,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":1.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.34238022750665303,"upper":1.0,"confidence_level":0.95}}},"pseudonymize":{"n":2,"raw":{"n":2,"acceptable":2,"unsafe":0,"review":0,"automated_count":2,"automated_acceptable":2,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":1.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.34238022750665303,"upper":1.0,"confidence_level":0.95}},"effective":{"n":2,"acceptable":2,"unsafe":0,"review":0,"automated_count":2,"automated_acceptable":2,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":1.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.34238022750665303,"upper":1.0,"confidence_level":0.95}}},"generalize":{"n":6,"raw":{"n":6,"acceptable":0,"unsafe":4,"review":0,"automated_count":6,"automated_acceptable":0,"automated_unsafe":4,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":0.6666666666666666,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":0.6666666666666666,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":2.7755575615628914e-17,"upper":0.3903342879021653,"confidence_level":0.95}},"effective":{"n":6,"acceptable":0,"unsafe":0,"review":6,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":6,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":2.7755575615628914e-17,"upper":0.3903342879021653,"confidence_level":0.95}}},"review":{"n":0,"raw":{"n":0,"acceptable":0,"unsafe":0,"review":0,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":null,"unsafe_rate":null,"review_rate":null,"automated_coverage":null,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":null},"effective":{"n":0,"acceptable":0,"unsafe":0,"review":0,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":null,"unsafe_rate":null,"review_rate":null,"automated_coverage":null,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":null}}}},"challenge56-no_samples":{"n":56,"raw":{"n":56,"acceptable":37,"unsafe":8,"review":2,"automated_count":54,"automated_acceptable":35,"automated_unsafe":8,"unnecessary_reviews":0,"accuracy":0.6607142857142857,"unsafe_rate":0.14285714285714285,"review_rate":0.03571428571428571,"automated_coverage":0.9642857142857143,"conditional_correctness":0.6481481481481481,"conditional_unsafe":0.14814814814814814,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.5299945300706023,"upper":0.770800275984081,"confidence_level":0.95}},"effective":{"n":56,"acceptable":26,"unsafe":0,"review":22,"automated_count":34,"automated_acceptable":24,"automated_unsafe":0,"unnecessary_reviews":20,"accuracy":0.4642857142857143,"unsafe_rate":0.0,"review_rate":0.39285714285714285,"automated_coverage":0.6071428571428571,"conditional_correctness":0.7058823529411765,"conditional_unsafe":0.0,"unnecessary_review_rate":0.35714285714285715,"accuracy_wilson_95":{"lower":0.34019852912223775,"upper":0.5929581806433883,"confidence_level":0.95}},"confusion":{"raw":{"keep":{"keep":8,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0},"drop":{"keep":0,"drop":11,"mask":11,"pseudonymize":0,"generalize":0,"review":0},"mask":{"keep":0,"drop":0,"mask":8,"pseudonymize":0,"generalize":0,"review":0},"pseudonymize":{"keep":0,"drop":0,"mask":0,"pseudonymize":8,"generalize":0,"review":0},"generalize":{"keep":8,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0},"review":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":2}},"effective":{"keep":{"keep":5,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":3},"drop":{"keep":0,"drop":11,"mask":10,"pseudonymize":0,"generalize":0,"review":1},"mask":{"keep":0,"drop":0,"mask":8,"pseudonymize":0,"generalize":0,"review":0},"pseudonymize":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":8},"generalize":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":8},"review":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":2}}},"probability_metrics":{"singleton_n":56,"multiacceptable_n":0,"multiacceptable_coverage":0.0,"nll":0.9883190611007676,"nll_zero_mass_n":0,"brier":0.5025003285364261,"acceptable_set_n":56,"acceptable_set_nll":0.9883190611007676,"acceptable_set_zero_mass_n":0},"calibration_all_labels":{"equal_width":{"n":56,"bin_count":10,"ece":0.15866069814988545,"signed_mean_gap":0.06188596040010453,"absolute_mean_gap":0.06188596040010453,"bins":[{"index":0,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.0,"upper":0.1,"upper_inclusive":false},{"index":1,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.1,"upper":0.2,"upper_inclusive":false},{"index":2,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.2,"upper":0.3,"upper_inclusive":false},{"index":3,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.3,"upper":0.4,"upper_inclusive":false},{"index":4,"count":6,"confidence":0.46436689297358197,"empirical_correctness":0.6666666666666666,"signed_gap":-0.20229977369308466,"absolute_gap":0.20229977369308466,"lower":0.4,"upper":0.5,"upper_inclusive":false},{"index":5,"count":8,"confidence":0.5339480713009834,"empirical_correctness":0.5,"signed_gap":0.03394807130098343,"absolute_gap":0.03394807130098343,"lower":0.5,"upper":0.6,"upper_inclusive":false},{"index":6,"count":8,"confidence":0.6361918821930885,"empirical_correctness":0.75,"signed_gap":-0.11380811780691147,"absolute_gap":0.11380811780691147,"lower":0.6,"upper":0.7,"upper_inclusive":false},{"index":7,"count":12,"confidence":0.7419768224159876,"empirical_correctness":0.25,"signed_gap":0.4919768224159876,"absolute_gap":0.4919768224159876,"lower":0.7,"upper":0.8,"upper_inclusive":false},{"index":8,"count":13,"confidence":0.8377288075593802,"empirical_correctness":0.8461538461538461,"signed_gap":-0.008425038594465972,"absolute_gap":0.008425038594465972,"lower":0.8,"upper":0.9,"upper_inclusive":false},{"index":9,"count":9,"confidence":0.9471218254831102,"empirical_correctness":1.0,"signed_gap":-0.05287817451688981,"absolute_gap":0.05287817451688981,"lower":0.9,"upper":1.0,"upper_inclusive":true}]},"equal_mass":{"n":56,"bin_count":10,"ece":0.1897646114230156,"signed_mean_gap":0.06188596040010454,"absolute_mean_gap":0.06188596040010454,"bins":[{"index":0,"count":5,"confidence":0.4583426833152771,"empirical_correctness":0.6,"signed_gap":-0.14165731668472287,"absolute_gap":0.14165731668472287,"min_confidence":0.40221303701400757,"max_confidence":0.4855169355869293},{"index":1,"count":6,"confidence":0.5142172078291575,"empirical_correctness":0.5,"signed_gap":0.014217207829157474,"absolute_gap":0.014217207829157474,"min_confidence":0.4944879412651062,"max_confidence":0.5397810935974121},{"index":2,"count":5,"confidence":0.5786940097808838,"empirical_correctness":0.8,"signed_gap":-0.22130599021911623,"absolute_gap":0.22130599021911623,"min_confidence":0.5410633087158203,"max_confidence":0.61060631275177},{"index":3,"count":6,"confidence":0.6461390455563863,"empirical_correctness":0.6666666666666666,"signed_gap":-0.02052762111028028,"absolute_gap":0.02052762111028028,"min_confidence":0.6183561086654663,"max_confidence":0.6765038967132568},{"index":4,"count":6,"confidence":0.7228083511193594,"empirical_correctness":0.16666666666666666,"signed_gap":0.5561416844526927,"absolute_gap":0.5561416844526927,"min_confidence":0.700154721736908,"max_confidence":0.7485248446464539},{"index":5,"count":5,"confidence":0.7582100629806519,"empirical_correctness":0.4,"signed_gap":0.35821006298065183,"absolute_gap":0.35821006298065183,"min_confidence":0.751663327217102,"max_confidence":0.7681053876876831},{"index":6,"count":6,"confidence":0.8055020570755005,"empirical_correctness":0.5,"signed_gap":0.3055020570755005,"absolute_gap":0.3055020570755005,"min_confidence":0.7758214473724365,"max_confidence":0.8278371691703796},{"index":7,"count":5,"confidence":0.8453255295753479,"empirical_correctness":1.0,"signed_gap":-0.1546744704246521,"absolute_gap":0.1546744704246521,"min_confidence":0.835033655166626,"max_confidence":0.8532711267471313},{"index":8,"count":6,"confidence":0.9024086793263754,"empirical_correctness":1.0,"signed_gap":-0.09759132067362464,"absolute_gap":0.09759132067362464,"min_confidence":0.863008975982666,"max_confidence":0.9444465041160583},{"index":9,"count":6,"confidence":0.952716718117396,"empirical_correctness":1.0,"signed_gap":-0.047283281882604,"absolute_gap":0.047283281882604,"min_confidence":0.9466617107391357,"max_confidence":0.9621486067771912}]}},"calibration_singleton_labels":{"equal_width":{"n":56,"bin_count":10,"ece":0.15866069814988545,"signed_mean_gap":0.06188596040010453,"absolute_mean_gap":0.06188596040010453,"bins":[{"index":0,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.0,"upper":0.1,"upper_inclusive":false},{"index":1,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.1,"upper":0.2,"upper_inclusive":false},{"index":2,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.2,"upper":0.3,"upper_inclusive":false},{"index":3,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.3,"upper":0.4,"upper_inclusive":false},{"index":4,"count":6,"confidence":0.46436689297358197,"empirical_correctness":0.6666666666666666,"signed_gap":-0.20229977369308466,"absolute_gap":0.20229977369308466,"lower":0.4,"upper":0.5,"upper_inclusive":false},{"index":5,"count":8,"confidence":0.5339480713009834,"empirical_correctness":0.5,"signed_gap":0.03394807130098343,"absolute_gap":0.03394807130098343,"lower":0.5,"upper":0.6,"upper_inclusive":false},{"index":6,"count":8,"confidence":0.6361918821930885,"empirical_correctness":0.75,"signed_gap":-0.11380811780691147,"absolute_gap":0.11380811780691147,"lower":0.6,"upper":0.7,"upper_inclusive":false},{"index":7,"count":12,"confidence":0.7419768224159876,"empirical_correctness":0.25,"signed_gap":0.4919768224159876,"absolute_gap":0.4919768224159876,"lower":0.7,"upper":0.8,"upper_inclusive":false},{"index":8,"count":13,"confidence":0.8377288075593802,"empirical_correctness":0.8461538461538461,"signed_gap":-0.008425038594465972,"absolute_gap":0.008425038594465972,"lower":0.8,"upper":0.9,"upper_inclusive":false},{"index":9,"count":9,"confidence":0.9471218254831102,"empirical_correctness":1.0,"signed_gap":-0.05287817451688981,"absolute_gap":0.05287817451688981,"lower":0.9,"upper":1.0,"upper_inclusive":true}]},"equal_mass":{"n":56,"bin_count":10,"ece":0.1897646114230156,"signed_mean_gap":0.06188596040010454,"absolute_mean_gap":0.06188596040010454,"bins":[{"index":0,"count":5,"confidence":0.4583426833152771,"empirical_correctness":0.6,"signed_gap":-0.14165731668472287,"absolute_gap":0.14165731668472287,"min_confidence":0.40221303701400757,"max_confidence":0.4855169355869293},{"index":1,"count":6,"confidence":0.5142172078291575,"empirical_correctness":0.5,"signed_gap":0.014217207829157474,"absolute_gap":0.014217207829157474,"min_confidence":0.4944879412651062,"max_confidence":0.5397810935974121},{"index":2,"count":5,"confidence":0.5786940097808838,"empirical_correctness":0.8,"signed_gap":-0.22130599021911623,"absolute_gap":0.22130599021911623,"min_confidence":0.5410633087158203,"max_confidence":0.61060631275177},{"index":3,"count":6,"confidence":0.6461390455563863,"empirical_correctness":0.6666666666666666,"signed_gap":-0.02052762111028028,"absolute_gap":0.02052762111028028,"min_confidence":0.6183561086654663,"max_confidence":0.6765038967132568},{"index":4,"count":6,"confidence":0.7228083511193594,"empirical_correctness":0.16666666666666666,"signed_gap":0.5561416844526927,"absolute_gap":0.5561416844526927,"min_confidence":0.700154721736908,"max_confidence":0.7485248446464539},{"index":5,"count":5,"confidence":0.7582100629806519,"empirical_correctness":0.4,"signed_gap":0.35821006298065183,"absolute_gap":0.35821006298065183,"min_confidence":0.751663327217102,"max_confidence":0.7681053876876831},{"index":6,"count":6,"confidence":0.8055020570755005,"empirical_correctness":0.5,"signed_gap":0.3055020570755005,"absolute_gap":0.3055020570755005,"min_confidence":0.7758214473724365,"max_confidence":0.8278371691703796},{"index":7,"count":5,"confidence":0.8453255295753479,"empirical_correctness":1.0,"signed_gap":-0.1546744704246521,"absolute_gap":0.1546744704246521,"min_confidence":0.835033655166626,"max_confidence":0.8532711267471313},{"index":8,"count":6,"confidence":0.9024086793263754,"empirical_correctness":1.0,"signed_gap":-0.09759132067362464,"absolute_gap":0.09759132067362464,"min_confidence":0.863008975982666,"max_confidence":0.9444465041160583},{"index":9,"count":6,"confidence":0.952716718117396,"empirical_correctness":1.0,"signed_gap":-0.047283281882604,"absolute_gap":0.047283281882604,"min_confidence":0.9466617107391357,"max_confidence":0.9621486067771912}]}},"latency_ms":{"n":56,"median":25.806155521422625,"p95":28.308474516961724},"per_action":{"keep":{"n":8,"raw":{"n":8,"acceptable":8,"unsafe":0,"review":0,"automated_count":8,"automated_acceptable":8,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":1.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.6755924351161198,"upper":1.0,"confidence_level":0.95}},"effective":{"n":8,"acceptable":5,"unsafe":0,"review":3,"automated_count":5,"automated_acceptable":5,"automated_unsafe":0,"unnecessary_reviews":3,"accuracy":0.625,"unsafe_rate":0.0,"review_rate":0.375,"automated_coverage":0.625,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.375,"accuracy_wilson_95":{"lower":0.30574239460262737,"upper":0.8631557141764026,"confidence_level":0.95}}},"drop":{"n":22,"raw":{"n":22,"acceptable":11,"unsafe":0,"review":0,"automated_count":22,"automated_acceptable":11,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":0.5,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.5,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.3072210627372502,"upper":0.6927789372627497,"confidence_level":0.95}},"effective":{"n":22,"acceptable":11,"unsafe":0,"review":1,"automated_count":21,"automated_acceptable":11,"automated_unsafe":0,"unnecessary_reviews":1,"accuracy":0.5,"unsafe_rate":0.0,"review_rate":0.045454545454545456,"automated_coverage":0.9545454545454546,"conditional_correctness":0.5238095238095238,"conditional_unsafe":0.0,"unnecessary_review_rate":0.045454545454545456,"accuracy_wilson_95":{"lower":0.3072210627372502,"upper":0.6927789372627497,"confidence_level":0.95}}},"mask":{"n":8,"raw":{"n":8,"acceptable":8,"unsafe":0,"review":0,"automated_count":8,"automated_acceptable":8,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":1.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.6755924351161198,"upper":1.0,"confidence_level":0.95}},"effective":{"n":8,"acceptable":8,"unsafe":0,"review":0,"automated_count":8,"automated_acceptable":8,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":1.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.6755924351161198,"upper":1.0,"confidence_level":0.95}}},"pseudonymize":{"n":8,"raw":{"n":8,"acceptable":8,"unsafe":0,"review":0,"automated_count":8,"automated_acceptable":8,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":1.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.6755924351161198,"upper":1.0,"confidence_level":0.95}},"effective":{"n":8,"acceptable":0,"unsafe":0,"review":8,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":8,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.32440756488388023,"confidence_level":0.95}}},"generalize":{"n":8,"raw":{"n":8,"acceptable":0,"unsafe":8,"review":0,"automated_count":8,"automated_acceptable":0,"automated_unsafe":8,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":1.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":1.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.32440756488388023,"confidence_level":0.95}},"effective":{"n":8,"acceptable":0,"unsafe":0,"review":8,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":8,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.32440756488388023,"confidence_level":0.95}}},"review":{"n":2,"raw":{"n":2,"acceptable":2,"unsafe":0,"review":2,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":1.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.34238022750665303,"upper":1.0,"confidence_level":0.95}},"effective":{"n":2,"acceptable":2,"unsafe":0,"review":2,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":1.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.34238022750665303,"upper":1.0,"confidence_level":0.95}}}}},"challenge56-compact_no_samples":{"n":56,"raw":{"n":56,"acceptable":32,"unsafe":24,"review":0,"automated_count":56,"automated_acceptable":32,"automated_unsafe":24,"unnecessary_reviews":0,"accuracy":0.5714285714285714,"unsafe_rate":0.42857142857142855,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.5714285714285714,"conditional_unsafe":0.42857142857142855,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.4413762073081303,"upper":0.6923103731606176,"confidence_level":0.95}},"effective":{"n":56,"acceptable":28,"unsafe":0,"review":30,"automated_count":26,"automated_acceptable":26,"automated_unsafe":0,"unnecessary_reviews":28,"accuracy":0.5,"unsafe_rate":0.0,"review_rate":0.5357142857142857,"automated_coverage":0.4642857142857143,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.5,"accuracy_wilson_95":{"lower":0.37331738801362335,"upper":0.6266826119863766,"confidence_level":0.95}},"confusion":{"raw":{"keep":{"keep":8,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0},"drop":{"keep":9,"drop":13,"mask":0,"pseudonymize":0,"generalize":0,"review":0},"mask":{"keep":0,"drop":0,"mask":8,"pseudonymize":0,"generalize":0,"review":0},"pseudonymize":{"keep":6,"drop":0,"mask":0,"pseudonymize":2,"generalize":0,"review":0},"generalize":{"keep":7,"drop":0,"mask":0,"pseudonymize":0,"generalize":1,"review":0},"review":{"keep":2,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0}},"effective":{"keep":{"keep":5,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":3},"drop":{"keep":0,"drop":12,"mask":0,"pseudonymize":0,"generalize":0,"review":10},"mask":{"keep":0,"drop":0,"mask":8,"pseudonymize":0,"generalize":0,"review":0},"pseudonymize":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":8},"generalize":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":1,"review":7},"review":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":2}}},"probability_metrics":{"singleton_n":56,"multiacceptable_n":0,"multiacceptable_coverage":0.0,"nll":0.9417865297702971,"nll_zero_mass_n":0,"brier":0.5417176536354577,"acceptable_set_n":56,"acceptable_set_nll":0.9417865297702971,"acceptable_set_zero_mass_n":0},"calibration_all_labels":{"equal_width":{"n":56,"bin_count":10,"ece":0.20867918218885148,"signed_mean_gap":0.17244469906602586,"absolute_mean_gap":0.17244469906602586,"bins":[{"index":0,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.0,"upper":0.1,"upper_inclusive":false},{"index":1,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.1,"upper":0.2,"upper_inclusive":false},{"index":2,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.2,"upper":0.3,"upper_inclusive":false},{"index":3,"count":1,"confidence":0.3578384518623352,"empirical_correctness":0.0,"signed_gap":0.3578384518623352,"absolute_gap":0.3578384518623352,"lower":0.3,"upper":0.4,"upper_inclusive":false},{"index":4,"count":5,"confidence":0.45188474655151367,"empirical_correctness":0.6,"signed_gap":-0.1481152534484863,"absolute_gap":0.1481152534484863,"lower":0.4,"upper":0.5,"upper_inclusive":false},{"index":5,"count":4,"confidence":0.5428381115198135,"empirical_correctness":0.25,"signed_gap":0.29283811151981354,"absolute_gap":0.29283811151981354,"lower":0.5,"upper":0.6,"upper_inclusive":false},{"index":6,"count":9,"confidence":0.6571731434928046,"empirical_correctness":0.2222222222222222,"signed_gap":0.43495092127058244,"absolute_gap":0.43495092127058244,"lower":0.6,"upper":0.7,"upper_inclusive":false},{"index":7,"count":11,"confidence":0.7351906245405023,"empirical_correctness":0.2727272727272727,"signed_gap":0.4624633518132296,"absolute_gap":0.4624633518132296,"lower":0.7,"upper":0.8,"upper_inclusive":false},{"index":8,"count":18,"confidence":0.8411457008785672,"empirical_correctness":0.8333333333333334,"signed_gap":0.007812367545233845,"absolute_gap":0.007812367545233845,"lower":0.8,"upper":0.9,"upper_inclusive":false},{"index":9,"count":8,"confidence":0.9657513424754143,"empirical_correctness":1.0,"signed_gap":-0.034248657524585724,"absolute_gap":0.034248657524585724,"lower":0.9,"upper":1.0,"upper_inclusive":true}]},"equal_mass":{"n":56,"bin_count":10,"ece":0.2584133691021374,"signed_mean_gap":0.1724446990660259,"absolute_mean_gap":0.1724446990660259,"bins":[{"index":0,"count":5,"confidence":0.425717693567276,"empirical_correctness":0.6,"signed_gap":-0.17428230643272397,"absolute_gap":0.17428230643272397,"min_confidence":0.3578384518623352,"max_confidence":0.45774927735328674},{"index":1,"count":6,"confidence":0.5435578574736913,"empirical_correctness":0.16666666666666666,"signed_gap":0.3768911908070247,"absolute_gap":0.3768911908070247,"min_confidence":0.48867371678352356,"max_confidence":0.6013209819793701},{"index":2,"count":5,"confidence":0.6478718876838684,"empirical_correctness":0.2,"signed_gap":0.4478718876838684,"absolute_gap":0.4478718876838684,"min_confidence":0.613499104976654,"max_confidence":0.6819421052932739},{"index":3,"count":6,"confidence":0.6983205179373423,"empirical_correctness":0.3333333333333333,"signed_gap":0.364987184604009,"absolute_gap":0.364987184604009,"min_confidence":0.6855185627937317,"max_confidence":0.7089816331863403},{"index":4,"count":6,"confidence":0.7336932023366293,"empirical_correctness":0.0,"signed_gap":0.7336932023366293,"absolute_gap":0.7336932023366293,"min_confidence":0.716028094291687,"max_confidence":0.7597385048866272},{"index":5,"count":5,"confidence":0.8006933569908142,"empirical_correctness":0.8,"signed_gap":0.0006933569908141868,"absolute_gap":0.0006933569908141868,"min_confidence":0.7826845645904541,"max_confidence":0.8184653520584106},{"index":6,"count":6,"confidence":0.8279617031415304,"empirical_correctness":0.6666666666666666,"signed_gap":0.16129503647486376,"absolute_gap":0.16129503647486376,"min_confidence":0.8195558190345764,"max_confidence":0.8399878144264221},{"index":7,"count":5,"confidence":0.8542581796646118,"empirical_correctness":1.0,"signed_gap":-0.14574182033538818,"absolute_gap":0.14574182033538818,"min_confidence":0.8500796556472778,"max_confidence":0.8588107228279114},{"index":8,"count":6,"confidence":0.8966828882694244,"empirical_correctness":1.0,"signed_gap":-0.10331711173057556,"absolute_gap":0.10331711173057556,"min_confidence":0.859541654586792,"max_confidence":0.9599233865737915},{"index":9,"count":6,"confidence":0.9688167572021484,"empirical_correctness":1.0,"signed_gap":-0.031183242797851562,"absolute_gap":0.031183242797851562,"min_confidence":0.9617803692817688,"max_confidence":0.9750701189041138}]}},"calibration_singleton_labels":{"equal_width":{"n":56,"bin_count":10,"ece":0.20867918218885148,"signed_mean_gap":0.17244469906602586,"absolute_mean_gap":0.17244469906602586,"bins":[{"index":0,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.0,"upper":0.1,"upper_inclusive":false},{"index":1,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.1,"upper":0.2,"upper_inclusive":false},{"index":2,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.2,"upper":0.3,"upper_inclusive":false},{"index":3,"count":1,"confidence":0.3578384518623352,"empirical_correctness":0.0,"signed_gap":0.3578384518623352,"absolute_gap":0.3578384518623352,"lower":0.3,"upper":0.4,"upper_inclusive":false},{"index":4,"count":5,"confidence":0.45188474655151367,"empirical_correctness":0.6,"signed_gap":-0.1481152534484863,"absolute_gap":0.1481152534484863,"lower":0.4,"upper":0.5,"upper_inclusive":false},{"index":5,"count":4,"confidence":0.5428381115198135,"empirical_correctness":0.25,"signed_gap":0.29283811151981354,"absolute_gap":0.29283811151981354,"lower":0.5,"upper":0.6,"upper_inclusive":false},{"index":6,"count":9,"confidence":0.6571731434928046,"empirical_correctness":0.2222222222222222,"signed_gap":0.43495092127058244,"absolute_gap":0.43495092127058244,"lower":0.6,"upper":0.7,"upper_inclusive":false},{"index":7,"count":11,"confidence":0.7351906245405023,"empirical_correctness":0.2727272727272727,"signed_gap":0.4624633518132296,"absolute_gap":0.4624633518132296,"lower":0.7,"upper":0.8,"upper_inclusive":false},{"index":8,"count":18,"confidence":0.8411457008785672,"empirical_correctness":0.8333333333333334,"signed_gap":0.007812367545233845,"absolute_gap":0.007812367545233845,"lower":0.8,"upper":0.9,"upper_inclusive":false},{"index":9,"count":8,"confidence":0.9657513424754143,"empirical_correctness":1.0,"signed_gap":-0.034248657524585724,"absolute_gap":0.034248657524585724,"lower":0.9,"upper":1.0,"upper_inclusive":true}]},"equal_mass":{"n":56,"bin_count":10,"ece":0.2584133691021374,"signed_mean_gap":0.1724446990660259,"absolute_mean_gap":0.1724446990660259,"bins":[{"index":0,"count":5,"confidence":0.425717693567276,"empirical_correctness":0.6,"signed_gap":-0.17428230643272397,"absolute_gap":0.17428230643272397,"min_confidence":0.3578384518623352,"max_confidence":0.45774927735328674},{"index":1,"count":6,"confidence":0.5435578574736913,"empirical_correctness":0.16666666666666666,"signed_gap":0.3768911908070247,"absolute_gap":0.3768911908070247,"min_confidence":0.48867371678352356,"max_confidence":0.6013209819793701},{"index":2,"count":5,"confidence":0.6478718876838684,"empirical_correctness":0.2,"signed_gap":0.4478718876838684,"absolute_gap":0.4478718876838684,"min_confidence":0.613499104976654,"max_confidence":0.6819421052932739},{"index":3,"count":6,"confidence":0.6983205179373423,"empirical_correctness":0.3333333333333333,"signed_gap":0.364987184604009,"absolute_gap":0.364987184604009,"min_confidence":0.6855185627937317,"max_confidence":0.7089816331863403},{"index":4,"count":6,"confidence":0.7336932023366293,"empirical_correctness":0.0,"signed_gap":0.7336932023366293,"absolute_gap":0.7336932023366293,"min_confidence":0.716028094291687,"max_confidence":0.7597385048866272},{"index":5,"count":5,"confidence":0.8006933569908142,"empirical_correctness":0.8,"signed_gap":0.0006933569908141868,"absolute_gap":0.0006933569908141868,"min_confidence":0.7826845645904541,"max_confidence":0.8184653520584106},{"index":6,"count":6,"confidence":0.8279617031415304,"empirical_correctness":0.6666666666666666,"signed_gap":0.16129503647486376,"absolute_gap":0.16129503647486376,"min_confidence":0.8195558190345764,"max_confidence":0.8399878144264221},{"index":7,"count":5,"confidence":0.8542581796646118,"empirical_correctness":1.0,"signed_gap":-0.14574182033538818,"absolute_gap":0.14574182033538818,"min_confidence":0.8500796556472778,"max_confidence":0.8588107228279114},{"index":8,"count":6,"confidence":0.8966828882694244,"empirical_correctness":1.0,"signed_gap":-0.10331711173057556,"absolute_gap":0.10331711173057556,"min_confidence":0.859541654586792,"max_confidence":0.9599233865737915},{"index":9,"count":6,"confidence":0.9688167572021484,"empirical_correctness":1.0,"signed_gap":-0.031183242797851562,"absolute_gap":0.031183242797851562,"min_confidence":0.9617803692817688,"max_confidence":0.9750701189041138}]}},"latency_ms":{"n":56,"median":28.414277505362406,"p95":28.98150697001256},"per_action":{"keep":{"n":8,"raw":{"n":8,"acceptable":8,"unsafe":0,"review":0,"automated_count":8,"automated_acceptable":8,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":1.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.6755924351161198,"upper":1.0,"confidence_level":0.95}},"effective":{"n":8,"acceptable":5,"unsafe":0,"review":3,"automated_count":5,"automated_acceptable":5,"automated_unsafe":0,"unnecessary_reviews":3,"accuracy":0.625,"unsafe_rate":0.0,"review_rate":0.375,"automated_coverage":0.625,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.375,"accuracy_wilson_95":{"lower":0.30574239460262737,"upper":0.8631557141764026,"confidence_level":0.95}}},"drop":{"n":22,"raw":{"n":22,"acceptable":13,"unsafe":9,"review":0,"automated_count":22,"automated_acceptable":13,"automated_unsafe":9,"unnecessary_reviews":0,"accuracy":0.5909090909090909,"unsafe_rate":0.4090909090909091,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.5909090909090909,"conditional_unsafe":0.4090909090909091,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.3873481847631513,"upper":0.7674418380352814,"confidence_level":0.95}},"effective":{"n":22,"acceptable":12,"unsafe":0,"review":10,"automated_count":12,"automated_acceptable":12,"automated_unsafe":0,"unnecessary_reviews":10,"accuracy":0.5454545454545454,"unsafe_rate":0.0,"review_rate":0.45454545454545453,"automated_coverage":0.5454545454545454,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.45454545454545453,"accuracy_wilson_95":{"lower":0.3465979532073494,"upper":0.7307970581918668,"confidence_level":0.95}}},"mask":{"n":8,"raw":{"n":8,"acceptable":8,"unsafe":0,"review":0,"automated_count":8,"automated_acceptable":8,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":1.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.6755924351161198,"upper":1.0,"confidence_level":0.95}},"effective":{"n":8,"acceptable":8,"unsafe":0,"review":0,"automated_count":8,"automated_acceptable":8,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":1.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.6755924351161198,"upper":1.0,"confidence_level":0.95}}},"pseudonymize":{"n":8,"raw":{"n":8,"acceptable":2,"unsafe":6,"review":0,"automated_count":8,"automated_acceptable":2,"automated_unsafe":6,"unnecessary_reviews":0,"accuracy":0.25,"unsafe_rate":0.75,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.25,"conditional_unsafe":0.75,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.071479212752109,"upper":0.5907245696898311,"confidence_level":0.95}},"effective":{"n":8,"acceptable":0,"unsafe":0,"review":8,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":8,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.32440756488388023,"confidence_level":0.95}}},"generalize":{"n":8,"raw":{"n":8,"acceptable":1,"unsafe":7,"review":0,"automated_count":8,"automated_acceptable":1,"automated_unsafe":7,"unnecessary_reviews":0,"accuracy":0.125,"unsafe_rate":0.875,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.125,"conditional_unsafe":0.875,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.02241749145005667,"upper":0.4708881822128535,"confidence_level":0.95}},"effective":{"n":8,"acceptable":1,"unsafe":0,"review":7,"automated_count":1,"automated_acceptable":1,"automated_unsafe":0,"unnecessary_reviews":7,"accuracy":0.125,"unsafe_rate":0.0,"review_rate":0.875,"automated_coverage":0.125,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.875,"accuracy_wilson_95":{"lower":0.02241749145005667,"upper":0.4708881822128535,"confidence_level":0.95}}},"review":{"n":2,"raw":{"n":2,"acceptable":0,"unsafe":2,"review":0,"automated_count":2,"automated_acceptable":0,"automated_unsafe":2,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":1.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":1.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.6576197724933469,"confidence_level":0.95}},"effective":{"n":2,"acceptable":2,"unsafe":0,"review":2,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":1.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.34238022750665303,"upper":1.0,"confidence_level":0.95}}}}}},"composite":{"n":198,"scope":"descriptive sum across four correlated panels","raw":{"n":198,"acceptable":116,"unsafe":47,"review":2,"automated_count":196,"automated_acceptable":114,"automated_unsafe":47,"unnecessary_reviews":0,"accuracy":0.5858585858585859,"unsafe_rate":0.23737373737373738,"review_rate":0.010101010101010102,"automated_coverage":0.98989898989899,"conditional_correctness":0.5816326530612245,"conditional_unsafe":0.23979591836734693,"unnecessary_review_rate":0.0},"effective":{"n":198,"acceptable":96,"unsafe":6,"review":79,"automated_count":119,"automated_acceptable":90,"automated_unsafe":6,"unnecessary_reviews":73,"accuracy":0.48484848484848486,"unsafe_rate":0.030303030303030304,"review_rate":0.398989898989899,"automated_coverage":0.601010101010101,"conditional_correctness":0.7563025210084033,"conditional_unsafe":0.05042016806722689,"unnecessary_review_rate":0.3686868686868687}}},{"id":"ghost-adapted-fp16","platform":"Local","model_id":"GhostScientist/jev-decisions-v1-model","revision":"d9f120f1bd44d606111e0041a4dd8dbefac33099","loading":"float16 / none","scoring_kind":"Adapted label scores","settings":{"dtype":"float16","quantization":"none","max_tokens":4096,"temperature":1.0,"truncation":false,"cpu_offload":false,"guard_threshold":0.45,"seed":0,"tf32":false},"scoring_metadata":{"backend":"ghost_letter_probability_adaptation","probability_source":"ADAPTED conditional label softmax T=1; publisher greedy temperature0 is NOT categorical probability contract","latency_scope":"synchronous full forward including input transfer and score readout; tokenization excluded","equal_mass_convention":"rank bins with floor boundaries at floor(i*n/k), k=min(10,n//2); minimum two rows"},"load_ms":null,"peak_allocated_bytes":1822531584,"peak_reserved_bytes":3034578944,"panels":{"diagnostic43-no_samples":{"n":43,"raw":{"n":43,"acceptable":10,"unsafe":11,"review":0,"automated_count":43,"automated_acceptable":10,"automated_unsafe":11,"unnecessary_reviews":0,"accuracy":0.23255813953488372,"unsafe_rate":0.2558139534883721,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.23255813953488372,"conditional_unsafe":0.2558139534883721,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.1315367847938448,"upper":0.37744520297809003,"confidence_level":0.95}},"effective":{"n":43,"acceptable":10,"unsafe":2,"review":23,"automated_count":20,"automated_acceptable":6,"automated_unsafe":2,"unnecessary_reviews":19,"accuracy":0.23255813953488372,"unsafe_rate":0.046511627906976744,"review_rate":0.5348837209302325,"automated_coverage":0.46511627906976744,"conditional_correctness":0.3,"conditional_unsafe":0.1,"unnecessary_review_rate":0.4418604651162791,"accuracy_wilson_95":{"lower":0.1315367847938448,"upper":0.37744520297809003,"confidence_level":0.95}},"confusion":{"raw":{"keep":{"keep":0,"drop":3,"mask":2,"pseudonymize":7,"generalize":0,"review":0},"drop":{"keep":0,"drop":7,"mask":3,"pseudonymize":6,"generalize":0,"review":0},"mask":{"keep":0,"drop":0,"mask":0,"pseudonymize":2,"generalize":0,"review":0},"pseudonymize":{"keep":0,"drop":0,"mask":1,"pseudonymize":1,"generalize":0,"review":0},"generalize":{"keep":0,"drop":3,"mask":1,"pseudonymize":2,"generalize":0,"review":0},"review":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0}},"effective":{"keep":{"keep":0,"drop":3,"mask":2,"pseudonymize":0,"generalize":0,"review":7},"drop":{"keep":0,"drop":4,"mask":3,"pseudonymize":2,"generalize":0,"review":7},"mask":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":2},"pseudonymize":{"keep":0,"drop":0,"mask":1,"pseudonymize":1,"generalize":0,"review":0},"generalize":{"keep":0,"drop":2,"mask":1,"pseudonymize":0,"generalize":0,"review":3},"review":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0}}},"probability_metrics":{"singleton_n":38,"multiacceptable_n":5,"multiacceptable_coverage":0.11627906976744186,"nll":2.9091865648130555,"nll_zero_mass_n":0,"brier":1.1946218831754505,"acceptable_set_n":43,"acceptable_set_nll":2.7142533791431434,"acceptable_set_zero_mass_n":0},"calibration_all_labels":{"equal_width":{"n":43,"bin_count":10,"ece":0.4590992567151092,"signed_mean_gap":0.4095939508704252,"absolute_mean_gap":0.4095939508704252,"bins":[{"index":0,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.0,"upper":0.1,"upper_inclusive":false},{"index":1,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.1,"upper":0.2,"upper_inclusive":false},{"index":2,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.2,"upper":0.3,"upper_inclusive":false},{"index":3,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.3,"upper":0.4,"upper_inclusive":false},{"index":4,"count":13,"confidence":0.45658737879533035,"empirical_correctness":0.5384615384615384,"signed_gap":-0.08187415966620809,"absolute_gap":0.08187415966620809,"lower":0.4,"upper":0.5,"upper_inclusive":false},{"index":5,"count":9,"confidence":0.5185331636004977,"empirical_correctness":0.2222222222222222,"signed_gap":0.2963109413782755,"absolute_gap":0.2963109413782755,"lower":0.5,"upper":0.6,"upper_inclusive":false},{"index":6,"count":1,"confidence":0.6962636113166809,"empirical_correctness":0.0,"signed_gap":0.6962636113166809,"absolute_gap":0.6962636113166809,"lower":0.6,"upper":0.7,"upper_inclusive":false},{"index":7,"count":11,"confidence":0.752804084257646,"empirical_correctness":0.0,"signed_gap":0.752804084257646,"absolute_gap":0.752804084257646,"lower":0.7,"upper":0.8,"upper_inclusive":false},{"index":8,"count":2,"confidence":0.8011903464794159,"empirical_correctness":0.0,"signed_gap":0.8011903464794159,"absolute_gap":0.8011903464794159,"lower":0.8,"upper":0.9,"upper_inclusive":false},{"index":9,"count":7,"confidence":0.9186594656535557,"empirical_correctness":0.14285714285714285,"signed_gap":0.7758023227964128,"absolute_gap":0.7758023227964128,"lower":0.9,"upper":1.0,"upper_inclusive":true}]},"equal_mass":{"n":43,"bin_count":10,"ece":0.47627385272536166,"signed_mean_gap":0.4095939508704252,"absolute_mean_gap":0.4095939508704252,"bins":[{"index":0,"count":4,"confidence":0.4369095489382744,"empirical_correctness":0.75,"signed_gap":-0.3130904510617256,"absolute_gap":0.3130904510617256,"min_confidence":0.43313857913017273,"max_confidence":0.4438575804233551},{"index":1,"count":4,"confidence":0.4546859785914421,"empirical_correctness":0.5,"signed_gap":-0.04531402140855789,"absolute_gap":0.04531402140855789,"min_confidence":0.4439994692802429,"max_confidence":0.4605490565299988},{"index":2,"count":4,"confidence":0.4706578701734543,"empirical_correctness":0.25,"signed_gap":0.22065787017345428,"absolute_gap":0.22065787017345428,"min_confidence":0.46073997020721436,"max_confidence":0.48344486951828003},{"index":3,"count":5,"confidence":0.5085788369178772,"empirical_correctness":0.4,"signed_gap":0.10857883691787718,"absolute_gap":0.10857883691787718,"min_confidence":0.48662233352661133,"max_confidence":0.5175841450691223},{"index":4,"count":4,"confidence":0.5216002315282822,"empirical_correctness":0.25,"signed_gap":0.27160023152828217,"absolute_gap":0.27160023152828217,"min_confidence":0.5191503763198853,"max_confidence":0.5238935947418213},{"index":5,"count":4,"confidence":0.6594237983226776,"empirical_correctness":0.0,"signed_gap":0.6594237983226776,"absolute_gap":0.6594237983226776,"min_confidence":0.5241256952285767,"max_confidence":0.7163318991661072},{"index":6,"count":5,"confidence":0.737715208530426,"empirical_correctness":0.0,"signed_gap":0.737715208530426,"absolute_gap":0.737715208530426,"min_confidence":0.7167637348175049,"max_confidence":0.7821816205978394},{"index":7,"count":4,"confidence":0.7937407493591309,"empirical_correctness":0.0,"signed_gap":0.7937407493591309,"absolute_gap":0.7937407493591309,"min_confidence":0.7905534505844116,"max_confidence":0.7969857454299927},{"index":8,"count":4,"confidence":0.854972779750824,"empirical_correctness":0.0,"signed_gap":0.854972779750824,"absolute_gap":0.854972779750824,"min_confidence":0.8001064658164978,"max_confidence":0.915843665599823},{"index":9,"count":5,"confidence":0.9226211667060852,"empirical_correctness":0.2,"signed_gap":0.7226211667060851,"absolute_gap":0.7226211667060851,"min_confidence":0.9178351163864136,"max_confidence":0.9304375052452087}]}},"calibration_singleton_labels":{"equal_width":{"n":38,"bin_count":10,"ece":0.42816456524949326,"signed_mean_gap":0.42816456524949326,"absolute_mean_gap":0.42816456524949326,"bins":[{"index":0,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.0,"upper":0.1,"upper_inclusive":false},{"index":1,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.1,"upper":0.2,"upper_inclusive":false},{"index":2,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.2,"upper":0.3,"upper_inclusive":false},{"index":3,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.3,"upper":0.4,"upper_inclusive":false},{"index":4,"count":11,"confidence":0.45709589394656097,"empirical_correctness":0.45454545454545453,"signed_gap":0.0025504394011064413,"absolute_gap":0.0025504394011064413,"lower":0.4,"upper":0.5,"upper_inclusive":false},{"index":5,"count":9,"confidence":0.5185331636004977,"empirical_correctness":0.2222222222222222,"signed_gap":0.2963109413782755,"absolute_gap":0.2963109413782755,"lower":0.5,"upper":0.6,"upper_inclusive":false},{"index":6,"count":1,"confidence":0.6962636113166809,"empirical_correctness":0.0,"signed_gap":0.6962636113166809,"absolute_gap":0.6962636113166809,"lower":0.6,"upper":0.7,"upper_inclusive":false},{"index":7,"count":10,"confidence":0.7564081192016602,"empirical_correctness":0.0,"signed_gap":0.7564081192016602,"absolute_gap":0.7564081192016602,"lower":0.7,"upper":0.8,"upper_inclusive":false},{"index":8,"count":1,"confidence":0.802274227142334,"empirical_correctness":0.0,"signed_gap":0.802274227142334,"absolute_gap":0.802274227142334,"lower":0.8,"upper":0.9,"upper_inclusive":false},{"index":9,"count":6,"confidence":0.9187968571980795,"empirical_correctness":0.16666666666666666,"signed_gap":0.7521301905314128,"absolute_gap":0.7521301905314128,"lower":0.9,"upper":1.0,"upper_inclusive":true}]},"equal_mass":{"n":38,"bin_count":10,"ece":0.4739464537093514,"signed_mean_gap":0.42816456524949326,"absolute_mean_gap":0.42816456524949326,"bins":[{"index":0,"count":3,"confidence":0.43713340163230896,"empirical_correctness":0.6666666666666666,"signed_gap":-0.22953326503435767,"absolute_gap":0.22953326503435767,"min_confidence":0.43313857913017273,"max_confidence":0.4438575804233551},{"index":1,"count":4,"confidence":0.4546859785914421,"empirical_correctness":0.5,"signed_gap":-0.04531402140855789,"absolute_gap":0.04531402140855789,"min_confidence":0.4439994692802429,"max_confidence":0.4605490565299988},{"index":2,"count":4,"confidence":0.4744776785373688,"empirical_correctness":0.25,"signed_gap":0.22447767853736877,"absolute_gap":0.22447767853736877,"min_confidence":0.46073997020721436,"max_confidence":0.48662233352661133},{"index":3,"count":4,"confidence":0.5140679627656937,"empirical_correctness":0.25,"signed_gap":0.26406796276569366,"absolute_gap":0.26406796276569366,"min_confidence":0.5077309608459473,"max_confidence":0.5175841450691223},{"index":4,"count":4,"confidence":0.5216002315282822,"empirical_correctness":0.25,"signed_gap":0.27160023152828217,"absolute_gap":0.27160023152828217,"min_confidence":0.5191503763198853,"max_confidence":0.5238935947418213},{"index":5,"count":3,"confidence":0.6404544313748678,"empirical_correctness":0.0,"signed_gap":0.6404544313748678,"absolute_gap":0.6404544313748678,"min_confidence":0.5241256952285767,"max_confidence":0.7009739875793457},{"index":6,"count":4,"confidence":0.7264906466007233,"empirical_correctness":0.0,"signed_gap":0.7264906466007233,"absolute_gap":0.7264906466007233,"min_confidence":0.7163318991661072,"max_confidence":0.7323832511901855},{"index":7,"count":4,"confidence":0.7900397181510925,"empirical_correctness":0.0,"signed_gap":0.7900397181510925,"absolute_gap":0.7900397181510925,"min_confidence":0.7821816205978394,"max_confidence":0.7940999269485474},{"index":8,"count":4,"confidence":0.8541925996541977,"empirical_correctness":0.0,"signed_gap":0.8541925996541977,"absolute_gap":0.8541925996541977,"min_confidence":0.7969857454299927,"max_confidence":0.915843665599823},{"index":9,"count":4,"confidence":0.9238176792860031,"empirical_correctness":0.25,"signed_gap":0.6738176792860031,"absolute_gap":0.6738176792860031,"min_confidence":0.9211474657058716,"max_confidence":0.9304375052452087}]}},"latency_ms":{"n":43,"median":97.60970016941428,"p95":121.57339004334062},"per_action":{"keep":{"n":12,"raw":{"n":12,"acceptable":0,"unsafe":0,"review":0,"automated_count":12,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":2.7755575615628914e-17,"upper":0.24249400665524085,"confidence_level":0.95}},"effective":{"n":12,"acceptable":0,"unsafe":0,"review":7,"automated_count":5,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":7,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":0.5833333333333334,"automated_coverage":0.4166666666666667,"conditional_correctness":0.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.5833333333333334,"accuracy_wilson_95":{"lower":2.7755575615628914e-17,"upper":0.24249400665524085,"confidence_level":0.95}}},"drop":{"n":16,"raw":{"n":16,"acceptable":7,"unsafe":6,"review":0,"automated_count":16,"automated_acceptable":7,"automated_unsafe":6,"unnecessary_reviews":0,"accuracy":0.4375,"unsafe_rate":0.375,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.4375,"conditional_unsafe":0.375,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.23098652405492354,"upper":0.6682144360118811,"confidence_level":0.95}},"effective":{"n":16,"acceptable":4,"unsafe":2,"review":7,"automated_count":9,"automated_acceptable":4,"automated_unsafe":2,"unnecessary_reviews":7,"accuracy":0.25,"unsafe_rate":0.125,"review_rate":0.4375,"automated_coverage":0.5625,"conditional_correctness":0.4444444444444444,"conditional_unsafe":0.2222222222222222,"unnecessary_review_rate":0.4375,"accuracy_wilson_95":{"lower":0.10182067491213048,"upper":0.49498316535508774,"confidence_level":0.95}}},"mask":{"n":2,"raw":{"n":2,"acceptable":0,"unsafe":2,"review":0,"automated_count":2,"automated_acceptable":0,"automated_unsafe":2,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":1.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":1.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.6576197724933469,"confidence_level":0.95}},"effective":{"n":2,"acceptable":0,"unsafe":0,"review":2,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":2,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.6576197724933469,"confidence_level":0.95}}},"pseudonymize":{"n":2,"raw":{"n":2,"acceptable":1,"unsafe":0,"review":0,"automated_count":2,"automated_acceptable":1,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":0.5,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.5,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.09453120573423074,"upper":0.9054687942657693,"confidence_level":0.95}},"effective":{"n":2,"acceptable":1,"unsafe":0,"review":0,"automated_count":2,"automated_acceptable":1,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":0.5,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.5,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.09453120573423074,"upper":0.9054687942657693,"confidence_level":0.95}}},"generalize":{"n":6,"raw":{"n":6,"acceptable":0,"unsafe":0,"review":0,"automated_count":6,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":2.7755575615628914e-17,"upper":0.3903342879021653,"confidence_level":0.95}},"effective":{"n":6,"acceptable":0,"unsafe":0,"review":3,"automated_count":3,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":3,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":0.5,"automated_coverage":0.5,"conditional_correctness":0.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.5,"accuracy_wilson_95":{"lower":2.7755575615628914e-17,"upper":0.3903342879021653,"confidence_level":0.95}}},"review":{"n":0,"raw":{"n":0,"acceptable":0,"unsafe":0,"review":0,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":null,"unsafe_rate":null,"review_rate":null,"automated_coverage":null,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":null},"effective":{"n":0,"acceptable":0,"unsafe":0,"review":0,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":null,"unsafe_rate":null,"review_rate":null,"automated_coverage":null,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":null}}}},"diagnostic43-compact_no_samples":{"n":43,"raw":{"n":43,"acceptable":11,"unsafe":10,"review":0,"automated_count":43,"automated_acceptable":11,"automated_unsafe":10,"unnecessary_reviews":0,"accuracy":0.2558139534883721,"unsafe_rate":0.23255813953488372,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.2558139534883721,"conditional_unsafe":0.23255813953488372,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.14929517123808245,"upper":0.40238403498846675,"confidence_level":0.95}},"effective":{"n":43,"acceptable":13,"unsafe":2,"review":19,"automated_count":24,"automated_acceptable":10,"automated_unsafe":2,"unnecessary_reviews":16,"accuracy":0.3023255813953488,"unsafe_rate":0.046511627906976744,"review_rate":0.4418604651162791,"automated_coverage":0.5581395348837209,"conditional_correctness":0.4166666666666667,"conditional_unsafe":0.08333333333333333,"unnecessary_review_rate":0.37209302325581395,"accuracy_wilson_95":{"lower":0.1860200013584818,"upper":0.45105364177729607,"confidence_level":0.95}},"confusion":{"raw":{"keep":{"keep":0,"drop":3,"mask":2,"pseudonymize":7,"generalize":0,"review":0},"drop":{"keep":0,"drop":7,"mask":3,"pseudonymize":6,"generalize":0,"review":0},"mask":{"keep":0,"drop":0,"mask":0,"pseudonymize":2,"generalize":0,"review":0},"pseudonymize":{"keep":0,"drop":0,"mask":1,"pseudonymize":1,"generalize":0,"review":0},"generalize":{"keep":0,"drop":3,"mask":1,"pseudonymize":2,"generalize":0,"review":0},"review":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0}},"effective":{"keep":{"keep":0,"drop":3,"mask":2,"pseudonymize":0,"generalize":0,"review":7},"drop":{"keep":0,"drop":7,"mask":2,"pseudonymize":2,"generalize":0,"review":5},"mask":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":2},"pseudonymize":{"keep":0,"drop":0,"mask":1,"pseudonymize":1,"generalize":0,"review":0},"generalize":{"keep":0,"drop":3,"mask":1,"pseudonymize":0,"generalize":0,"review":2},"review":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0}}},"probability_metrics":{"singleton_n":38,"multiacceptable_n":5,"multiacceptable_coverage":0.11627906976744186,"nll":2.6721374646293943,"nll_zero_mass_n":0,"brier":1.0405595327360544,"acceptable_set_n":43,"acceptable_set_nll":2.4621060407111623,"acceptable_set_zero_mass_n":0},"calibration_all_labels":{"equal_width":{"n":43,"bin_count":10,"ece":0.36221379526825837,"signed_mean_gap":0.36221379526825837,"absolute_mean_gap":0.36221379526825837,"bins":[{"index":0,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.0,"upper":0.1,"upper_inclusive":false},{"index":1,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.1,"upper":0.2,"upper_inclusive":false},{"index":2,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.2,"upper":0.3,"upper_inclusive":false},{"index":3,"count":1,"confidence":0.39938780665397644,"empirical_correctness":0.0,"signed_gap":0.39938780665397644,"absolute_gap":0.39938780665397644,"lower":0.3,"upper":0.4,"upper_inclusive":false},{"index":4,"count":12,"confidence":0.44748063882191974,"empirical_correctness":0.08333333333333333,"signed_gap":0.3641473054885864,"absolute_gap":0.3641473054885864,"lower":0.4,"upper":0.5,"upper_inclusive":false},{"index":5,"count":9,"confidence":0.5677584211031595,"empirical_correctness":0.1111111111111111,"signed_gap":0.45664730999204844,"absolute_gap":0.45664730999204844,"lower":0.5,"upper":0.6,"upper_inclusive":false},{"index":6,"count":7,"confidence":0.644172191619873,"empirical_correctness":0.5714285714285714,"signed_gap":0.07274362019130165,"absolute_gap":0.07274362019130165,"lower":0.6,"upper":0.7,"upper_inclusive":false},{"index":7,"count":7,"confidence":0.7259935651506696,"empirical_correctness":0.5714285714285714,"signed_gap":0.1545649937220982,"absolute_gap":0.1545649937220982,"lower":0.7,"upper":0.8,"upper_inclusive":false},{"index":8,"count":6,"confidence":0.8639034231503805,"empirical_correctness":0.16666666666666666,"signed_gap":0.6972367564837139,"absolute_gap":0.6972367564837139,"lower":0.8,"upper":0.9,"upper_inclusive":false},{"index":9,"count":1,"confidence":0.9216310977935791,"empirical_correctness":0.0,"signed_gap":0.9216310977935791,"absolute_gap":0.9216310977935791,"lower":0.9,"upper":1.0,"upper_inclusive":true}]},"equal_mass":{"n":43,"bin_count":10,"ece":0.36221379526825837,"signed_mean_gap":0.36221379526825837,"absolute_mean_gap":0.36221379526825837,"bins":[{"index":0,"count":4,"confidence":0.40974465757608414,"empirical_correctness":0.25,"signed_gap":0.15974465757608414,"absolute_gap":0.15974465757608414,"min_confidence":0.39938780665397644,"max_confidence":0.42064109444618225},{"index":1,"count":4,"confidence":0.4383739233016968,"empirical_correctness":0.0,"signed_gap":0.4383739233016968,"absolute_gap":0.4383739233016968,"min_confidence":0.42186489701271057,"max_confidence":0.46122321486473083},{"index":2,"count":4,"confidence":0.4723464921116829,"empirical_correctness":0.0,"signed_gap":0.4723464921116829,"absolute_gap":0.4723464921116829,"min_confidence":0.46443384885787964,"max_confidence":0.4816943109035492},{"index":3,"count":5,"confidence":0.5288011014461518,"empirical_correctness":0.0,"signed_gap":0.5288011014461518,"absolute_gap":0.5288011014461518,"min_confidence":0.4872951805591583,"max_confidence":0.5503191351890564},{"index":4,"count":4,"confidence":0.5883086025714874,"empirical_correctness":0.25,"signed_gap":0.3383086025714874,"absolute_gap":0.3383086025714874,"min_confidence":0.5648568868637085,"max_confidence":0.5997852087020874},{"index":5,"count":4,"confidence":0.6192685961723328,"empirical_correctness":0.5,"signed_gap":0.11926859617233276,"absolute_gap":0.11926859617233276,"min_confidence":0.5998810529708862,"max_confidence":0.6380338668823242},{"index":6,"count":5,"confidence":0.6671175003051758,"empirical_correctness":0.6,"signed_gap":0.06711750030517583,"absolute_gap":0.06711750030517583,"min_confidence":0.6494589447975159,"max_confidence":0.7035754919052124},{"index":7,"count":4,"confidence":0.7203742265701294,"empirical_correctness":0.5,"signed_gap":0.2203742265701294,"absolute_gap":0.2203742265701294,"min_confidence":0.7120617628097534,"max_confidence":0.729042112827301},{"index":8,"count":4,"confidence":0.7930146604776382,"empirical_correctness":0.25,"signed_gap":0.5430146604776382,"absolute_gap":0.5430146604776382,"min_confidence":0.7427778244018555,"max_confidence":0.8530886769294739},{"index":9,"count":5,"confidence":0.8859751105308533,"empirical_correctness":0.2,"signed_gap":0.6859751105308534,"absolute_gap":0.6859751105308534,"min_confidence":0.8563480377197266,"max_confidence":0.9216310977935791}]}},"calibration_singleton_labels":{"equal_width":{"n":38,"bin_count":10,"ece":0.407483005209973,"signed_mean_gap":0.407483005209973,"absolute_mean_gap":0.407483005209973,"bins":[{"index":0,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.0,"upper":0.1,"upper_inclusive":false},{"index":1,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.1,"upper":0.2,"upper_inclusive":false},{"index":2,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.2,"upper":0.3,"upper_inclusive":false},{"index":3,"count":1,"confidence":0.39938780665397644,"empirical_correctness":0.0,"signed_gap":0.39938780665397644,"absolute_gap":0.39938780665397644,"lower":0.3,"upper":0.4,"upper_inclusive":false},{"index":4,"count":11,"confidence":0.45159654996611853,"empirical_correctness":0.0,"signed_gap":0.45159654996611853,"absolute_gap":0.45159654996611853,"lower":0.4,"upper":0.5,"upper_inclusive":false},{"index":5,"count":7,"confidence":0.5714064495904105,"empirical_correctness":0.0,"signed_gap":0.5714064495904105,"absolute_gap":0.5714064495904105,"lower":0.5,"upper":0.6,"upper_inclusive":false},{"index":6,"count":6,"confidence":0.6420306166013082,"empirical_correctness":0.5,"signed_gap":0.14203061660130822,"absolute_gap":0.14203061660130822,"lower":0.6,"upper":0.7,"upper_inclusive":false},{"index":7,"count":7,"confidence":0.7259935651506696,"empirical_correctness":0.5714285714285714,"signed_gap":0.1545649937220982,"absolute_gap":0.1545649937220982,"lower":0.7,"upper":0.8,"upper_inclusive":false},{"index":8,"count":6,"confidence":0.8639034231503805,"empirical_correctness":0.16666666666666666,"signed_gap":0.6972367564837139,"absolute_gap":0.6972367564837139,"lower":0.8,"upper":0.9,"upper_inclusive":false},{"index":9,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.9,"upper":1.0,"upper_inclusive":true}]},"equal_mass":{"n":38,"bin_count":10,"ece":0.4139465140668969,"signed_mean_gap":0.407483005209973,"absolute_mean_gap":0.407483005209973,"bins":[{"index":0,"count":3,"confidence":0.41225767135620117,"empirical_correctness":0.0,"signed_gap":0.41225767135620117,"absolute_gap":0.41225767135620117,"min_confidence":0.39938780665397644,"max_confidence":0.42064109444618225},{"index":1,"count":4,"confidence":0.4383739233016968,"empirical_correctness":0.0,"signed_gap":0.4383739233016968,"absolute_gap":0.4383739233016968,"min_confidence":0.42186489701271057,"max_confidence":0.46122321486473083},{"index":2,"count":4,"confidence":0.4723464921116829,"empirical_correctness":0.0,"signed_gap":0.4723464921116829,"absolute_gap":0.4723464921116829,"min_confidence":0.46443384885787964,"max_confidence":0.4816943109035492},{"index":3,"count":4,"confidence":0.5247204378247261,"empirical_correctness":0.0,"signed_gap":0.5247204378247261,"absolute_gap":0.5247204378247261,"min_confidence":0.4872951805591583,"max_confidence":0.5503191351890564},{"index":4,"count":4,"confidence":0.5970646440982819,"empirical_correctness":0.0,"signed_gap":0.5970646440982819,"absolute_gap":0.5970646440982819,"min_confidence":0.5915042757987976,"max_confidence":0.5998810529708862},{"index":5,"count":3,"confidence":0.6257311105728149,"empirical_correctness":0.6666666666666666,"signed_gap":-0.04093555609385169,"absolute_gap":0.04093555609385169,"min_confidence":0.6054701209068298,"max_confidence":0.6380338668823242},{"index":6,"count":4,"confidence":0.6696414649486542,"empirical_correctness":0.5,"signed_gap":0.16964146494865417,"absolute_gap":0.16964146494865417,"min_confidence":0.6494589447975159,"max_confidence":0.7035754919052124},{"index":7,"count":4,"confidence":0.7203742265701294,"empirical_correctness":0.5,"signed_gap":0.2203742265701294,"absolute_gap":0.2203742265701294,"min_confidence":0.7120617628097534,"max_confidence":0.729042112827301},{"index":8,"count":4,"confidence":0.7930146604776382,"empirical_correctness":0.25,"signed_gap":0.5430146604776382,"absolute_gap":0.5430146604776382,"min_confidence":0.7427778244018555,"max_confidence":0.8530886769294739},{"index":9,"count":4,"confidence":0.8770611137151718,"empirical_correctness":0.25,"signed_gap":0.6270611137151718,"absolute_gap":0.6270611137151718,"min_confidence":0.8563480377197266,"max_confidence":0.8908900022506714}]}},"latency_ms":{"n":43,"median":107.14790015481412,"p95":129.61283985059708},"per_action":{"keep":{"n":12,"raw":{"n":12,"acceptable":0,"unsafe":0,"review":0,"automated_count":12,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":2.7755575615628914e-17,"upper":0.24249400665524085,"confidence_level":0.95}},"effective":{"n":12,"acceptable":0,"unsafe":0,"review":7,"automated_count":5,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":7,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":0.5833333333333334,"automated_coverage":0.4166666666666667,"conditional_correctness":0.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.5833333333333334,"accuracy_wilson_95":{"lower":2.7755575615628914e-17,"upper":0.24249400665524085,"confidence_level":0.95}}},"drop":{"n":16,"raw":{"n":16,"acceptable":7,"unsafe":6,"review":0,"automated_count":16,"automated_acceptable":7,"automated_unsafe":6,"unnecessary_reviews":0,"accuracy":0.4375,"unsafe_rate":0.375,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.4375,"conditional_unsafe":0.375,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.23098652405492354,"upper":0.6682144360118811,"confidence_level":0.95}},"effective":{"n":16,"acceptable":7,"unsafe":2,"review":5,"automated_count":11,"automated_acceptable":7,"automated_unsafe":2,"unnecessary_reviews":5,"accuracy":0.4375,"unsafe_rate":0.125,"review_rate":0.3125,"automated_coverage":0.6875,"conditional_correctness":0.6363636363636364,"conditional_unsafe":0.18181818181818182,"unnecessary_review_rate":0.3125,"accuracy_wilson_95":{"lower":0.23098652405492354,"upper":0.6682144360118811,"confidence_level":0.95}}},"mask":{"n":2,"raw":{"n":2,"acceptable":0,"unsafe":2,"review":0,"automated_count":2,"automated_acceptable":0,"automated_unsafe":2,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":1.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":1.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.6576197724933469,"confidence_level":0.95}},"effective":{"n":2,"acceptable":0,"unsafe":0,"review":2,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":2,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.6576197724933469,"confidence_level":0.95}}},"pseudonymize":{"n":2,"raw":{"n":2,"acceptable":1,"unsafe":0,"review":0,"automated_count":2,"automated_acceptable":1,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":0.5,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.5,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.09453120573423074,"upper":0.9054687942657693,"confidence_level":0.95}},"effective":{"n":2,"acceptable":1,"unsafe":0,"review":0,"automated_count":2,"automated_acceptable":1,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":0.5,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.5,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.09453120573423074,"upper":0.9054687942657693,"confidence_level":0.95}}},"generalize":{"n":6,"raw":{"n":6,"acceptable":0,"unsafe":0,"review":0,"automated_count":6,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":2.7755575615628914e-17,"upper":0.3903342879021653,"confidence_level":0.95}},"effective":{"n":6,"acceptable":0,"unsafe":0,"review":2,"automated_count":4,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":2,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":0.3333333333333333,"automated_coverage":0.6666666666666666,"conditional_correctness":0.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.3333333333333333,"accuracy_wilson_95":{"lower":2.7755575615628914e-17,"upper":0.3903342879021653,"confidence_level":0.95}}},"review":{"n":0,"raw":{"n":0,"acceptable":0,"unsafe":0,"review":0,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":null,"unsafe_rate":null,"review_rate":null,"automated_coverage":null,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":null},"effective":{"n":0,"acceptable":0,"unsafe":0,"review":0,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":null,"unsafe_rate":null,"review_rate":null,"automated_coverage":null,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":null}}}},"challenge56-no_samples":{"n":56,"raw":{"n":56,"acceptable":13,"unsafe":3,"review":0,"automated_count":56,"automated_acceptable":13,"automated_unsafe":3,"unnecessary_reviews":0,"accuracy":0.23214285714285715,"unsafe_rate":0.05357142857142857,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.23214285714285715,"conditional_unsafe":0.05357142857142857,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.1409937910920862,"upper":0.35768153215010856,"confidence_level":0.95}},"effective":{"n":56,"acceptable":10,"unsafe":0,"review":30,"automated_count":26,"automated_acceptable":10,"automated_unsafe":0,"unnecessary_reviews":30,"accuracy":0.17857142857142858,"unsafe_rate":0.0,"review_rate":0.5357142857142857,"automated_coverage":0.4642857142857143,"conditional_correctness":0.38461538461538464,"conditional_unsafe":0.0,"unnecessary_review_rate":0.5357142857142857,"accuracy_wilson_95":{"lower":0.09999856752412305,"upper":0.2984118203665106,"confidence_level":0.95}},"confusion":{"raw":{"keep":{"keep":0,"drop":6,"mask":0,"pseudonymize":2,"generalize":0,"review":0},"drop":{"keep":0,"drop":12,"mask":10,"pseudonymize":0,"generalize":0,"review":0},"mask":{"keep":3,"drop":5,"mask":0,"pseudonymize":0,"generalize":0,"review":0},"pseudonymize":{"keep":0,"drop":3,"mask":5,"pseudonymize":0,"generalize":0,"review":0},"generalize":{"keep":0,"drop":4,"mask":3,"pseudonymize":0,"generalize":1,"review":0},"review":{"keep":0,"drop":2,"mask":0,"pseudonymize":0,"generalize":0,"review":0}},"effective":{"keep":{"keep":0,"drop":5,"mask":0,"pseudonymize":0,"generalize":0,"review":3},"drop":{"keep":0,"drop":10,"mask":4,"pseudonymize":0,"generalize":0,"review":8},"mask":{"keep":0,"drop":5,"mask":0,"pseudonymize":0,"generalize":0,"review":3},"pseudonymize":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":8},"generalize":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":8},"review":{"keep":0,"drop":2,"mask":0,"pseudonymize":0,"generalize":0,"review":0}}},"probability_metrics":{"singleton_n":56,"multiacceptable_n":0,"multiacceptable_coverage":0.0,"nll":1.8999337069342643,"nll_zero_mass_n":0,"brier":0.8706510826946038,"acceptable_set_n":56,"acceptable_set_nll":1.8999337069342643,"acceptable_set_zero_mass_n":0},"calibration_all_labels":{"equal_width":{"n":56,"bin_count":10,"ece":0.29991353436240126,"signed_mean_gap":0.2891084362885782,"absolute_mean_gap":0.2891084362885782,"bins":[{"index":0,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.0,"upper":0.1,"upper_inclusive":false},{"index":1,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.1,"upper":0.2,"upper_inclusive":false},{"index":2,"count":5,"confidence":0.2668216735124588,"empirical_correctness":0.2,"signed_gap":0.06682167351245877,"absolute_gap":0.06682167351245877,"lower":0.2,"upper":0.3,"upper_inclusive":false},{"index":3,"count":12,"confidence":0.3519146740436554,"empirical_correctness":0.08333333333333333,"signed_gap":0.2685813407103221,"absolute_gap":0.2685813407103221,"lower":0.3,"upper":0.4,"upper_inclusive":false},{"index":4,"count":14,"confidence":0.4502741140978677,"empirical_correctness":0.07142857142857142,"signed_gap":0.37884554266929626,"absolute_gap":0.37884554266929626,"lower":0.4,"upper":0.5,"upper_inclusive":false},{"index":5,"count":12,"confidence":0.5286975105603536,"empirical_correctness":0.16666666666666666,"signed_gap":0.362030843893687,"absolute_gap":0.362030843893687,"lower":0.5,"upper":0.6,"upper_inclusive":false},{"index":6,"count":4,"confidence":0.6443253010511398,"empirical_correctness":0.0,"signed_gap":0.6443253010511398,"absolute_gap":0.6443253010511398,"lower":0.6,"upper":0.7,"upper_inclusive":false},{"index":7,"count":1,"confidence":0.7100217938423157,"empirical_correctness":0.0,"signed_gap":0.7100217938423157,"absolute_gap":0.7100217938423157,"lower":0.7,"upper":0.8,"upper_inclusive":false},{"index":8,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.8,"upper":0.9,"upper_inclusive":false},{"index":9,"count":8,"confidence":0.9621821567416191,"empirical_correctness":1.0,"signed_gap":-0.03781784325838089,"absolute_gap":0.03781784325838089,"lower":0.9,"upper":1.0,"upper_inclusive":true}]},"equal_mass":{"n":56,"bin_count":10,"ece":0.2967058827302286,"signed_mean_gap":0.2891084362885782,"absolute_mean_gap":0.2891084362885782,"bins":[{"index":0,"count":5,"confidence":0.2668216735124588,"empirical_correctness":0.2,"signed_gap":0.06682167351245877,"absolute_gap":0.06682167351245877,"min_confidence":0.2400607019662857,"max_confidence":0.297025203704834},{"index":1,"count":6,"confidence":0.3282810151576996,"empirical_correctness":0.16666666666666666,"signed_gap":0.16161434849103293,"absolute_gap":0.16161434849103293,"min_confidence":0.3003014326095581,"max_confidence":0.3518160581588745},{"index":2,"count":5,"confidence":0.3707720398902893,"empirical_correctness":0.0,"signed_gap":0.3707720398902893,"absolute_gap":0.3707720398902893,"min_confidence":0.35699203610420227,"max_confidence":0.38674238324165344},{"index":3,"count":6,"confidence":0.41261953115463257,"empirical_correctness":0.0,"signed_gap":0.41261953115463257,"absolute_gap":0.41261953115463257,"min_confidence":0.3994297981262207,"max_confidence":0.42699503898620605},{"index":4,"count":6,"confidence":0.4592045744260152,"empirical_correctness":0.16666666666666666,"signed_gap":0.2925379077593485,"absolute_gap":0.2925379077593485,"min_confidence":0.43463483452796936,"max_confidence":0.4835910201072693},{"index":5,"count":5,"confidence":0.49666881561279297,"empirical_correctness":0.2,"signed_gap":0.29666881561279296,"absolute_gap":0.29666881561279296,"min_confidence":0.4872243106365204,"max_confidence":0.5081062912940979},{"index":6,"count":6,"confidence":0.5181731383005778,"empirical_correctness":0.16666666666666666,"signed_gap":0.35150647163391113,"absolute_gap":0.35150647163391113,"min_confidence":0.5128776431083679,"max_confidence":0.5264645218849182},{"index":7,"count":5,"confidence":0.5652825236320496,"empirical_correctness":0.0,"signed_gap":0.5652825236320496,"absolute_gap":0.5652825236320496,"min_confidence":0.5462964177131653,"max_confidence":0.6021026372909546},{"index":8,"count":6,"confidence":0.7659010191758474,"empirical_correctness":0.3333333333333333,"signed_gap":0.4325676858425141,"absolute_gap":0.4325676858425141,"min_confidence":0.6086038947105408,"max_confidence":0.9555509686470032},{"index":9,"count":6,"confidence":0.9645452499389648,"empirical_correctness":1.0,"signed_gap":-0.035454750061035156,"absolute_gap":0.035454750061035156,"min_confidence":0.9565359354019165,"max_confidence":0.971866250038147}]}},"calibration_singleton_labels":{"equal_width":{"n":56,"bin_count":10,"ece":0.29991353436240126,"signed_mean_gap":0.2891084362885782,"absolute_mean_gap":0.2891084362885782,"bins":[{"index":0,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.0,"upper":0.1,"upper_inclusive":false},{"index":1,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.1,"upper":0.2,"upper_inclusive":false},{"index":2,"count":5,"confidence":0.2668216735124588,"empirical_correctness":0.2,"signed_gap":0.06682167351245877,"absolute_gap":0.06682167351245877,"lower":0.2,"upper":0.3,"upper_inclusive":false},{"index":3,"count":12,"confidence":0.3519146740436554,"empirical_correctness":0.08333333333333333,"signed_gap":0.2685813407103221,"absolute_gap":0.2685813407103221,"lower":0.3,"upper":0.4,"upper_inclusive":false},{"index":4,"count":14,"confidence":0.4502741140978677,"empirical_correctness":0.07142857142857142,"signed_gap":0.37884554266929626,"absolute_gap":0.37884554266929626,"lower":0.4,"upper":0.5,"upper_inclusive":false},{"index":5,"count":12,"confidence":0.5286975105603536,"empirical_correctness":0.16666666666666666,"signed_gap":0.362030843893687,"absolute_gap":0.362030843893687,"lower":0.5,"upper":0.6,"upper_inclusive":false},{"index":6,"count":4,"confidence":0.6443253010511398,"empirical_correctness":0.0,"signed_gap":0.6443253010511398,"absolute_gap":0.6443253010511398,"lower":0.6,"upper":0.7,"upper_inclusive":false},{"index":7,"count":1,"confidence":0.7100217938423157,"empirical_correctness":0.0,"signed_gap":0.7100217938423157,"absolute_gap":0.7100217938423157,"lower":0.7,"upper":0.8,"upper_inclusive":false},{"index":8,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.8,"upper":0.9,"upper_inclusive":false},{"index":9,"count":8,"confidence":0.9621821567416191,"empirical_correctness":1.0,"signed_gap":-0.03781784325838089,"absolute_gap":0.03781784325838089,"lower":0.9,"upper":1.0,"upper_inclusive":true}]},"equal_mass":{"n":56,"bin_count":10,"ece":0.2967058827302286,"signed_mean_gap":0.2891084362885782,"absolute_mean_gap":0.2891084362885782,"bins":[{"index":0,"count":5,"confidence":0.2668216735124588,"empirical_correctness":0.2,"signed_gap":0.06682167351245877,"absolute_gap":0.06682167351245877,"min_confidence":0.2400607019662857,"max_confidence":0.297025203704834},{"index":1,"count":6,"confidence":0.3282810151576996,"empirical_correctness":0.16666666666666666,"signed_gap":0.16161434849103293,"absolute_gap":0.16161434849103293,"min_confidence":0.3003014326095581,"max_confidence":0.3518160581588745},{"index":2,"count":5,"confidence":0.3707720398902893,"empirical_correctness":0.0,"signed_gap":0.3707720398902893,"absolute_gap":0.3707720398902893,"min_confidence":0.35699203610420227,"max_confidence":0.38674238324165344},{"index":3,"count":6,"confidence":0.41261953115463257,"empirical_correctness":0.0,"signed_gap":0.41261953115463257,"absolute_gap":0.41261953115463257,"min_confidence":0.3994297981262207,"max_confidence":0.42699503898620605},{"index":4,"count":6,"confidence":0.4592045744260152,"empirical_correctness":0.16666666666666666,"signed_gap":0.2925379077593485,"absolute_gap":0.2925379077593485,"min_confidence":0.43463483452796936,"max_confidence":0.4835910201072693},{"index":5,"count":5,"confidence":0.49666881561279297,"empirical_correctness":0.2,"signed_gap":0.29666881561279296,"absolute_gap":0.29666881561279296,"min_confidence":0.4872243106365204,"max_confidence":0.5081062912940979},{"index":6,"count":6,"confidence":0.5181731383005778,"empirical_correctness":0.16666666666666666,"signed_gap":0.35150647163391113,"absolute_gap":0.35150647163391113,"min_confidence":0.5128776431083679,"max_confidence":0.5264645218849182},{"index":7,"count":5,"confidence":0.5652825236320496,"empirical_correctness":0.0,"signed_gap":0.5652825236320496,"absolute_gap":0.5652825236320496,"min_confidence":0.5462964177131653,"max_confidence":0.6021026372909546},{"index":8,"count":6,"confidence":0.7659010191758474,"empirical_correctness":0.3333333333333333,"signed_gap":0.4325676858425141,"absolute_gap":0.4325676858425141,"min_confidence":0.6086038947105408,"max_confidence":0.9555509686470032},{"index":9,"count":6,"confidence":0.9645452499389648,"empirical_correctness":1.0,"signed_gap":-0.035454750061035156,"absolute_gap":0.035454750061035156,"min_confidence":0.9565359354019165,"max_confidence":0.971866250038147}]}},"latency_ms":{"n":56,"median":88.55939994100481,"p95":110.23305007256567},"per_action":{"keep":{"n":8,"raw":{"n":8,"acceptable":0,"unsafe":0,"review":0,"automated_count":8,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.32440756488388023,"confidence_level":0.95}},"effective":{"n":8,"acceptable":0,"unsafe":0,"review":3,"automated_count":5,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":3,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":0.375,"automated_coverage":0.625,"conditional_correctness":0.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.375,"accuracy_wilson_95":{"lower":0.0,"upper":0.32440756488388023,"confidence_level":0.95}}},"drop":{"n":22,"raw":{"n":22,"acceptable":12,"unsafe":0,"review":0,"automated_count":22,"automated_acceptable":12,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":0.5454545454545454,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.5454545454545454,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.3465979532073494,"upper":0.7307970581918668,"confidence_level":0.95}},"effective":{"n":22,"acceptable":10,"unsafe":0,"review":8,"automated_count":14,"automated_acceptable":10,"automated_unsafe":0,"unnecessary_reviews":8,"accuracy":0.45454545454545453,"unsafe_rate":0.0,"review_rate":0.36363636363636365,"automated_coverage":0.6363636363636364,"conditional_correctness":0.7142857142857143,"conditional_unsafe":0.0,"unnecessary_review_rate":0.36363636363636365,"accuracy_wilson_95":{"lower":0.2692029418081331,"upper":0.6534020467926505,"confidence_level":0.95}}},"mask":{"n":8,"raw":{"n":8,"acceptable":0,"unsafe":3,"review":0,"automated_count":8,"automated_acceptable":0,"automated_unsafe":3,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":0.375,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":0.375,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.32440756488388023,"confidence_level":0.95}},"effective":{"n":8,"acceptable":0,"unsafe":0,"review":3,"automated_count":5,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":3,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":0.375,"automated_coverage":0.625,"conditional_correctness":0.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.375,"accuracy_wilson_95":{"lower":0.0,"upper":0.32440756488388023,"confidence_level":0.95}}},"pseudonymize":{"n":8,"raw":{"n":8,"acceptable":0,"unsafe":0,"review":0,"automated_count":8,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.32440756488388023,"confidence_level":0.95}},"effective":{"n":8,"acceptable":0,"unsafe":0,"review":8,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":8,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.32440756488388023,"confidence_level":0.95}}},"generalize":{"n":8,"raw":{"n":8,"acceptable":1,"unsafe":0,"review":0,"automated_count":8,"automated_acceptable":1,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":0.125,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.125,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.02241749145005667,"upper":0.4708881822128535,"confidence_level":0.95}},"effective":{"n":8,"acceptable":0,"unsafe":0,"review":8,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":8,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.32440756488388023,"confidence_level":0.95}}},"review":{"n":2,"raw":{"n":2,"acceptable":0,"unsafe":0,"review":0,"automated_count":2,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.6576197724933469,"confidence_level":0.95}},"effective":{"n":2,"acceptable":0,"unsafe":0,"review":0,"automated_count":2,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.6576197724933469,"confidence_level":0.95}}}}},"challenge56-compact_no_samples":{"n":56,"raw":{"n":56,"acceptable":14,"unsafe":4,"review":0,"automated_count":56,"automated_acceptable":14,"automated_unsafe":4,"unnecessary_reviews":0,"accuracy":0.25,"unsafe_rate":0.07142857142857142,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.25,"conditional_unsafe":0.07142857142857142,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.15517054688270684,"upper":0.376926421476675,"confidence_level":0.95}},"effective":{"n":56,"acceptable":14,"unsafe":0,"review":20,"automated_count":36,"automated_acceptable":14,"automated_unsafe":0,"unnecessary_reviews":20,"accuracy":0.25,"unsafe_rate":0.0,"review_rate":0.35714285714285715,"automated_coverage":0.6428571428571429,"conditional_correctness":0.3888888888888889,"conditional_unsafe":0.0,"unnecessary_review_rate":0.35714285714285715,"accuracy_wilson_95":{"lower":0.15517054688270684,"upper":0.376926421476675,"confidence_level":0.95}},"confusion":{"raw":{"keep":{"keep":0,"drop":8,"mask":0,"pseudonymize":0,"generalize":0,"review":0},"drop":{"keep":0,"drop":14,"mask":8,"pseudonymize":0,"generalize":0,"review":0},"mask":{"keep":4,"drop":4,"mask":0,"pseudonymize":0,"generalize":0,"review":0},"pseudonymize":{"keep":0,"drop":7,"mask":1,"pseudonymize":0,"generalize":0,"review":0},"generalize":{"keep":0,"drop":8,"mask":0,"pseudonymize":0,"generalize":0,"review":0},"review":{"keep":0,"drop":2,"mask":0,"pseudonymize":0,"generalize":0,"review":0}},"effective":{"keep":{"keep":0,"drop":8,"mask":0,"pseudonymize":0,"generalize":0,"review":0},"drop":{"keep":0,"drop":14,"mask":1,"pseudonymize":0,"generalize":0,"review":7},"mask":{"keep":0,"drop":4,"mask":0,"pseudonymize":0,"generalize":0,"review":4},"pseudonymize":{"keep":0,"drop":5,"mask":0,"pseudonymize":0,"generalize":0,"review":3},"generalize":{"keep":0,"drop":2,"mask":0,"pseudonymize":0,"generalize":0,"review":6},"review":{"keep":0,"drop":2,"mask":0,"pseudonymize":0,"generalize":0,"review":0}}},"probability_metrics":{"singleton_n":56,"multiacceptable_n":0,"multiacceptable_coverage":0.0,"nll":1.9955215913325912,"nll_zero_mass_n":0,"brier":0.9024670681464668,"acceptable_set_n":56,"acceptable_set_nll":1.9955215913325912,"acceptable_set_zero_mass_n":0},"calibration_all_labels":{"equal_width":{"n":56,"bin_count":10,"ece":0.35414879822305273,"signed_mean_gap":0.3490610457956791,"absolute_mean_gap":0.3490610457956791,"bins":[{"index":0,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.0,"upper":0.1,"upper_inclusive":false},{"index":1,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.1,"upper":0.2,"upper_inclusive":false},{"index":2,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.2,"upper":0.3,"upper_inclusive":false},{"index":3,"count":11,"confidence":0.3742094039916992,"empirical_correctness":0.0,"signed_gap":0.3742094039916992,"absolute_gap":0.3742094039916992,"lower":0.3,"upper":0.4,"upper_inclusive":false},{"index":4,"count":15,"confidence":0.45134328802426654,"empirical_correctness":0.0,"signed_gap":0.45134328802426654,"absolute_gap":0.45134328802426654,"lower":0.4,"upper":0.5,"upper_inclusive":false},{"index":5,"count":7,"confidence":0.556661035333361,"empirical_correctness":0.2857142857142857,"signed_gap":0.2709467496190753,"absolute_gap":0.2709467496190753,"lower":0.5,"upper":0.6,"upper_inclusive":false},{"index":6,"count":7,"confidence":0.6541212286267962,"empirical_correctness":0.42857142857142855,"signed_gap":0.22554980005536768,"absolute_gap":0.22554980005536768,"lower":0.6,"upper":0.7,"upper_inclusive":false},{"index":7,"count":4,"confidence":0.7522743344306946,"empirical_correctness":0.25,"signed_gap":0.5022743344306946,"absolute_gap":0.5022743344306946,"lower":0.7,"upper":0.8,"upper_inclusive":false},{"index":8,"count":4,"confidence":0.8297124207019806,"empirical_correctness":0.0,"signed_gap":0.8297124207019806,"absolute_gap":0.8297124207019806,"lower":0.8,"upper":0.9,"upper_inclusive":false},{"index":9,"count":8,"confidence":0.9821928665041924,"empirical_correctness":1.0,"signed_gap":-0.017807133495807648,"absolute_gap":0.017807133495807648,"lower":0.9,"upper":1.0,"upper_inclusive":true}]},"equal_mass":{"n":56,"bin_count":10,"ece":0.352733988314867,"signed_mean_gap":0.3490610457956791,"absolute_mean_gap":0.3490610457956791,"bins":[{"index":0,"count":5,"confidence":0.3598038196563721,"empirical_correctness":0.0,"signed_gap":0.3598038196563721,"absolute_gap":0.3598038196563721,"min_confidence":0.351349800825119,"max_confidence":0.3686171770095825},{"index":1,"count":6,"confidence":0.38621405760447186,"empirical_correctness":0.0,"signed_gap":0.38621405760447186,"absolute_gap":0.38621405760447186,"min_confidence":0.3757701814174652,"max_confidence":0.39028966426849365},{"index":2,"count":5,"confidence":0.4142238020896912,"empirical_correctness":0.0,"signed_gap":0.4142238020896912,"absolute_gap":0.4142238020896912,"min_confidence":0.4002283215522766,"max_confidence":0.44683411717414856},{"index":3,"count":6,"confidence":0.46090452869733173,"empirical_correctness":0.0,"signed_gap":0.46090452869733173,"absolute_gap":0.46090452869733173,"min_confidence":0.45304229855537415,"max_confidence":0.46985501050949097},{"index":4,"count":6,"confidence":0.4977312634388606,"empirical_correctness":0.16666666666666666,"signed_gap":0.3310645967721939,"absolute_gap":0.3310645967721939,"min_confidence":0.47145965695381165,"max_confidence":0.5397228002548218},{"index":5,"count":5,"confidence":0.5687685608863831,"empirical_correctness":0.2,"signed_gap":0.36876856088638305,"absolute_gap":0.36876856088638305,"min_confidence":0.5589953064918518,"max_confidence":0.585946798324585},{"index":6,"count":6,"confidence":0.6488093733787537,"empirical_correctness":0.3333333333333333,"signed_gap":0.31547604004542035,"absolute_gap":0.31547604004542035,"min_confidence":0.6002270579338074,"max_confidence":0.682620108127594},{"index":7,"count":5,"confidence":0.7390179395675659,"empirical_correctness":0.4,"signed_gap":0.3390179395675659,"absolute_gap":0.3390179395675659,"min_confidence":0.6859923601150513,"max_confidence":0.7828909158706665},{"index":8,"count":6,"confidence":0.8798725008964539,"empirical_correctness":0.3333333333333333,"signed_gap":0.5465391675631206,"absolute_gap":0.5465391675631206,"min_confidence":0.8105234503746033,"max_confidence":0.9811062812805176},{"index":9,"count":6,"confidence":0.982859601577123,"empirical_correctness":1.0,"signed_gap":-0.017140398422876957,"absolute_gap":0.017140398422876957,"min_confidence":0.9812053442001343,"max_confidence":0.9841926097869873}]}},"calibration_singleton_labels":{"equal_width":{"n":56,"bin_count":10,"ece":0.35414879822305273,"signed_mean_gap":0.3490610457956791,"absolute_mean_gap":0.3490610457956791,"bins":[{"index":0,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.0,"upper":0.1,"upper_inclusive":false},{"index":1,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.1,"upper":0.2,"upper_inclusive":false},{"index":2,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.2,"upper":0.3,"upper_inclusive":false},{"index":3,"count":11,"confidence":0.3742094039916992,"empirical_correctness":0.0,"signed_gap":0.3742094039916992,"absolute_gap":0.3742094039916992,"lower":0.3,"upper":0.4,"upper_inclusive":false},{"index":4,"count":15,"confidence":0.45134328802426654,"empirical_correctness":0.0,"signed_gap":0.45134328802426654,"absolute_gap":0.45134328802426654,"lower":0.4,"upper":0.5,"upper_inclusive":false},{"index":5,"count":7,"confidence":0.556661035333361,"empirical_correctness":0.2857142857142857,"signed_gap":0.2709467496190753,"absolute_gap":0.2709467496190753,"lower":0.5,"upper":0.6,"upper_inclusive":false},{"index":6,"count":7,"confidence":0.6541212286267962,"empirical_correctness":0.42857142857142855,"signed_gap":0.22554980005536768,"absolute_gap":0.22554980005536768,"lower":0.6,"upper":0.7,"upper_inclusive":false},{"index":7,"count":4,"confidence":0.7522743344306946,"empirical_correctness":0.25,"signed_gap":0.5022743344306946,"absolute_gap":0.5022743344306946,"lower":0.7,"upper":0.8,"upper_inclusive":false},{"index":8,"count":4,"confidence":0.8297124207019806,"empirical_correctness":0.0,"signed_gap":0.8297124207019806,"absolute_gap":0.8297124207019806,"lower":0.8,"upper":0.9,"upper_inclusive":false},{"index":9,"count":8,"confidence":0.9821928665041924,"empirical_correctness":1.0,"signed_gap":-0.017807133495807648,"absolute_gap":0.017807133495807648,"lower":0.9,"upper":1.0,"upper_inclusive":true}]},"equal_mass":{"n":56,"bin_count":10,"ece":0.352733988314867,"signed_mean_gap":0.3490610457956791,"absolute_mean_gap":0.3490610457956791,"bins":[{"index":0,"count":5,"confidence":0.3598038196563721,"empirical_correctness":0.0,"signed_gap":0.3598038196563721,"absolute_gap":0.3598038196563721,"min_confidence":0.351349800825119,"max_confidence":0.3686171770095825},{"index":1,"count":6,"confidence":0.38621405760447186,"empirical_correctness":0.0,"signed_gap":0.38621405760447186,"absolute_gap":0.38621405760447186,"min_confidence":0.3757701814174652,"max_confidence":0.39028966426849365},{"index":2,"count":5,"confidence":0.4142238020896912,"empirical_correctness":0.0,"signed_gap":0.4142238020896912,"absolute_gap":0.4142238020896912,"min_confidence":0.4002283215522766,"max_confidence":0.44683411717414856},{"index":3,"count":6,"confidence":0.46090452869733173,"empirical_correctness":0.0,"signed_gap":0.46090452869733173,"absolute_gap":0.46090452869733173,"min_confidence":0.45304229855537415,"max_confidence":0.46985501050949097},{"index":4,"count":6,"confidence":0.4977312634388606,"empirical_correctness":0.16666666666666666,"signed_gap":0.3310645967721939,"absolute_gap":0.3310645967721939,"min_confidence":0.47145965695381165,"max_confidence":0.5397228002548218},{"index":5,"count":5,"confidence":0.5687685608863831,"empirical_correctness":0.2,"signed_gap":0.36876856088638305,"absolute_gap":0.36876856088638305,"min_confidence":0.5589953064918518,"max_confidence":0.585946798324585},{"index":6,"count":6,"confidence":0.6488093733787537,"empirical_correctness":0.3333333333333333,"signed_gap":0.31547604004542035,"absolute_gap":0.31547604004542035,"min_confidence":0.6002270579338074,"max_confidence":0.682620108127594},{"index":7,"count":5,"confidence":0.7390179395675659,"empirical_correctness":0.4,"signed_gap":0.3390179395675659,"absolute_gap":0.3390179395675659,"min_confidence":0.6859923601150513,"max_confidence":0.7828909158706665},{"index":8,"count":6,"confidence":0.8798725008964539,"empirical_correctness":0.3333333333333333,"signed_gap":0.5465391675631206,"absolute_gap":0.5465391675631206,"min_confidence":0.8105234503746033,"max_confidence":0.9811062812805176},{"index":9,"count":6,"confidence":0.982859601577123,"empirical_correctness":1.0,"signed_gap":-0.017140398422876957,"absolute_gap":0.017140398422876957,"min_confidence":0.9812053442001343,"max_confidence":0.9841926097869873}]}},"latency_ms":{"n":56,"median":96.92625002935529,"p95":119.15640003280714},"per_action":{"keep":{"n":8,"raw":{"n":8,"acceptable":0,"unsafe":0,"review":0,"automated_count":8,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.32440756488388023,"confidence_level":0.95}},"effective":{"n":8,"acceptable":0,"unsafe":0,"review":0,"automated_count":8,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.32440756488388023,"confidence_level":0.95}}},"drop":{"n":22,"raw":{"n":22,"acceptable":14,"unsafe":0,"review":0,"automated_count":22,"automated_acceptable":14,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":0.6363636363636364,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.6363636363636364,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.42951708090131013,"upper":0.8026679532963387,"confidence_level":0.95}},"effective":{"n":22,"acceptable":14,"unsafe":0,"review":7,"automated_count":15,"automated_acceptable":14,"automated_unsafe":0,"unnecessary_reviews":7,"accuracy":0.6363636363636364,"unsafe_rate":0.0,"review_rate":0.3181818181818182,"automated_coverage":0.6818181818181818,"conditional_correctness":0.9333333333333333,"conditional_unsafe":0.0,"unnecessary_review_rate":0.3181818181818182,"accuracy_wilson_95":{"lower":0.42951708090131013,"upper":0.8026679532963387,"confidence_level":0.95}}},"mask":{"n":8,"raw":{"n":8,"acceptable":0,"unsafe":4,"review":0,"automated_count":8,"automated_acceptable":0,"automated_unsafe":4,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":0.5,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":0.5,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.32440756488388023,"confidence_level":0.95}},"effective":{"n":8,"acceptable":0,"unsafe":0,"review":4,"automated_count":4,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":4,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":0.5,"automated_coverage":0.5,"conditional_correctness":0.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.5,"accuracy_wilson_95":{"lower":0.0,"upper":0.32440756488388023,"confidence_level":0.95}}},"pseudonymize":{"n":8,"raw":{"n":8,"acceptable":0,"unsafe":0,"review":0,"automated_count":8,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.32440756488388023,"confidence_level":0.95}},"effective":{"n":8,"acceptable":0,"unsafe":0,"review":3,"automated_count":5,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":3,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":0.375,"automated_coverage":0.625,"conditional_correctness":0.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.375,"accuracy_wilson_95":{"lower":0.0,"upper":0.32440756488388023,"confidence_level":0.95}}},"generalize":{"n":8,"raw":{"n":8,"acceptable":0,"unsafe":0,"review":0,"automated_count":8,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.32440756488388023,"confidence_level":0.95}},"effective":{"n":8,"acceptable":0,"unsafe":0,"review":6,"automated_count":2,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":6,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":0.75,"automated_coverage":0.25,"conditional_correctness":0.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.75,"accuracy_wilson_95":{"lower":0.0,"upper":0.32440756488388023,"confidence_level":0.95}}},"review":{"n":2,"raw":{"n":2,"acceptable":0,"unsafe":0,"review":0,"automated_count":2,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.6576197724933469,"confidence_level":0.95}},"effective":{"n":2,"acceptable":0,"unsafe":0,"review":0,"automated_count":2,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.6576197724933469,"confidence_level":0.95}}}}}},"composite":{"n":198,"scope":"descriptive sum across four correlated panels","raw":{"n":198,"acceptable":48,"unsafe":28,"review":0,"automated_count":198,"automated_acceptable":48,"automated_unsafe":28,"unnecessary_reviews":0,"accuracy":0.24242424242424243,"unsafe_rate":0.1414141414141414,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.24242424242424243,"conditional_unsafe":0.1414141414141414,"unnecessary_review_rate":0.0},"effective":{"n":198,"acceptable":47,"unsafe":4,"review":92,"automated_count":106,"automated_acceptable":40,"automated_unsafe":4,"unnecessary_reviews":85,"accuracy":0.23737373737373738,"unsafe_rate":0.020202020202020204,"review_rate":0.46464646464646464,"automated_coverage":0.5353535353535354,"conditional_correctness":0.37735849056603776,"conditional_unsafe":0.03773584905660377,"unnecessary_review_rate":0.4292929292929293}}},{"id":"laya-typed-fp32","platform":"Local","model_id":"convaiinnovations/laya-typed-decisions","revision":"1a793eb568e6718f15941d08f85432581df534e3","loading":"float32 / none","scoring_kind":"Native decision readout","settings":{"dtype":"float32","quantization":"none","max_tokens":1024,"head_max_tokens":256,"temperature":1.0000158548355103,"autocast":false,"truncation":false,"cpu_offload":false,"guard_threshold":0.45,"seed":0,"tf32":false},"scoring_metadata":{"backend":"laya_native_local_builtin","readout":"native_mask_slot_scorer_numpy_float32_softmax","probability_source":"native_choice_unrounded_published_temperature_not_task_calibrated","latency_scope":"synchronized serial complete encoder+head+act forward,input transfer and native readout; tokenization excluded","native_disclosures":["Stock installed torch attention in place of ONNX dynamic equivalent; no custom kernels or repo imports.","FP32 parameters and no autocast are explicit precision condition; saved F16 tensor conversion, no quantization.","Unrounded native probability; guard uses winner p, not entropy-concentration confidence.","Act head fully loaded and evaluated; not used to alter six-action answer.","Typed choice:6-10 bucket1.0000158548355103; soft fallback choice index0=1.0266985893249512; no fitting here."],"equal_mass_convention":"rank bins with floor boundaries at floor(i*n/k), k=min(10,n//2); minimum two rows"},"load_ms":null,"peak_allocated_bytes":1748760576,"peak_reserved_bytes":1870659584,"panels":{"diagnostic43-no_samples":{"n":43,"raw":{"n":43,"acceptable":13,"unsafe":23,"review":3,"automated_count":40,"automated_acceptable":12,"automated_unsafe":23,"unnecessary_reviews":2,"accuracy":0.3023255813953488,"unsafe_rate":0.5348837209302325,"review_rate":0.06976744186046512,"automated_coverage":0.9302325581395349,"conditional_correctness":0.3,"conditional_unsafe":0.575,"unnecessary_review_rate":0.046511627906976744,"accuracy_wilson_95":{"lower":0.1860200013584818,"upper":0.45105364177729607,"confidence_level":0.95}},"effective":{"n":43,"acceptable":5,"unsafe":0,"review":41,"automated_count":2,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":36,"accuracy":0.11627906976744186,"unsafe_rate":0.0,"review_rate":0.9534883720930233,"automated_coverage":0.046511627906976744,"conditional_correctness":0.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.8372093023255814,"accuracy_wilson_95":{"lower":0.05070427739804166,"upper":0.24479161810082128,"confidence_level":0.95}},"confusion":{"raw":{"keep":{"keep":9,"drop":0,"mask":3,"pseudonymize":0,"generalize":0,"review":0},"drop":{"keep":13,"drop":0,"mask":2,"pseudonymize":0,"generalize":0,"review":1},"mask":{"keep":1,"drop":0,"mask":1,"pseudonymize":0,"generalize":0,"review":0},"pseudonymize":{"keep":1,"drop":0,"mask":0,"pseudonymize":1,"generalize":0,"review":0},"generalize":{"keep":5,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":1},"review":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0}},"effective":{"keep":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":12},"drop":{"keep":0,"drop":0,"mask":2,"pseudonymize":0,"generalize":0,"review":14},"mask":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":2},"pseudonymize":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":2},"generalize":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":6},"review":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0}}},"probability_metrics":{"singleton_n":38,"multiacceptable_n":5,"multiacceptable_coverage":0.11627906976744186,"nll":1.7989330180470835,"nll_zero_mass_n":0,"brier":0.8303938150195039,"acceptable_set_n":43,"acceptable_set_nll":1.6743434880146937,"acceptable_set_zero_mass_n":0},"calibration_all_labels":{"equal_width":{"n":43,"bin_count":10,"ece":0.08532737819261331,"signed_mean_gap":-0.04179863423802134,"absolute_mean_gap":0.04179863423802134,"bins":[{"index":0,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.0,"upper":0.1,"upper_inclusive":false},{"index":1,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.1,"upper":0.2,"upper_inclusive":false},{"index":2,"count":40,"confidence":0.2566697683185339,"empirical_correctness":0.325,"signed_gap":-0.06833023168146612,"absolute_gap":0.06833023168146612,"lower":0.2,"upper":0.3,"upper_inclusive":false},{"index":3,"count":3,"confidence":0.3119559983412425,"empirical_correctness":0.0,"signed_gap":0.3119559983412425,"absolute_gap":0.3119559983412425,"lower":0.3,"upper":0.4,"upper_inclusive":false},{"index":4,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.4,"upper":0.5,"upper_inclusive":false},{"index":5,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.5,"upper":0.6,"upper_inclusive":false},{"index":6,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.6,"upper":0.7,"upper_inclusive":false},{"index":7,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.7,"upper":0.8,"upper_inclusive":false},{"index":8,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.8,"upper":0.9,"upper_inclusive":false},{"index":9,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.9,"upper":1.0,"upper_inclusive":true}]},"equal_mass":{"n":43,"bin_count":10,"ece":0.13000762635885282,"signed_mean_gap":-0.041798634238021326,"absolute_mean_gap":0.041798634238021326,"bins":[{"index":0,"count":4,"confidence":0.216038566082716,"empirical_correctness":0.25,"signed_gap":-0.03396143391728401,"absolute_gap":0.03396143391728401,"min_confidence":0.21457460522651672,"max_confidence":0.219246044754982},{"index":1,"count":4,"confidence":0.22723399475216866,"empirical_correctness":0.25,"signed_gap":-0.022766005247831345,"absolute_gap":0.022766005247831345,"min_confidence":0.22472792863845825,"max_confidence":0.23107974231243134},{"index":2,"count":4,"confidence":0.23306472599506378,"empirical_correctness":0.25,"signed_gap":-0.016935274004936218,"absolute_gap":0.016935274004936218,"min_confidence":0.23239243030548096,"max_confidence":0.23375284671783447},{"index":3,"count":5,"confidence":0.24302954375743865,"empirical_correctness":0.2,"signed_gap":0.04302954375743864,"absolute_gap":0.04302954375743864,"min_confidence":0.2350941300392151,"max_confidence":0.2521871030330658},{"index":4,"count":4,"confidence":0.25393105298280716,"empirical_correctness":0.25,"signed_gap":0.003931052982807159,"absolute_gap":0.003931052982807159,"min_confidence":0.253241628408432,"max_confidence":0.2545304298400879},{"index":5,"count":4,"confidence":0.2628161981701851,"empirical_correctness":0.5,"signed_gap":-0.2371838018298149,"absolute_gap":0.2371838018298149,"min_confidence":0.25509506464004517,"max_confidence":0.26803773641586304},{"index":6,"count":5,"confidence":0.27501959800720216,"empirical_correctness":0.4,"signed_gap":-0.12498040199279786,"absolute_gap":0.12498040199279786,"min_confidence":0.2731766104698181,"max_confidence":0.2763719856739044},{"index":7,"count":4,"confidence":0.282598540186882,"empirical_correctness":0.0,"signed_gap":0.282598540186882,"absolute_gap":0.282598540186882,"min_confidence":0.27667760848999023,"max_confidence":0.288049578666687},{"index":8,"count":4,"confidence":0.29361336678266525,"empirical_correctness":0.75,"signed_gap":-0.45638663321733475,"absolute_gap":0.45638663321733475,"min_confidence":0.2891256511211395,"max_confidence":0.2989068925380707},{"index":9,"count":5,"confidence":0.3070454478263855,"empirical_correctness":0.2,"signed_gap":0.10704544782638548,"absolute_gap":0.10704544782638548,"min_confidence":0.2994295656681061,"max_confidence":0.32313865423202515}]}},"calibration_singleton_labels":{"equal_width":{"n":38,"bin_count":10,"ece":0.07789213720120883,"signed_mean_gap":-0.028635926936802125,"absolute_mean_gap":0.028635926936802125,"bins":[{"index":0,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.0,"upper":0.1,"upper_inclusive":false},{"index":1,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.1,"upper":0.2,"upper_inclusive":false},{"index":2,"count":35,"confidence":0.2564561937536512,"empirical_correctness":0.3142857142857143,"signed_gap":-0.05782952053206308,"absolute_gap":0.05782952053206308,"lower":0.2,"upper":0.3,"upper_inclusive":false},{"index":3,"count":3,"confidence":0.3119559983412425,"empirical_correctness":0.0,"signed_gap":0.3119559983412425,"absolute_gap":0.3119559983412425,"lower":0.3,"upper":0.4,"upper_inclusive":false},{"index":4,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.4,"upper":0.5,"upper_inclusive":false},{"index":5,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.5,"upper":0.6,"upper_inclusive":false},{"index":6,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.6,"upper":0.7,"upper_inclusive":false},{"index":7,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.7,"upper":0.8,"upper_inclusive":false},{"index":8,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.8,"upper":0.9,"upper_inclusive":false},{"index":9,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.9,"upper":1.0,"upper_inclusive":true}]},"equal_mass":{"n":38,"bin_count":10,"ece":0.2873565209539313,"signed_mean_gap":-0.028635926936802104,"absolute_mean_gap":0.028635926936802104,"bins":[{"index":0,"count":3,"confidence":0.21633497873942056,"empirical_correctness":0.0,"signed_gap":0.21633497873942056,"absolute_gap":0.21633497873942056,"min_confidence":0.21457460522651672,"max_confidence":0.219246044754982},{"index":1,"count":4,"confidence":0.2290985770523548,"empirical_correctness":0.0,"signed_gap":0.2290985770523548,"absolute_gap":0.2290985770523548,"min_confidence":0.22472792863845825,"max_confidence":0.23239243030548096},{"index":2,"count":4,"confidence":0.23374015092849731,"empirical_correctness":0.5,"signed_gap":-0.2662598490715027,"absolute_gap":0.2662598490715027,"min_confidence":0.23276428878307343,"max_confidence":0.2350941300392151},{"index":3,"count":4,"confidence":0.24501339718699455,"empirical_correctness":0.0,"signed_gap":0.24501339718699455,"absolute_gap":0.24501339718699455,"min_confidence":0.23679396510124207,"max_confidence":0.2521871030330658},{"index":4,"count":4,"confidence":0.25393105298280716,"empirical_correctness":0.25,"signed_gap":0.003931052982807159,"absolute_gap":0.003931052982807159,"min_confidence":0.253241628408432,"max_confidence":0.2545304298400879},{"index":5,"count":3,"confidence":0.2610756854216258,"empirical_correctness":0.6666666666666666,"signed_gap":-0.40559098124504084,"absolute_gap":0.40559098124504084,"min_confidence":0.25509506464004517,"max_confidence":0.26517215371131897},{"index":6,"count":4,"confidence":0.2746815010905266,"empirical_correctness":0.5,"signed_gap":-0.22531849890947342,"absolute_gap":0.22531849890947342,"min_confidence":0.2731766104698181,"max_confidence":0.27602463960647583},{"index":7,"count":4,"confidence":0.27967914193868637,"empirical_correctness":0.0,"signed_gap":0.27967914193868637,"absolute_gap":0.27967914193868637,"min_confidence":0.2763719856739044,"max_confidence":0.28660741448402405},{"index":8,"count":4,"confidence":0.29480745643377304,"empirical_correctness":1.0,"signed_gap":-0.705192543566227,"absolute_gap":0.705192543566227,"min_confidence":0.2891256511211395,"max_confidence":0.2994295656681061},{"index":9,"count":4,"confidence":0.30894941836595535,"empirical_correctness":0.0,"signed_gap":0.30894941836595535,"absolute_gap":0.30894941836595535,"min_confidence":0.299929678440094,"max_confidence":0.32313865423202515}]}},"latency_ms":{"n":43,"median":18.125823000445962,"p95":19.284309080103412},"per_action":{"keep":{"n":12,"raw":{"n":12,"acceptable":9,"unsafe":0,"review":0,"automated_count":12,"automated_acceptable":9,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":0.75,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.75,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.46769466506643426,"upper":0.9110583316059453,"confidence_level":0.95}},"effective":{"n":12,"acceptable":0,"unsafe":0,"review":12,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":12,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":2.7755575615628914e-17,"upper":0.24249400665524085,"confidence_level":0.95}}},"drop":{"n":16,"raw":{"n":16,"acceptable":0,"unsafe":13,"review":1,"automated_count":15,"automated_acceptable":0,"automated_unsafe":13,"unnecessary_reviews":1,"accuracy":0.0,"unsafe_rate":0.8125,"review_rate":0.0625,"automated_coverage":0.9375,"conditional_correctness":0.0,"conditional_unsafe":0.8666666666666667,"unnecessary_review_rate":0.0625,"accuracy_wilson_95":{"lower":0.0,"upper":0.1936076805344365,"confidence_level":0.95}},"effective":{"n":16,"acceptable":0,"unsafe":0,"review":14,"automated_count":2,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":14,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":0.875,"automated_coverage":0.125,"conditional_correctness":0.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.875,"accuracy_wilson_95":{"lower":0.0,"upper":0.1936076805344365,"confidence_level":0.95}}},"mask":{"n":2,"raw":{"n":2,"acceptable":1,"unsafe":1,"review":0,"automated_count":2,"automated_acceptable":1,"automated_unsafe":1,"unnecessary_reviews":0,"accuracy":0.5,"unsafe_rate":0.5,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.5,"conditional_unsafe":0.5,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.09453120573423074,"upper":0.9054687942657693,"confidence_level":0.95}},"effective":{"n":2,"acceptable":0,"unsafe":0,"review":2,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":2,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.6576197724933469,"confidence_level":0.95}}},"pseudonymize":{"n":2,"raw":{"n":2,"acceptable":1,"unsafe":1,"review":0,"automated_count":2,"automated_acceptable":1,"automated_unsafe":1,"unnecessary_reviews":0,"accuracy":0.5,"unsafe_rate":0.5,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.5,"conditional_unsafe":0.5,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.09453120573423074,"upper":0.9054687942657693,"confidence_level":0.95}},"effective":{"n":2,"acceptable":0,"unsafe":0,"review":2,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":2,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.6576197724933469,"confidence_level":0.95}}},"generalize":{"n":6,"raw":{"n":6,"acceptable":0,"unsafe":5,"review":1,"automated_count":5,"automated_acceptable":0,"automated_unsafe":5,"unnecessary_reviews":1,"accuracy":0.0,"unsafe_rate":0.8333333333333334,"review_rate":0.16666666666666666,"automated_coverage":0.8333333333333334,"conditional_correctness":0.0,"conditional_unsafe":1.0,"unnecessary_review_rate":0.16666666666666666,"accuracy_wilson_95":{"lower":2.7755575615628914e-17,"upper":0.3903342879021653,"confidence_level":0.95}},"effective":{"n":6,"acceptable":0,"unsafe":0,"review":6,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":6,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":2.7755575615628914e-17,"upper":0.3903342879021653,"confidence_level":0.95}}},"review":{"n":0,"raw":{"n":0,"acceptable":0,"unsafe":0,"review":0,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":null,"unsafe_rate":null,"review_rate":null,"automated_coverage":null,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":null},"effective":{"n":0,"acceptable":0,"unsafe":0,"review":0,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":null,"unsafe_rate":null,"review_rate":null,"automated_coverage":null,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":null}}}},"diagnostic43-compact_no_samples":{"n":43,"raw":{"n":43,"acceptable":16,"unsafe":26,"review":1,"automated_count":42,"automated_acceptable":15,"automated_unsafe":26,"unnecessary_reviews":0,"accuracy":0.37209302325581395,"unsafe_rate":0.6046511627906976,"review_rate":0.023255813953488372,"automated_coverage":0.9767441860465116,"conditional_correctness":0.35714285714285715,"conditional_unsafe":0.6190476190476191,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.24376339009650572,"upper":0.5214019084031153,"confidence_level":0.95}},"effective":{"n":43,"acceptable":5,"unsafe":0,"review":39,"automated_count":4,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":34,"accuracy":0.11627906976744186,"unsafe_rate":0.0,"review_rate":0.9069767441860465,"automated_coverage":0.09302325581395349,"conditional_correctness":0.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.7906976744186046,"accuracy_wilson_95":{"lower":0.05070427739804166,"upper":0.24479161810082128,"confidence_level":0.95}},"confusion":{"raw":{"keep":{"keep":11,"drop":0,"mask":0,"pseudonymize":0,"generalize":1,"review":0},"drop":{"keep":16,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0},"mask":{"keep":0,"drop":0,"mask":2,"pseudonymize":0,"generalize":0,"review":0},"pseudonymize":{"keep":2,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0},"generalize":{"keep":4,"drop":0,"mask":0,"pseudonymize":0,"generalize":2,"review":0},"review":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0}},"effective":{"keep":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":12},"drop":{"keep":0,"drop":0,"mask":4,"pseudonymize":0,"generalize":0,"review":12},"mask":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":2},"pseudonymize":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":2},"generalize":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":6},"review":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0}}},"probability_metrics":{"singleton_n":38,"multiacceptable_n":5,"multiacceptable_coverage":0.11627906976744186,"nll":1.6834327811299037,"nll_zero_mass_n":0,"brier":0.7893349490452487,"acceptable_set_n":43,"acceptable_set_nll":1.565398103661593,"acceptable_set_zero_mass_n":0},"calibration_all_labels":{"equal_width":{"n":43,"bin_count":10,"ece":0.09770893773367238,"signed_mean_gap":-0.09770893773367238,"absolute_mean_gap":0.09770893773367238,"bins":[{"index":0,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.0,"upper":0.1,"upper_inclusive":false},{"index":1,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.1,"upper":0.2,"upper_inclusive":false},{"index":2,"count":37,"confidence":0.2642357373559797,"empirical_correctness":0.32432432432432434,"signed_gap":-0.060088586968344626,"absolute_gap":0.060088586968344626,"lower":0.2,"upper":0.3,"upper_inclusive":false},{"index":3,"count":6,"confidence":0.33696556588013965,"empirical_correctness":0.6666666666666666,"signed_gap":-0.329701100786527,"absolute_gap":0.329701100786527,"lower":0.3,"upper":0.4,"upper_inclusive":false},{"index":4,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.4,"upper":0.5,"upper_inclusive":false},{"index":5,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.5,"upper":0.6,"upper_inclusive":false},{"index":6,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.6,"upper":0.7,"upper_inclusive":false},{"index":7,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.7,"upper":0.8,"upper_inclusive":false},{"index":8,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.8,"upper":0.9,"upper_inclusive":false},{"index":9,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.9,"upper":1.0,"upper_inclusive":true}]},"equal_mass":{"n":43,"bin_count":10,"ece":0.18546577595001043,"signed_mean_gap":-0.09770893773367241,"absolute_mean_gap":0.09770893773367241,"bins":[{"index":0,"count":4,"confidence":0.23365075513720512,"empirical_correctness":0.5,"signed_gap":-0.2663492448627949,"absolute_gap":0.2663492448627949,"min_confidence":0.22967767715454102,"max_confidence":0.23945875465869904},{"index":1,"count":4,"confidence":0.24364622682332993,"empirical_correctness":0.5,"signed_gap":-0.2563537731766701,"absolute_gap":0.2563537731766701,"min_confidence":0.2418738752603531,"max_confidence":0.24505674839019775},{"index":2,"count":4,"confidence":0.2508857883512974,"empirical_correctness":0.25,"signed_gap":0.0008857883512973785,"absolute_gap":0.0008857883512973785,"min_confidence":0.24673889577388763,"max_confidence":0.25283873081207275},{"index":3,"count":5,"confidence":0.25548720359802246,"empirical_correctness":0.2,"signed_gap":0.05548720359802245,"absolute_gap":0.05548720359802245,"min_confidence":0.2544228732585907,"max_confidence":0.2568170428276062},{"index":4,"count":4,"confidence":0.2679465562105179,"empirical_correctness":0.25,"signed_gap":0.017946556210517883,"absolute_gap":0.017946556210517883,"min_confidence":0.2669439911842346,"max_confidence":0.2703724205493927},{"index":5,"count":4,"confidence":0.2717153802514076,"empirical_correctness":0.5,"signed_gap":-0.22828461974859238,"absolute_gap":0.22828461974859238,"min_confidence":0.2704392373561859,"max_confidence":0.27315157651901245},{"index":6,"count":5,"confidence":0.2754616141319275,"empirical_correctness":0.0,"signed_gap":0.2754616141319275,"absolute_gap":0.2754616141319275,"min_confidence":0.2731993496417999,"max_confidence":0.2779177129268646},{"index":7,"count":4,"confidence":0.2891746386885643,"empirical_correctness":0.25,"signed_gap":0.0391746386885643,"absolute_gap":0.0391746386885643,"min_confidence":0.2803385257720947,"max_confidence":0.2970096170902252},{"index":8,"count":4,"confidence":0.3007211908698082,"empirical_correctness":0.5,"signed_gap":-0.1992788091301918,"absolute_gap":0.1992788091301918,"min_confidence":0.29746580123901367,"max_confidence":0.3089839518070221},{"index":9,"count":5,"confidence":0.3425618886947632,"empirical_correctness":0.8,"signed_gap":-0.45743811130523687,"absolute_gap":0.45743811130523687,"min_confidence":0.31688761711120605,"max_confidence":0.36262282729148865}]}},"calibration_singleton_labels":{"equal_width":{"n":38,"bin_count":10,"ece":0.11774255688253202,"signed_mean_gap":-0.11774255688253202,"absolute_mean_gap":0.11774255688253202,"bins":[{"index":0,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.0,"upper":0.1,"upper_inclusive":false},{"index":1,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.1,"upper":0.2,"upper_inclusive":false},{"index":2,"count":32,"confidence":0.26574967009946704,"empirical_correctness":0.34375,"signed_gap":-0.07800032990053296,"absolute_gap":0.07800032990053296,"lower":0.2,"upper":0.3,"upper_inclusive":false},{"index":3,"count":6,"confidence":0.33696556588013965,"empirical_correctness":0.6666666666666666,"signed_gap":-0.329701100786527,"absolute_gap":0.329701100786527,"lower":0.3,"upper":0.4,"upper_inclusive":false},{"index":4,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.4,"upper":0.5,"upper_inclusive":false},{"index":5,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.5,"upper":0.6,"upper_inclusive":false},{"index":6,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.6,"upper":0.7,"upper_inclusive":false},{"index":7,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.7,"upper":0.8,"upper_inclusive":false},{"index":8,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.8,"upper":0.9,"upper_inclusive":false},{"index":9,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.9,"upper":1.0,"upper_inclusive":true}]},"equal_mass":{"n":38,"bin_count":10,"ece":0.19165276657593877,"signed_mean_gap":-0.11774255688253202,"absolute_mean_gap":0.11774255688253202,"bins":[{"index":0,"count":3,"confidence":0.2357801546653112,"empirical_correctness":0.3333333333333333,"signed_gap":-0.09755317866802213,"absolute_gap":0.09755317866802213,"min_confidence":0.23221829533576965,"max_confidence":0.2418738752603531},{"index":1,"count":4,"confidence":0.24486248195171356,"empirical_correctness":0.5,"signed_gap":-0.25513751804828644,"absolute_gap":0.25513751804828644,"min_confidence":0.24261315166950226,"max_confidence":0.24673889577388763},{"index":2,"count":4,"confidence":0.25280678272247314,"empirical_correctness":0.25,"signed_gap":0.0028067827224731445,"absolute_gap":0.0028067827224731445,"min_confidence":0.25129228830337524,"max_confidence":0.2544228732585907},{"index":3,"count":4,"confidence":0.2587108016014099,"empirical_correctness":0.25,"signed_gap":0.008710801601409912,"absolute_gap":0.008710801601409912,"min_confidence":0.2551446855068207,"max_confidence":0.2669439911842346},{"index":4,"count":4,"confidence":0.2688203677535057,"empirical_correctness":0.25,"signed_gap":0.018820367753505707,"absolute_gap":0.018820367753505707,"min_confidence":0.2670896649360657,"max_confidence":0.2704392373561859},{"index":5,"count":3,"confidence":0.272156685590744,"empirical_correctness":0.6666666666666666,"signed_gap":-0.3945099810759226,"absolute_gap":0.3945099810759226,"min_confidence":0.27087143063545227,"max_confidence":0.2731993496417999},{"index":6,"count":4,"confidence":0.2772790864109993,"empirical_correctness":0.25,"signed_gap":0.027279086410999298,"absolute_gap":0.027279086410999298,"min_confidence":0.2749217748641968,"max_confidence":0.2803385257720947},{"index":7,"count":4,"confidence":0.29345645755529404,"empirical_correctness":0.0,"signed_gap":0.29345645755529404,"absolute_gap":0.29345645755529404,"min_confidence":0.2842664122581482,"max_confidence":0.29746580123901367},{"index":8,"count":4,"confidence":0.3055766448378563,"empirical_correctness":0.5,"signed_gap":-0.1944233551621437,"absolute_gap":0.1944233551621437,"min_confidence":0.29814377427101135,"max_confidence":0.31688761711120605},{"index":9,"count":4,"confidence":0.34898045659065247,"empirical_correctness":1.0,"signed_gap":-0.6510195434093475,"absolute_gap":0.6510195434093475,"min_confidence":0.3179469108581543,"max_confidence":0.36262282729148865}]}},"latency_ms":{"n":43,"median":20.785522006917745,"p95":21.167922078166157},"per_action":{"keep":{"n":12,"raw":{"n":12,"acceptable":11,"unsafe":0,"review":0,"automated_count":12,"automated_acceptable":11,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":0.9166666666666666,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.9166666666666666,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.646120088858883,"upper":0.9851349055950829,"confidence_level":0.95}},"effective":{"n":12,"acceptable":0,"unsafe":0,"review":12,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":12,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":2.7755575615628914e-17,"upper":0.24249400665524085,"confidence_level":0.95}}},"drop":{"n":16,"raw":{"n":16,"acceptable":0,"unsafe":16,"review":0,"automated_count":16,"automated_acceptable":0,"automated_unsafe":16,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":1.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":1.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.1936076805344365,"confidence_level":0.95}},"effective":{"n":16,"acceptable":0,"unsafe":0,"review":12,"automated_count":4,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":12,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":0.75,"automated_coverage":0.25,"conditional_correctness":0.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.75,"accuracy_wilson_95":{"lower":0.0,"upper":0.1936076805344365,"confidence_level":0.95}}},"mask":{"n":2,"raw":{"n":2,"acceptable":2,"unsafe":0,"review":0,"automated_count":2,"automated_acceptable":2,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":1.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.34238022750665303,"upper":1.0,"confidence_level":0.95}},"effective":{"n":2,"acceptable":0,"unsafe":0,"review":2,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":2,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.6576197724933469,"confidence_level":0.95}}},"pseudonymize":{"n":2,"raw":{"n":2,"acceptable":0,"unsafe":2,"review":0,"automated_count":2,"automated_acceptable":0,"automated_unsafe":2,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":1.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":1.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.6576197724933469,"confidence_level":0.95}},"effective":{"n":2,"acceptable":0,"unsafe":0,"review":2,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":2,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.6576197724933469,"confidence_level":0.95}}},"generalize":{"n":6,"raw":{"n":6,"acceptable":2,"unsafe":4,"review":0,"automated_count":6,"automated_acceptable":2,"automated_unsafe":4,"unnecessary_reviews":0,"accuracy":0.3333333333333333,"unsafe_rate":0.6666666666666666,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.3333333333333333,"conditional_unsafe":0.6666666666666666,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.09677141110578041,"upper":0.700006684861608,"confidence_level":0.95}},"effective":{"n":6,"acceptable":0,"unsafe":0,"review":6,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":6,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":2.7755575615628914e-17,"upper":0.3903342879021653,"confidence_level":0.95}}},"review":{"n":0,"raw":{"n":0,"acceptable":0,"unsafe":0,"review":0,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":null,"unsafe_rate":null,"review_rate":null,"automated_coverage":null,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":null},"effective":{"n":0,"acceptable":0,"unsafe":0,"review":0,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":null,"unsafe_rate":null,"review_rate":null,"automated_coverage":null,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":null}}}},"challenge56-no_samples":{"n":56,"raw":{"n":56,"acceptable":16,"unsafe":39,"review":1,"automated_count":55,"automated_acceptable":16,"automated_unsafe":39,"unnecessary_reviews":1,"accuracy":0.2857142857142857,"unsafe_rate":0.6964285714285714,"review_rate":0.017857142857142856,"automated_coverage":0.9821428571428571,"conditional_correctness":0.2909090909090909,"conditional_unsafe":0.7090909090909091,"unnecessary_review_rate":0.017857142857142856,"accuracy_wilson_95":{"lower":0.18418775143858906,"upper":0.4147525071551667,"confidence_level":0.95}},"effective":{"n":56,"acceptable":7,"unsafe":0,"review":51,"automated_count":5,"automated_acceptable":5,"automated_unsafe":0,"unnecessary_reviews":49,"accuracy":0.125,"unsafe_rate":0.0,"review_rate":0.9107142857142857,"automated_coverage":0.08928571428571429,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.875,"accuracy_wilson_95":{"lower":0.06189068695717122,"upper":0.2362547655819014,"confidence_level":0.95}},"confusion":{"raw":{"keep":{"keep":8,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0},"drop":{"keep":21,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":1},"mask":{"keep":0,"drop":0,"mask":8,"pseudonymize":0,"generalize":0,"review":0},"pseudonymize":{"keep":8,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0},"generalize":{"keep":8,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0},"review":{"keep":2,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0}},"effective":{"keep":{"keep":3,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":5},"drop":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":22},"mask":{"keep":0,"drop":0,"mask":2,"pseudonymize":0,"generalize":0,"review":6},"pseudonymize":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":8},"generalize":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":8},"review":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":2}}},"probability_metrics":{"singleton_n":56,"multiacceptable_n":0,"multiacceptable_coverage":0.0,"nll":1.5510072271195017,"nll_zero_mass_n":0,"brier":0.732010733931295,"acceptable_set_n":56,"acceptable_set_nll":1.5510072271195017,"acceptable_set_zero_mass_n":0},"calibration_all_labels":{"equal_width":{"n":56,"bin_count":10,"ece":0.21705703836466586,"signed_mean_gap":0.032507271372846196,"absolute_mean_gap":0.032507271372846196,"bins":[{"index":0,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.0,"upper":0.1,"upper_inclusive":false},{"index":1,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.1,"upper":0.2,"upper_inclusive":false},{"index":2,"count":28,"confidence":0.24956430973751204,"empirical_correctness":0.0,"signed_gap":0.24956430973751204,"absolute_gap":0.24956430973751204,"lower":0.2,"upper":0.3,"upper_inclusive":false},{"index":3,"count":17,"confidence":0.34830797770444083,"empirical_correctness":0.35294117647058826,"signed_gap":-0.004633198766147428,"absolute_gap":0.004633198766147428,"lower":0.3,"upper":0.4,"upper_inclusive":false},{"index":4,"count":11,"confidence":0.4464882639321414,"empirical_correctness":0.9090909090909091,"signed_gap":-0.46260264515876764,"absolute_gap":0.46260264515876764,"lower":0.4,"upper":0.5,"upper_inclusive":false},{"index":5,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.5,"upper":0.6,"upper_inclusive":false},{"index":6,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.6,"upper":0.7,"upper_inclusive":false},{"index":7,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.7,"upper":0.8,"upper_inclusive":false},{"index":8,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.8,"upper":0.9,"upper_inclusive":false},{"index":9,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.9,"upper":1.0,"upper_inclusive":true}]},"equal_mass":{"n":56,"bin_count":10,"ece":0.2556725535541773,"signed_mean_gap":0.03250727137284619,"absolute_mean_gap":0.03250727137284619,"bins":[{"index":0,"count":5,"confidence":0.20784208476543425,"empirical_correctness":0.0,"signed_gap":0.20784208476543425,"absolute_gap":0.20784208476543425,"min_confidence":0.20523115992546082,"max_confidence":0.2096370905637741},{"index":1,"count":6,"confidence":0.22559662411610285,"empirical_correctness":0.0,"signed_gap":0.22559662411610285,"absolute_gap":0.22559662411610285,"min_confidence":0.21693119406700134,"max_confidence":0.2387024611234665},{"index":2,"count":5,"confidence":0.25356864035129545,"empirical_correctness":0.0,"signed_gap":0.25356864035129545,"absolute_gap":0.25356864035129545,"min_confidence":0.23900891840457916,"max_confidence":0.25897255539894104},{"index":3,"count":6,"confidence":0.26568761964639026,"empirical_correctness":0.0,"signed_gap":0.26568761964639026,"absolute_gap":0.26568761964639026,"min_confidence":0.25927865505218506,"max_confidence":0.2725227177143097},{"index":4,"count":6,"confidence":0.28884026408195496,"empirical_correctness":0.0,"signed_gap":0.28884026408195496,"absolute_gap":0.28884026408195496,"min_confidence":0.27446964383125305,"max_confidence":0.2998443841934204},{"index":5,"count":5,"confidence":0.30944548845291137,"empirical_correctness":0.4,"signed_gap":-0.09055451154708866,"absolute_gap":0.09055451154708866,"min_confidence":0.30249398946762085,"max_confidence":0.3193577229976654},{"index":6,"count":6,"confidence":0.34687240421772003,"empirical_correctness":0.16666666666666666,"signed_gap":0.18020573755105337,"absolute_gap":0.18020573755105337,"min_confidence":0.32922646403312683,"max_confidence":0.3644832372665405},{"index":7,"count":5,"confidence":0.37895127534866335,"empirical_correctness":0.4,"signed_gap":-0.02104872465133667,"absolute_gap":0.02104872465133667,"min_confidence":0.37122467160224915,"max_confidence":0.3920534551143646},{"index":8,"count":6,"confidence":0.4156064639488856,"empirical_correctness":0.8333333333333334,"signed_gap":-0.41772686938444775,"absolute_gap":0.41772686938444775,"min_confidence":0.39801737666130066,"max_confidence":0.4311739504337311},{"index":9,"count":6,"confidence":0.4692915827035904,"empirical_correctness":1.0,"signed_gap":-0.5307084172964096,"absolute_gap":0.5307084172964096,"min_confidence":0.4391837418079376,"max_confidence":0.49350541830062866}]}},"calibration_singleton_labels":{"equal_width":{"n":56,"bin_count":10,"ece":0.21705703836466586,"signed_mean_gap":0.032507271372846196,"absolute_mean_gap":0.032507271372846196,"bins":[{"index":0,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.0,"upper":0.1,"upper_inclusive":false},{"index":1,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.1,"upper":0.2,"upper_inclusive":false},{"index":2,"count":28,"confidence":0.24956430973751204,"empirical_correctness":0.0,"signed_gap":0.24956430973751204,"absolute_gap":0.24956430973751204,"lower":0.2,"upper":0.3,"upper_inclusive":false},{"index":3,"count":17,"confidence":0.34830797770444083,"empirical_correctness":0.35294117647058826,"signed_gap":-0.004633198766147428,"absolute_gap":0.004633198766147428,"lower":0.3,"upper":0.4,"upper_inclusive":false},{"index":4,"count":11,"confidence":0.4464882639321414,"empirical_correctness":0.9090909090909091,"signed_gap":-0.46260264515876764,"absolute_gap":0.46260264515876764,"lower":0.4,"upper":0.5,"upper_inclusive":false},{"index":5,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.5,"upper":0.6,"upper_inclusive":false},{"index":6,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.6,"upper":0.7,"upper_inclusive":false},{"index":7,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.7,"upper":0.8,"upper_inclusive":false},{"index":8,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.8,"upper":0.9,"upper_inclusive":false},{"index":9,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.9,"upper":1.0,"upper_inclusive":true}]},"equal_mass":{"n":56,"bin_count":10,"ece":0.2556725535541773,"signed_mean_gap":0.03250727137284619,"absolute_mean_gap":0.03250727137284619,"bins":[{"index":0,"count":5,"confidence":0.20784208476543425,"empirical_correctness":0.0,"signed_gap":0.20784208476543425,"absolute_gap":0.20784208476543425,"min_confidence":0.20523115992546082,"max_confidence":0.2096370905637741},{"index":1,"count":6,"confidence":0.22559662411610285,"empirical_correctness":0.0,"signed_gap":0.22559662411610285,"absolute_gap":0.22559662411610285,"min_confidence":0.21693119406700134,"max_confidence":0.2387024611234665},{"index":2,"count":5,"confidence":0.25356864035129545,"empirical_correctness":0.0,"signed_gap":0.25356864035129545,"absolute_gap":0.25356864035129545,"min_confidence":0.23900891840457916,"max_confidence":0.25897255539894104},{"index":3,"count":6,"confidence":0.26568761964639026,"empirical_correctness":0.0,"signed_gap":0.26568761964639026,"absolute_gap":0.26568761964639026,"min_confidence":0.25927865505218506,"max_confidence":0.2725227177143097},{"index":4,"count":6,"confidence":0.28884026408195496,"empirical_correctness":0.0,"signed_gap":0.28884026408195496,"absolute_gap":0.28884026408195496,"min_confidence":0.27446964383125305,"max_confidence":0.2998443841934204},{"index":5,"count":5,"confidence":0.30944548845291137,"empirical_correctness":0.4,"signed_gap":-0.09055451154708866,"absolute_gap":0.09055451154708866,"min_confidence":0.30249398946762085,"max_confidence":0.3193577229976654},{"index":6,"count":6,"confidence":0.34687240421772003,"empirical_correctness":0.16666666666666666,"signed_gap":0.18020573755105337,"absolute_gap":0.18020573755105337,"min_confidence":0.32922646403312683,"max_confidence":0.3644832372665405},{"index":7,"count":5,"confidence":0.37895127534866335,"empirical_correctness":0.4,"signed_gap":-0.02104872465133667,"absolute_gap":0.02104872465133667,"min_confidence":0.37122467160224915,"max_confidence":0.3920534551143646},{"index":8,"count":6,"confidence":0.4156064639488856,"empirical_correctness":0.8333333333333334,"signed_gap":-0.41772686938444775,"absolute_gap":0.41772686938444775,"min_confidence":0.39801737666130066,"max_confidence":0.4311739504337311},{"index":9,"count":6,"confidence":0.4692915827035904,"empirical_correctness":1.0,"signed_gap":-0.5307084172964096,"absolute_gap":0.5307084172964096,"min_confidence":0.4391837418079376,"max_confidence":0.49350541830062866}]}},"latency_ms":{"n":56,"median":15.505382500123233,"p95":15.765098534757271},"per_action":{"keep":{"n":8,"raw":{"n":8,"acceptable":8,"unsafe":0,"review":0,"automated_count":8,"automated_acceptable":8,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":1.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.6755924351161198,"upper":1.0,"confidence_level":0.95}},"effective":{"n":8,"acceptable":3,"unsafe":0,"review":5,"automated_count":3,"automated_acceptable":3,"automated_unsafe":0,"unnecessary_reviews":5,"accuracy":0.375,"unsafe_rate":0.0,"review_rate":0.625,"automated_coverage":0.375,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.625,"accuracy_wilson_95":{"lower":0.13684428582359742,"upper":0.6942576053973727,"confidence_level":0.95}}},"drop":{"n":22,"raw":{"n":22,"acceptable":0,"unsafe":21,"review":1,"automated_count":21,"automated_acceptable":0,"automated_unsafe":21,"unnecessary_reviews":1,"accuracy":0.0,"unsafe_rate":0.9545454545454546,"review_rate":0.045454545454545456,"automated_coverage":0.9545454545454546,"conditional_correctness":0.0,"conditional_unsafe":1.0,"unnecessary_review_rate":0.045454545454545456,"accuracy_wilson_95":{"lower":0.0,"upper":0.14865487460862087,"confidence_level":0.95}},"effective":{"n":22,"acceptable":0,"unsafe":0,"review":22,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":22,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.14865487460862087,"confidence_level":0.95}}},"mask":{"n":8,"raw":{"n":8,"acceptable":8,"unsafe":0,"review":0,"automated_count":8,"automated_acceptable":8,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":1.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.6755924351161198,"upper":1.0,"confidence_level":0.95}},"effective":{"n":8,"acceptable":2,"unsafe":0,"review":6,"automated_count":2,"automated_acceptable":2,"automated_unsafe":0,"unnecessary_reviews":6,"accuracy":0.25,"unsafe_rate":0.0,"review_rate":0.75,"automated_coverage":0.25,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.75,"accuracy_wilson_95":{"lower":0.071479212752109,"upper":0.5907245696898311,"confidence_level":0.95}}},"pseudonymize":{"n":8,"raw":{"n":8,"acceptable":0,"unsafe":8,"review":0,"automated_count":8,"automated_acceptable":0,"automated_unsafe":8,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":1.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":1.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.32440756488388023,"confidence_level":0.95}},"effective":{"n":8,"acceptable":0,"unsafe":0,"review":8,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":8,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.32440756488388023,"confidence_level":0.95}}},"generalize":{"n":8,"raw":{"n":8,"acceptable":0,"unsafe":8,"review":0,"automated_count":8,"automated_acceptable":0,"automated_unsafe":8,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":1.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":1.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.32440756488388023,"confidence_level":0.95}},"effective":{"n":8,"acceptable":0,"unsafe":0,"review":8,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":8,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.32440756488388023,"confidence_level":0.95}}},"review":{"n":2,"raw":{"n":2,"acceptable":0,"unsafe":2,"review":0,"automated_count":2,"automated_acceptable":0,"automated_unsafe":2,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":1.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":1.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.6576197724933469,"confidence_level":0.95}},"effective":{"n":2,"acceptable":2,"unsafe":0,"review":2,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":1.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.34238022750665303,"upper":1.0,"confidence_level":0.95}}}}},"challenge56-compact_no_samples":{"n":56,"raw":{"n":56,"acceptable":18,"unsafe":38,"review":0,"automated_count":56,"automated_acceptable":18,"automated_unsafe":38,"unnecessary_reviews":0,"accuracy":0.32142857142857145,"unsafe_rate":0.6785714285714286,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.32142857142857145,"conditional_unsafe":0.6785714285714286,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.21400994792551523,"upper":0.45177360090261476,"confidence_level":0.95}},"effective":{"n":56,"acceptable":2,"unsafe":0,"review":56,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":54,"accuracy":0.03571428571428571,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":0.9642857142857143,"accuracy_wilson_95":{"lower":0.00984942514823639,"upper":0.12118780180490107,"confidence_level":0.95}},"confusion":{"raw":{"keep":{"keep":8,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0},"drop":{"keep":22,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0},"mask":{"keep":0,"drop":0,"mask":8,"pseudonymize":0,"generalize":0,"review":0},"pseudonymize":{"keep":8,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0},"generalize":{"keep":6,"drop":0,"mask":0,"pseudonymize":0,"generalize":2,"review":0},"review":{"keep":2,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0}},"effective":{"keep":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":8},"drop":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":22},"mask":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":8},"pseudonymize":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":8},"generalize":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":8},"review":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":2}}},"probability_metrics":{"singleton_n":56,"multiacceptable_n":0,"multiacceptable_coverage":0.0,"nll":1.6614748621856112,"nll_zero_mass_n":0,"brier":0.7809899230517298,"acceptable_set_n":56,"acceptable_set_nll":1.6614748621856112,"acceptable_set_zero_mass_n":0},"calibration_all_labels":{"equal_width":{"n":56,"bin_count":10,"ece":0.2018810013043029,"signed_mean_gap":-0.027334205540163175,"absolute_mean_gap":0.027334205540163175,"bins":[{"index":0,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.0,"upper":0.1,"upper_inclusive":false},{"index":1,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.1,"upper":0.2,"upper_inclusive":false},{"index":2,"count":24,"confidence":0.24530459505816302,"empirical_correctness":0.041666666666666664,"signed_gap":0.20363792839149636,"absolute_gap":0.20363792839149636,"lower":0.2,"upper":0.3,"upper_inclusive":false},{"index":3,"count":32,"confidence":0.3306866940110922,"empirical_correctness":0.53125,"signed_gap":-0.20056330598890781,"absolute_gap":0.20056330598890781,"lower":0.3,"upper":0.4,"upper_inclusive":false},{"index":4,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.4,"upper":0.5,"upper_inclusive":false},{"index":5,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.5,"upper":0.6,"upper_inclusive":false},{"index":6,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.6,"upper":0.7,"upper_inclusive":false},{"index":7,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.7,"upper":0.8,"upper_inclusive":false},{"index":8,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.8,"upper":0.9,"upper_inclusive":false},{"index":9,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.9,"upper":1.0,"upper_inclusive":true}]},"equal_mass":{"n":56,"bin_count":10,"ece":0.2594527200396572,"signed_mean_gap":-0.027334205540163175,"absolute_mean_gap":0.027334205540163175,"bins":[{"index":0,"count":5,"confidence":0.21105519831180572,"empirical_correctness":0.0,"signed_gap":0.21105519831180572,"absolute_gap":0.21105519831180572,"min_confidence":0.20450805127620697,"max_confidence":0.21594882011413574},{"index":1,"count":6,"confidence":0.23233833412329355,"empirical_correctness":0.0,"signed_gap":0.23233833412329355,"absolute_gap":0.23233833412329355,"min_confidence":0.22411395609378815,"max_confidence":0.24099871516227722},{"index":2,"count":5,"confidence":0.2466604471206665,"empirical_correctness":0.0,"signed_gap":0.2466604471206665,"absolute_gap":0.2466604471206665,"min_confidence":0.24122968316078186,"max_confidence":0.2585531175136566},{"index":3,"count":6,"confidence":0.2705639998118083,"empirical_correctness":0.0,"signed_gap":0.2705639998118083,"absolute_gap":0.2705639998118083,"min_confidence":0.26151418685913086,"max_confidence":0.2877606451511383},{"index":4,"count":6,"confidence":0.29848267634709674,"empirical_correctness":0.3333333333333333,"signed_gap":-0.03485065698623657,"absolute_gap":0.03485065698623657,"min_confidence":0.2881524860858917,"max_confidence":0.30555301904678345},{"index":5,"count":5,"confidence":0.3109567940235138,"empirical_correctness":0.6,"signed_gap":-0.2890432059764862,"absolute_gap":0.2890432059764862,"min_confidence":0.3072390854358673,"max_confidence":0.3153274357318878},{"index":6,"count":6,"confidence":0.3203598956267039,"empirical_correctness":0.5,"signed_gap":-0.17964010437329608,"absolute_gap":0.17964010437329608,"min_confidence":0.31822362542152405,"max_confidence":0.32199999690055847},{"index":7,"count":5,"confidence":0.3304227590560913,"empirical_correctness":0.8,"signed_gap":-0.46957724094390874,"absolute_gap":0.46957724094390874,"min_confidence":0.32440632581710815,"max_confidence":0.3364259600639343},{"index":8,"count":6,"confidence":0.3416688144207001,"empirical_correctness":0.8333333333333334,"signed_gap":-0.4916645189126333,"absolute_gap":0.4916645189126333,"min_confidence":0.34032854437828064,"max_confidence":0.3449741005897522},{"index":9,"count":6,"confidence":0.3655543625354767,"empirical_correctness":0.16666666666666666,"signed_gap":0.19888769586881003,"absolute_gap":0.19888769586881003,"min_confidence":0.34558624029159546,"max_confidence":0.3838796615600586}]}},"calibration_singleton_labels":{"equal_width":{"n":56,"bin_count":10,"ece":0.2018810013043029,"signed_mean_gap":-0.027334205540163175,"absolute_mean_gap":0.027334205540163175,"bins":[{"index":0,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.0,"upper":0.1,"upper_inclusive":false},{"index":1,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.1,"upper":0.2,"upper_inclusive":false},{"index":2,"count":24,"confidence":0.24530459505816302,"empirical_correctness":0.041666666666666664,"signed_gap":0.20363792839149636,"absolute_gap":0.20363792839149636,"lower":0.2,"upper":0.3,"upper_inclusive":false},{"index":3,"count":32,"confidence":0.3306866940110922,"empirical_correctness":0.53125,"signed_gap":-0.20056330598890781,"absolute_gap":0.20056330598890781,"lower":0.3,"upper":0.4,"upper_inclusive":false},{"index":4,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.4,"upper":0.5,"upper_inclusive":false},{"index":5,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.5,"upper":0.6,"upper_inclusive":false},{"index":6,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.6,"upper":0.7,"upper_inclusive":false},{"index":7,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.7,"upper":0.8,"upper_inclusive":false},{"index":8,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.8,"upper":0.9,"upper_inclusive":false},{"index":9,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.9,"upper":1.0,"upper_inclusive":true}]},"equal_mass":{"n":56,"bin_count":10,"ece":0.2594527200396572,"signed_mean_gap":-0.027334205540163175,"absolute_mean_gap":0.027334205540163175,"bins":[{"index":0,"count":5,"confidence":0.21105519831180572,"empirical_correctness":0.0,"signed_gap":0.21105519831180572,"absolute_gap":0.21105519831180572,"min_confidence":0.20450805127620697,"max_confidence":0.21594882011413574},{"index":1,"count":6,"confidence":0.23233833412329355,"empirical_correctness":0.0,"signed_gap":0.23233833412329355,"absolute_gap":0.23233833412329355,"min_confidence":0.22411395609378815,"max_confidence":0.24099871516227722},{"index":2,"count":5,"confidence":0.2466604471206665,"empirical_correctness":0.0,"signed_gap":0.2466604471206665,"absolute_gap":0.2466604471206665,"min_confidence":0.24122968316078186,"max_confidence":0.2585531175136566},{"index":3,"count":6,"confidence":0.2705639998118083,"empirical_correctness":0.0,"signed_gap":0.2705639998118083,"absolute_gap":0.2705639998118083,"min_confidence":0.26151418685913086,"max_confidence":0.2877606451511383},{"index":4,"count":6,"confidence":0.29848267634709674,"empirical_correctness":0.3333333333333333,"signed_gap":-0.03485065698623657,"absolute_gap":0.03485065698623657,"min_confidence":0.2881524860858917,"max_confidence":0.30555301904678345},{"index":5,"count":5,"confidence":0.3109567940235138,"empirical_correctness":0.6,"signed_gap":-0.2890432059764862,"absolute_gap":0.2890432059764862,"min_confidence":0.3072390854358673,"max_confidence":0.3153274357318878},{"index":6,"count":6,"confidence":0.3203598956267039,"empirical_correctness":0.5,"signed_gap":-0.17964010437329608,"absolute_gap":0.17964010437329608,"min_confidence":0.31822362542152405,"max_confidence":0.32199999690055847},{"index":7,"count":5,"confidence":0.3304227590560913,"empirical_correctness":0.8,"signed_gap":-0.46957724094390874,"absolute_gap":0.46957724094390874,"min_confidence":0.32440632581710815,"max_confidence":0.3364259600639343},{"index":8,"count":6,"confidence":0.3416688144207001,"empirical_correctness":0.8333333333333334,"signed_gap":-0.4916645189126333,"absolute_gap":0.4916645189126333,"min_confidence":0.34032854437828064,"max_confidence":0.3449741005897522},{"index":9,"count":6,"confidence":0.3655543625354767,"empirical_correctness":0.16666666666666666,"signed_gap":0.19888769586881003,"absolute_gap":0.19888769586881003,"min_confidence":0.34558624029159546,"max_confidence":0.3838796615600586}]}},"latency_ms":{"n":56,"median":18.033748026937246,"p95":18.403644469799474},"per_action":{"keep":{"n":8,"raw":{"n":8,"acceptable":8,"unsafe":0,"review":0,"automated_count":8,"automated_acceptable":8,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":1.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.6755924351161198,"upper":1.0,"confidence_level":0.95}},"effective":{"n":8,"acceptable":0,"unsafe":0,"review":8,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":8,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.32440756488388023,"confidence_level":0.95}}},"drop":{"n":22,"raw":{"n":22,"acceptable":0,"unsafe":22,"review":0,"automated_count":22,"automated_acceptable":0,"automated_unsafe":22,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":1.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":1.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.14865487460862087,"confidence_level":0.95}},"effective":{"n":22,"acceptable":0,"unsafe":0,"review":22,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":22,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.14865487460862087,"confidence_level":0.95}}},"mask":{"n":8,"raw":{"n":8,"acceptable":8,"unsafe":0,"review":0,"automated_count":8,"automated_acceptable":8,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":1.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.6755924351161198,"upper":1.0,"confidence_level":0.95}},"effective":{"n":8,"acceptable":0,"unsafe":0,"review":8,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":8,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.32440756488388023,"confidence_level":0.95}}},"pseudonymize":{"n":8,"raw":{"n":8,"acceptable":0,"unsafe":8,"review":0,"automated_count":8,"automated_acceptable":0,"automated_unsafe":8,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":1.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":1.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.32440756488388023,"confidence_level":0.95}},"effective":{"n":8,"acceptable":0,"unsafe":0,"review":8,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":8,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.32440756488388023,"confidence_level":0.95}}},"generalize":{"n":8,"raw":{"n":8,"acceptable":2,"unsafe":6,"review":0,"automated_count":8,"automated_acceptable":2,"automated_unsafe":6,"unnecessary_reviews":0,"accuracy":0.25,"unsafe_rate":0.75,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.25,"conditional_unsafe":0.75,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.071479212752109,"upper":0.5907245696898311,"confidence_level":0.95}},"effective":{"n":8,"acceptable":0,"unsafe":0,"review":8,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":8,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.32440756488388023,"confidence_level":0.95}}},"review":{"n":2,"raw":{"n":2,"acceptable":0,"unsafe":2,"review":0,"automated_count":2,"automated_acceptable":0,"automated_unsafe":2,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":1.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":1.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.6576197724933469,"confidence_level":0.95}},"effective":{"n":2,"acceptable":2,"unsafe":0,"review":2,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":1.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.34238022750665303,"upper":1.0,"confidence_level":0.95}}}}}},"composite":{"n":198,"scope":"descriptive sum across four correlated panels","raw":{"n":198,"acceptable":63,"unsafe":126,"review":5,"automated_count":193,"automated_acceptable":61,"automated_unsafe":126,"unnecessary_reviews":3,"accuracy":0.3181818181818182,"unsafe_rate":0.6363636363636364,"review_rate":0.025252525252525252,"automated_coverage":0.9747474747474747,"conditional_correctness":0.3160621761658031,"conditional_unsafe":0.6528497409326425,"unnecessary_review_rate":0.015151515151515152},"effective":{"n":198,"acceptable":19,"unsafe":0,"review":187,"automated_count":11,"automated_acceptable":5,"automated_unsafe":0,"unnecessary_reviews":173,"accuracy":0.09595959595959595,"unsafe_rate":0.0,"review_rate":0.9444444444444444,"automated_coverage":0.05555555555555555,"conditional_correctness":0.45454545454545453,"conditional_unsafe":0.0,"unnecessary_review_rate":0.8737373737373737}}},{"id":"soft-decider-fp32","platform":"Local","model_id":"winwinwinbb/soft-decider-421m","revision":"ad67f6b0b178a726a399919f5f6d443e1f772890","loading":"float32 / none","scoring_kind":"Native decision readout","settings":{"dtype":"float32","quantization":"none","max_tokens":1024,"head_max_tokens":256,"temperature":1.0266985893249512,"autocast":false,"truncation":false,"cpu_offload":false,"guard_threshold":0.45,"seed":0,"tf32":false},"scoring_metadata":{"backend":"laya_native_local_builtin","readout":"native_mask_slot_scorer_numpy_float32_softmax","probability_source":"native_choice_unrounded_published_temperature_not_task_calibrated","latency_scope":"synchronized serial complete encoder+head+act forward,input transfer and native readout; tokenization excluded","native_disclosures":["Stock installed torch attention in place of ONNX dynamic equivalent; no custom kernels or repo imports.","FP32 parameters and no autocast are explicit precision condition; saved F16 tensor conversion, no quantization.","Unrounded native probability; guard uses winner p, not entropy-concentration confidence.","Act head fully loaded and evaluated; not used to alter six-action answer.","Typed choice:6-10 bucket1.0000158548355103; soft fallback choice index0=1.0266985893249512; no fitting here."],"equal_mass_convention":"rank bins with floor boundaries at floor(i*n/k), k=min(10,n//2); minimum two rows"},"load_ms":null,"peak_allocated_bytes":1748760576,"peak_reserved_bytes":1870659584,"panels":{"diagnostic43-no_samples":{"n":43,"raw":{"n":43,"acceptable":14,"unsafe":27,"review":2,"automated_count":41,"automated_acceptable":13,"automated_unsafe":27,"unnecessary_reviews":1,"accuracy":0.32558139534883723,"unsafe_rate":0.627906976744186,"review_rate":0.046511627906976744,"automated_coverage":0.9534883720930233,"conditional_correctness":0.3170731707317073,"conditional_unsafe":0.6585365853658537,"unnecessary_review_rate":0.023255813953488372,"accuracy_wilson_95":{"lower":0.20493297126568708,"upper":0.47483789032470525,"confidence_level":0.95}},"effective":{"n":43,"acceptable":6,"unsafe":0,"review":40,"automated_count":3,"automated_acceptable":1,"automated_unsafe":0,"unnecessary_reviews":35,"accuracy":0.13953488372093023,"unsafe_rate":0.0,"review_rate":0.9302325581395349,"automated_coverage":0.06976744186046512,"conditional_correctness":0.3333333333333333,"conditional_unsafe":0.0,"unnecessary_review_rate":0.813953488372093,"accuracy_wilson_95":{"lower":0.06555723427964878,"upper":0.2726358796738286,"confidence_level":0.95}},"confusion":{"raw":{"keep":{"keep":12,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0},"drop":{"keep":14,"drop":0,"mask":1,"pseudonymize":0,"generalize":0,"review":1},"mask":{"keep":1,"drop":0,"mask":1,"pseudonymize":0,"generalize":0,"review":0},"pseudonymize":{"keep":2,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0},"generalize":{"keep":6,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0},"review":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0}},"effective":{"keep":{"keep":1,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":11},"drop":{"keep":0,"drop":0,"mask":2,"pseudonymize":0,"generalize":0,"review":14},"mask":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":2},"pseudonymize":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":2},"generalize":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":6},"review":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0}}},"probability_metrics":{"singleton_n":38,"multiacceptable_n":5,"multiacceptable_coverage":0.11627906976744186,"nll":1.726312593297564,"nll_zero_mass_n":0,"brier":0.8049259399000701,"acceptable_set_n":43,"acceptable_set_nll":1.6262730829333534,"acceptable_set_zero_mass_n":0},"calibration_all_labels":{"equal_width":{"n":43,"bin_count":10,"ece":0.11650890701038893,"signed_mean_gap":-0.011319784577502755,"absolute_mean_gap":0.011319784577502755,"bins":[{"index":0,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.0,"upper":0.1,"upper_inclusive":false},{"index":1,"count":1,"confidence":0.1990603506565094,"empirical_correctness":0.0,"signed_gap":0.1990603506565094,"absolute_gap":0.1990603506565094,"lower":0.1,"upper":0.2,"upper_inclusive":false},{"index":2,"count":17,"confidence":0.24649182224974914,"empirical_correctness":0.29411764705882354,"signed_gap":-0.0476258248090744,"absolute_gap":0.0476258248090744,"lower":0.2,"upper":0.3,"upper_inclusive":false},{"index":3,"count":18,"confidence":0.3367401195896996,"empirical_correctness":0.4444444444444444,"signed_gap":-0.10770432485474479,"absolute_gap":0.10770432485474479,"lower":0.3,"upper":0.4,"upper_inclusive":false},{"index":4,"count":7,"confidence":0.4375008259500776,"empirical_correctness":0.14285714285714285,"signed_gap":0.29464368309293476,"absolute_gap":0.29464368309293476,"lower":0.4,"upper":0.5,"upper_inclusive":false},{"index":5,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.5,"upper":0.6,"upper_inclusive":false},{"index":6,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.6,"upper":0.7,"upper_inclusive":false},{"index":7,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.7,"upper":0.8,"upper_inclusive":false},{"index":8,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.8,"upper":0.9,"upper_inclusive":false},{"index":9,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.9,"upper":1.0,"upper_inclusive":true}]},"equal_mass":{"n":43,"bin_count":10,"ece":0.22564365454884464,"signed_mean_gap":-0.01131978457750276,"absolute_mean_gap":0.01131978457750276,"bins":[{"index":0,"count":4,"confidence":0.2042226567864418,"empirical_correctness":0.25,"signed_gap":-0.0457773432135582,"absolute_gap":0.0457773432135582,"min_confidence":0.1990603506565094,"max_confidence":0.21112041175365448},{"index":1,"count":4,"confidence":0.22469474375247955,"empirical_correctness":0.25,"signed_gap":-0.025305256247520447,"absolute_gap":0.025305256247520447,"min_confidence":0.21854814887046814,"max_confidence":0.23029568791389465},{"index":2,"count":4,"confidence":0.24163758382201195,"empirical_correctness":0.5,"signed_gap":-0.25836241617798805,"absolute_gap":0.25836241617798805,"min_confidence":0.23213662207126617,"max_confidence":0.25210678577423096},{"index":3,"count":5,"confidence":0.28212732672691343,"empirical_correctness":0.0,"signed_gap":0.28212732672691343,"absolute_gap":0.28212732672691343,"min_confidence":0.2579452693462372,"max_confidence":0.2949696183204651},{"index":4,"count":4,"confidence":0.30577607452869415,"empirical_correctness":0.5,"signed_gap":-0.19422392547130585,"absolute_gap":0.19422392547130585,"min_confidence":0.2965647578239441,"max_confidence":0.3168673515319824},{"index":5,"count":4,"confidence":0.3262729197740555,"empirical_correctness":0.5,"signed_gap":-0.17372708022594452,"absolute_gap":0.17372708022594452,"min_confidence":0.32353219389915466,"max_confidence":0.3294776380062103},{"index":6,"count":5,"confidence":0.3389740288257599,"empirical_correctness":0.8,"signed_gap":-0.46102597117424016,"absolute_gap":0.46102597117424016,"min_confidence":0.3358517289161682,"max_confidence":0.34475359320640564},{"index":7,"count":4,"confidence":0.348966121673584,"empirical_correctness":0.25,"signed_gap":0.09896612167358398,"absolute_gap":0.09896612167358398,"min_confidence":0.34583479166030884,"max_confidence":0.35228803753852844},{"index":8,"count":4,"confidence":0.3907874524593353,"empirical_correctness":0.0,"signed_gap":0.3907874524593353,"absolute_gap":0.3907874524593353,"min_confidence":0.3693584203720093,"max_confidence":0.4177217185497284},{"index":9,"count":5,"confidence":0.44766245484352113,"empirical_correctness":0.2,"signed_gap":0.24766245484352112,"absolute_gap":0.24766245484352112,"min_confidence":0.42048341035842896,"max_confidence":0.4903034269809723}]}},"calibration_singleton_labels":{"equal_width":{"n":38,"bin_count":10,"ece":0.10190898728998082,"signed_mean_gap":-0.027418509125709523,"absolute_mean_gap":0.027418509125709523,"bins":[{"index":0,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.0,"upper":0.1,"upper_inclusive":false},{"index":1,"count":1,"confidence":0.1990603506565094,"empirical_correctness":0.0,"signed_gap":0.1990603506565094,"absolute_gap":0.1990603506565094,"lower":0.1,"upper":0.2,"upper_inclusive":false},{"index":2,"count":14,"confidence":0.24867538682052068,"empirical_correctness":0.2857142857142857,"signed_gap":-0.03703889889376502,"absolute_gap":0.03703889889376502,"lower":0.2,"upper":0.3,"upper_inclusive":false},{"index":3,"count":18,"confidence":0.3367401195896996,"empirical_correctness":0.4444444444444444,"signed_gap":-0.10770432485474479,"absolute_gap":0.10770432485474479,"lower":0.3,"upper":0.4,"upper_inclusive":false},{"index":4,"count":5,"confidence":0.44325174689292907,"empirical_correctness":0.2,"signed_gap":0.24325174689292905,"absolute_gap":0.24325174689292905,"lower":0.4,"upper":0.5,"upper_inclusive":false},{"index":5,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.5,"upper":0.6,"upper_inclusive":false},{"index":6,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.6,"upper":0.7,"upper_inclusive":false},{"index":7,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.7,"upper":0.8,"upper_inclusive":false},{"index":8,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.8,"upper":0.9,"upper_inclusive":false},{"index":9,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.9,"upper":1.0,"upper_inclusive":true}]},"equal_mass":{"n":38,"bin_count":10,"ece":0.20342071040680534,"signed_mean_gap":-0.027418509125709534,"absolute_mean_gap":0.027418509125709534,"bins":[{"index":0,"count":3,"confidence":0.20375743508338928,"empirical_correctness":0.0,"signed_gap":0.20375743508338928,"absolute_gap":0.20375743508338928,"min_confidence":0.1990603506565094,"max_confidence":0.21112041175365448},{"index":1,"count":4,"confidence":0.2254224419593811,"empirical_correctness":0.5,"signed_gap":-0.2745775580406189,"absolute_gap":0.2745775580406189,"min_confidence":0.21854814887046814,"max_confidence":0.23213662207126617},{"index":2,"count":4,"confidence":0.2480897456407547,"empirical_correctness":0.25,"signed_gap":-0.0019102543592453003,"absolute_gap":0.0019102543592453003,"min_confidence":0.23594002425670624,"max_confidence":0.2579452693462372},{"index":3,"count":4,"confidence":0.29379867762327194,"empirical_correctness":0.25,"signed_gap":0.04379867762327194,"absolute_gap":0.04379867762327194,"min_confidence":0.28903892636299133,"max_confidence":0.2965647578239441},{"index":4,"count":4,"confidence":0.3125179335474968,"empirical_correctness":0.25,"signed_gap":0.0625179335474968,"absolute_gap":0.0625179335474968,"min_confidence":0.30376964807510376,"max_confidence":0.32353219389915466},{"index":5,"count":3,"confidence":0.3271864950656891,"empirical_correctness":0.6666666666666666,"signed_gap":-0.33948017160097754,"absolute_gap":0.33948017160097754,"min_confidence":0.32576829195022583,"max_confidence":0.3294776380062103},{"index":6,"count":4,"confidence":0.33752913773059845,"empirical_correctness":0.75,"signed_gap":-0.41247086226940155,"absolute_gap":0.41247086226940155,"min_confidence":0.3358517289161682,"max_confidence":0.33832207322120667},{"index":7,"count":4,"confidence":0.3470825105905533,"empirical_correctness":0.5,"signed_gap":-0.15291748940944672,"absolute_gap":0.15291748940944672,"min_confidence":0.34475359320640564,"max_confidence":0.3510771691799164},{"index":8,"count":4,"confidence":0.37442903220653534,"empirical_correctness":0.0,"signed_gap":0.37442903220653534,"absolute_gap":0.37442903220653534,"min_confidence":0.35228803753852844,"max_confidence":0.40647178888320923},{"index":9,"count":4,"confidence":0.45244673639535904,"empirical_correctness":0.25,"signed_gap":0.20244673639535904,"absolute_gap":0.20244673639535904,"min_confidence":0.42048341035842896,"max_confidence":0.4903034269809723}]}},"latency_ms":{"n":43,"median":67.71219987422228,"p95":69.30560984183103},"per_action":{"keep":{"n":12,"raw":{"n":12,"acceptable":12,"unsafe":0,"review":0,"automated_count":12,"automated_acceptable":12,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":1.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.7575059933447592,"upper":1.0,"confidence_level":0.95}},"effective":{"n":12,"acceptable":1,"unsafe":0,"review":11,"automated_count":1,"automated_acceptable":1,"automated_unsafe":0,"unnecessary_reviews":11,"accuracy":0.08333333333333333,"unsafe_rate":0.0,"review_rate":0.9166666666666666,"automated_coverage":0.08333333333333333,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.9166666666666666,"accuracy_wilson_95":{"lower":0.014865094404917095,"upper":0.35387991114111694,"confidence_level":0.95}}},"drop":{"n":16,"raw":{"n":16,"acceptable":0,"unsafe":14,"review":1,"automated_count":15,"automated_acceptable":0,"automated_unsafe":14,"unnecessary_reviews":1,"accuracy":0.0,"unsafe_rate":0.875,"review_rate":0.0625,"automated_coverage":0.9375,"conditional_correctness":0.0,"conditional_unsafe":0.9333333333333333,"unnecessary_review_rate":0.0625,"accuracy_wilson_95":{"lower":0.0,"upper":0.1936076805344365,"confidence_level":0.95}},"effective":{"n":16,"acceptable":0,"unsafe":0,"review":14,"automated_count":2,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":14,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":0.875,"automated_coverage":0.125,"conditional_correctness":0.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.875,"accuracy_wilson_95":{"lower":0.0,"upper":0.1936076805344365,"confidence_level":0.95}}},"mask":{"n":2,"raw":{"n":2,"acceptable":1,"unsafe":1,"review":0,"automated_count":2,"automated_acceptable":1,"automated_unsafe":1,"unnecessary_reviews":0,"accuracy":0.5,"unsafe_rate":0.5,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.5,"conditional_unsafe":0.5,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.09453120573423074,"upper":0.9054687942657693,"confidence_level":0.95}},"effective":{"n":2,"acceptable":0,"unsafe":0,"review":2,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":2,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.6576197724933469,"confidence_level":0.95}}},"pseudonymize":{"n":2,"raw":{"n":2,"acceptable":0,"unsafe":2,"review":0,"automated_count":2,"automated_acceptable":0,"automated_unsafe":2,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":1.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":1.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.6576197724933469,"confidence_level":0.95}},"effective":{"n":2,"acceptable":0,"unsafe":0,"review":2,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":2,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.6576197724933469,"confidence_level":0.95}}},"generalize":{"n":6,"raw":{"n":6,"acceptable":0,"unsafe":6,"review":0,"automated_count":6,"automated_acceptable":0,"automated_unsafe":6,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":1.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":1.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":2.7755575615628914e-17,"upper":0.3903342879021653,"confidence_level":0.95}},"effective":{"n":6,"acceptable":0,"unsafe":0,"review":6,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":6,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":2.7755575615628914e-17,"upper":0.3903342879021653,"confidence_level":0.95}}},"review":{"n":0,"raw":{"n":0,"acceptable":0,"unsafe":0,"review":0,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":null,"unsafe_rate":null,"review_rate":null,"automated_coverage":null,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":null},"effective":{"n":0,"acceptable":0,"unsafe":0,"review":0,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":null,"unsafe_rate":null,"review_rate":null,"automated_coverage":null,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":null}}}},"diagnostic43-compact_no_samples":{"n":43,"raw":{"n":43,"acceptable":13,"unsafe":30,"review":0,"automated_count":43,"automated_acceptable":13,"automated_unsafe":30,"unnecessary_reviews":0,"accuracy":0.3023255813953488,"unsafe_rate":0.6976744186046512,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.3023255813953488,"conditional_unsafe":0.6976744186046512,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.1860200013584818,"upper":0.45105364177729607,"confidence_level":0.95}},"effective":{"n":43,"acceptable":8,"unsafe":0,"review":36,"automated_count":7,"automated_acceptable":3,"automated_unsafe":0,"unnecessary_reviews":31,"accuracy":0.18604651162790697,"unsafe_rate":0.0,"review_rate":0.8372093023255814,"automated_coverage":0.16279069767441862,"conditional_correctness":0.42857142857142855,"conditional_unsafe":0.0,"unnecessary_review_rate":0.7209302325581395,"accuracy_wilson_95":{"lower":0.09741751428996226,"upper":0.32617003657274385,"confidence_level":0.95}},"confusion":{"raw":{"keep":{"keep":12,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0},"drop":{"keep":16,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0},"mask":{"keep":2,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0},"pseudonymize":{"keep":2,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0},"generalize":{"keep":5,"drop":0,"mask":0,"pseudonymize":0,"generalize":1,"review":0},"review":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0}},"effective":{"keep":{"keep":2,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":10},"drop":{"keep":0,"drop":0,"mask":4,"pseudonymize":0,"generalize":0,"review":12},"mask":{"keep":0,"drop":0,"mask":1,"pseudonymize":0,"generalize":0,"review":1},"pseudonymize":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":2},"generalize":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":6},"review":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0}}},"probability_metrics":{"singleton_n":38,"multiacceptable_n":5,"multiacceptable_coverage":0.11627906976744186,"nll":1.692212459825314,"nll_zero_mass_n":0,"brier":0.7869456611798459,"acceptable_set_n":43,"acceptable_set_nll":1.5962586227140856,"acceptable_set_zero_mass_n":0},"calibration_all_labels":{"equal_width":{"n":43,"bin_count":10,"ece":0.08580728602963825,"signed_mean_gap":0.06295166875040809,"absolute_mean_gap":0.06295166875040809,"bins":[{"index":0,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.0,"upper":0.1,"upper_inclusive":false},{"index":1,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.1,"upper":0.2,"upper_inclusive":false},{"index":2,"count":6,"confidence":0.28414801756540936,"empirical_correctness":0.16666666666666666,"signed_gap":0.1174813508987427,"absolute_gap":0.1174813508987427,"lower":0.2,"upper":0.3,"upper_inclusive":false},{"index":3,"count":26,"confidence":0.3560701104310843,"empirical_correctness":0.2692307692307692,"signed_gap":0.08683934120031506,"absolute_gap":0.08683934120031506,"lower":0.3,"upper":0.4,"upper_inclusive":false},{"index":4,"count":10,"confidence":0.4235606551170349,"empirical_correctness":0.4,"signed_gap":0.023560655117034868,"absolute_gap":0.023560655117034868,"lower":0.4,"upper":0.5,"upper_inclusive":false},{"index":5,"count":1,"confidence":0.5086042284965515,"empirical_correctness":1.0,"signed_gap":-0.4913957715034485,"absolute_gap":0.4913957715034485,"lower":0.5,"upper":0.6,"upper_inclusive":false},{"index":6,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.6,"upper":0.7,"upper_inclusive":false},{"index":7,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.7,"upper":0.8,"upper_inclusive":false},{"index":8,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.8,"upper":0.9,"upper_inclusive":false},{"index":9,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.9,"upper":1.0,"upper_inclusive":true}]},"equal_mass":{"n":43,"bin_count":10,"ece":0.09741384068200755,"signed_mean_gap":0.06295166875040809,"absolute_mean_gap":0.06295166875040809,"bins":[{"index":0,"count":4,"confidence":0.27747800946235657,"empirical_correctness":0.25,"signed_gap":0.027478009462356567,"absolute_gap":0.027478009462356567,"min_confidence":0.24494808912277222,"max_confidence":0.2960280478000641},{"index":1,"count":4,"confidence":0.303131140768528,"empirical_correctness":0.25,"signed_gap":0.053131140768527985,"absolute_gap":0.053131140768527985,"min_confidence":0.2963498830795288,"max_confidence":0.3114579916000366},{"index":2,"count":4,"confidence":0.3165987879037857,"empirical_correctness":0.25,"signed_gap":0.0665987879037857,"absolute_gap":0.0665987879037857,"min_confidence":0.3135300576686859,"max_confidence":0.32071271538734436},{"index":3,"count":5,"confidence":0.3408970534801483,"empirical_correctness":0.2,"signed_gap":0.1408970534801483,"absolute_gap":0.1408970534801483,"min_confidence":0.3224175274372101,"max_confidence":0.3500707745552063},{"index":4,"count":4,"confidence":0.3583533242344856,"empirical_correctness":0.25,"signed_gap":0.10835332423448563,"absolute_gap":0.10835332423448563,"min_confidence":0.3540751039981842,"max_confidence":0.3620409071445465},{"index":5,"count":4,"confidence":0.37446631491184235,"empirical_correctness":0.25,"signed_gap":0.12446631491184235,"absolute_gap":0.12446631491184235,"min_confidence":0.365102082490921,"max_confidence":0.3860052824020386},{"index":6,"count":5,"confidence":0.388287615776062,"empirical_correctness":0.2,"signed_gap":0.188287615776062,"absolute_gap":0.188287615776062,"min_confidence":0.38632863759994507,"max_confidence":0.3938152492046356},{"index":7,"count":4,"confidence":0.4022049605846405,"empirical_correctness":0.5,"signed_gap":-0.0977950394153595,"absolute_gap":0.0977950394153595,"min_confidence":0.39705735445022583,"max_confidence":0.4064539968967438},{"index":8,"count":4,"confidence":0.4125608652830124,"empirical_correctness":0.5,"signed_gap":-0.08743913471698761,"absolute_gap":0.08743913471698761,"min_confidence":0.4069371819496155,"max_confidence":0.41529813408851624},{"index":9,"count":5,"confidence":0.4563649594783783,"empirical_correctness":0.4,"signed_gap":0.056364959478378274,"absolute_gap":0.056364959478378274,"min_confidence":0.41786637902259827,"max_confidence":0.5086042284965515}]}},"calibration_singleton_labels":{"equal_width":{"n":38,"bin_count":10,"ece":0.05202402723462956,"signed_mean_gap":0.026161091892342794,"absolute_mean_gap":0.026161091892342794,"bins":[{"index":0,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.0,"upper":0.1,"upper_inclusive":false},{"index":1,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.1,"upper":0.2,"upper_inclusive":false},{"index":2,"count":5,"confidence":0.28170764446258545,"empirical_correctness":0.2,"signed_gap":0.08170764446258544,"absolute_gap":0.08170764446258544,"lower":0.2,"upper":0.3,"upper_inclusive":false},{"index":3,"count":22,"confidence":0.35642602226950904,"empirical_correctness":0.3181818181818182,"signed_gap":0.03824420408769086,"absolute_gap":0.03824420408769086,"lower":0.3,"upper":0.4,"upper_inclusive":false},{"index":4,"count":10,"confidence":0.4235606551170349,"empirical_correctness":0.4,"signed_gap":0.023560655117034868,"absolute_gap":0.023560655117034868,"lower":0.4,"upper":0.5,"upper_inclusive":false},{"index":5,"count":1,"confidence":0.5086042284965515,"empirical_correctness":1.0,"signed_gap":-0.4913957715034485,"absolute_gap":0.4913957715034485,"lower":0.5,"upper":0.6,"upper_inclusive":false},{"index":6,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.6,"upper":0.7,"upper_inclusive":false},{"index":7,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.7,"upper":0.8,"upper_inclusive":false},{"index":8,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.8,"upper":0.9,"upper_inclusive":false},{"index":9,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.9,"upper":1.0,"upper_inclusive":true}]},"equal_mass":{"n":38,"bin_count":10,"ece":0.11279092964373137,"signed_mean_gap":0.026161091892342822,"absolute_mean_gap":0.026161091892342822,"bins":[{"index":0,"count":3,"confidence":0.2712946633497874,"empirical_correctness":0.3333333333333333,"signed_gap":-0.0620386699835459,"absolute_gap":0.0620386699835459,"min_confidence":0.24494808912277222,"max_confidence":0.29296764731407166},{"index":1,"count":4,"confidence":0.3030506819486618,"empirical_correctness":0.25,"signed_gap":0.053050681948661804,"absolute_gap":0.053050681948661804,"min_confidence":0.2960280478000641,"max_confidence":0.3114579916000366},{"index":2,"count":4,"confidence":0.3165987879037857,"empirical_correctness":0.25,"signed_gap":0.0665987879037857,"absolute_gap":0.0665987879037857,"min_confidence":0.3135300576686859,"max_confidence":0.32071271538734436},{"index":3,"count":4,"confidence":0.3488038405776024,"empirical_correctness":0.25,"signed_gap":0.09880384057760239,"absolute_gap":0.09880384057760239,"min_confidence":0.34162434935569763,"max_confidence":0.3540751039981842},{"index":4,"count":4,"confidence":0.36416661739349365,"empirical_correctness":0.5,"signed_gap":-0.13583338260650635,"absolute_gap":0.13583338260650635,"min_confidence":0.35800716280937195,"max_confidence":0.37151631712913513},{"index":5,"count":3,"confidence":0.38252516587575275,"empirical_correctness":0.0,"signed_gap":0.38252516587575275,"absolute_gap":0.38252516587575275,"min_confidence":0.37524157762527466,"max_confidence":0.38632863759994507},{"index":6,"count":4,"confidence":0.3895878866314888,"empirical_correctness":0.5,"signed_gap":-0.1104121133685112,"absolute_gap":0.1104121133685112,"min_confidence":0.3866591453552246,"max_confidence":0.39705735445022583},{"index":7,"count":4,"confidence":0.4046749174594879,"empirical_correctness":0.25,"signed_gap":0.15467491745948792,"absolute_gap":0.15467491745948792,"min_confidence":0.3996199667453766,"max_confidence":0.4069371819496155},{"index":8,"count":4,"confidence":0.4152931645512581,"empirical_correctness":0.5,"signed_gap":-0.08470683544874191,"absolute_gap":0.08470683544874191,"min_confidence":0.41385021805763245,"max_confidence":0.41786637902259827},{"index":9,"count":4,"confidence":0.4659896045923233,"empirical_correctness":0.5,"signed_gap":-0.0340103954076767,"absolute_gap":0.0340103954076767,"min_confidence":0.42839154601097107,"max_confidence":0.5086042284965515}]}},"latency_ms":{"n":43,"median":78.26660014688969,"p95":80.25793994311243},"per_action":{"keep":{"n":12,"raw":{"n":12,"acceptable":12,"unsafe":0,"review":0,"automated_count":12,"automated_acceptable":12,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":1.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.7575059933447592,"upper":1.0,"confidence_level":0.95}},"effective":{"n":12,"acceptable":2,"unsafe":0,"review":10,"automated_count":2,"automated_acceptable":2,"automated_unsafe":0,"unnecessary_reviews":10,"accuracy":0.16666666666666666,"unsafe_rate":0.0,"review_rate":0.8333333333333334,"automated_coverage":0.16666666666666666,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.8333333333333334,"accuracy_wilson_95":{"lower":0.04696514218385378,"upper":0.4480308622529734,"confidence_level":0.95}}},"drop":{"n":16,"raw":{"n":16,"acceptable":0,"unsafe":16,"review":0,"automated_count":16,"automated_acceptable":0,"automated_unsafe":16,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":1.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":1.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.1936076805344365,"confidence_level":0.95}},"effective":{"n":16,"acceptable":0,"unsafe":0,"review":12,"automated_count":4,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":12,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":0.75,"automated_coverage":0.25,"conditional_correctness":0.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.75,"accuracy_wilson_95":{"lower":0.0,"upper":0.1936076805344365,"confidence_level":0.95}}},"mask":{"n":2,"raw":{"n":2,"acceptable":0,"unsafe":2,"review":0,"automated_count":2,"automated_acceptable":0,"automated_unsafe":2,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":1.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":1.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.6576197724933469,"confidence_level":0.95}},"effective":{"n":2,"acceptable":1,"unsafe":0,"review":1,"automated_count":1,"automated_acceptable":1,"automated_unsafe":0,"unnecessary_reviews":1,"accuracy":0.5,"unsafe_rate":0.0,"review_rate":0.5,"automated_coverage":0.5,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.5,"accuracy_wilson_95":{"lower":0.09453120573423074,"upper":0.9054687942657693,"confidence_level":0.95}}},"pseudonymize":{"n":2,"raw":{"n":2,"acceptable":0,"unsafe":2,"review":0,"automated_count":2,"automated_acceptable":0,"automated_unsafe":2,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":1.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":1.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.6576197724933469,"confidence_level":0.95}},"effective":{"n":2,"acceptable":0,"unsafe":0,"review":2,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":2,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.6576197724933469,"confidence_level":0.95}}},"generalize":{"n":6,"raw":{"n":6,"acceptable":1,"unsafe":5,"review":0,"automated_count":6,"automated_acceptable":1,"automated_unsafe":5,"unnecessary_reviews":0,"accuracy":0.16666666666666666,"unsafe_rate":0.8333333333333334,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.16666666666666666,"conditional_unsafe":0.8333333333333334,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.030053369748306635,"upper":0.5635028221864702,"confidence_level":0.95}},"effective":{"n":6,"acceptable":0,"unsafe":0,"review":6,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":6,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":2.7755575615628914e-17,"upper":0.3903342879021653,"confidence_level":0.95}}},"review":{"n":0,"raw":{"n":0,"acceptable":0,"unsafe":0,"review":0,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":null,"unsafe_rate":null,"review_rate":null,"automated_coverage":null,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":null},"effective":{"n":0,"acceptable":0,"unsafe":0,"review":0,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":null,"unsafe_rate":null,"review_rate":null,"automated_coverage":null,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":null}}}},"challenge56-no_samples":{"n":56,"raw":{"n":56,"acceptable":15,"unsafe":41,"review":0,"automated_count":56,"automated_acceptable":15,"automated_unsafe":41,"unnecessary_reviews":0,"accuracy":0.26785714285714285,"unsafe_rate":0.7321428571428571,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.26785714285714285,"conditional_unsafe":0.7321428571428571,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.16957305418501728,"upper":0.39594555929155156,"confidence_level":0.95}},"effective":{"n":56,"acceptable":7,"unsafe":0,"review":51,"automated_count":5,"automated_acceptable":5,"automated_unsafe":0,"unnecessary_reviews":49,"accuracy":0.125,"unsafe_rate":0.0,"review_rate":0.9107142857142857,"automated_coverage":0.08928571428571429,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.875,"accuracy_wilson_95":{"lower":0.06189068695717122,"upper":0.2362547655819014,"confidence_level":0.95}},"confusion":{"raw":{"keep":{"keep":8,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0},"drop":{"keep":22,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0},"mask":{"keep":1,"drop":0,"mask":7,"pseudonymize":0,"generalize":0,"review":0},"pseudonymize":{"keep":8,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0},"generalize":{"keep":8,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0},"review":{"keep":2,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0}},"effective":{"keep":{"keep":5,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":3},"drop":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":22},"mask":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":8},"pseudonymize":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":8},"generalize":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":8},"review":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":2}}},"probability_metrics":{"singleton_n":56,"multiacceptable_n":0,"multiacceptable_coverage":0.0,"nll":1.5639117922441839,"nll_zero_mass_n":0,"brier":0.7472651942499821,"acceptable_set_n":56,"acceptable_set_nll":1.5639117922441839,"acceptable_set_zero_mass_n":0},"calibration_all_labels":{"equal_width":{"n":56,"bin_count":10,"ece":0.16472209404621804,"signed_mean_gap":0.11973283812403679,"absolute_mean_gap":0.11973283812403679,"bins":[{"index":0,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.0,"upper":0.1,"upper_inclusive":false},{"index":1,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.1,"upper":0.2,"upper_inclusive":false},{"index":2,"count":17,"confidence":0.2516643878291635,"empirical_correctness":0.0,"signed_gap":0.2516643878291635,"absolute_gap":0.2516643878291635,"lower":0.2,"upper":0.3,"upper_inclusive":false},{"index":3,"count":14,"confidence":0.3461874042238508,"empirical_correctness":0.2857142857142857,"signed_gap":0.060473118509565105,"absolute_gap":0.060473118509565105,"lower":0.3,"upper":0.4,"upper_inclusive":false},{"index":4,"count":17,"confidence":0.44569968826630535,"empirical_correctness":0.29411764705882354,"signed_gap":0.1515820412074818,"absolute_gap":0.1515820412074818,"lower":0.4,"upper":0.5,"upper_inclusive":false},{"index":5,"count":4,"confidence":0.5657312870025635,"empirical_correctness":0.5,"signed_gap":0.06573128700256348,"absolute_gap":0.06573128700256348,"lower":0.5,"upper":0.6,"upper_inclusive":false},{"index":6,"count":2,"confidence":0.6562583148479462,"empirical_correctness":1.0,"signed_gap":-0.34374168515205383,"absolute_gap":0.34374168515205383,"lower":0.6,"upper":0.7,"upper_inclusive":false},{"index":7,"count":2,"confidence":0.7138921022415161,"empirical_correctness":1.0,"signed_gap":-0.2861078977584839,"absolute_gap":0.2861078977584839,"lower":0.7,"upper":0.8,"upper_inclusive":false},{"index":8,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.8,"upper":0.9,"upper_inclusive":false},{"index":9,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.9,"upper":1.0,"upper_inclusive":true}]},"equal_mass":{"n":56,"bin_count":10,"ece":0.21157458584223474,"signed_mean_gap":0.11973283812403679,"absolute_mean_gap":0.11973283812403679,"bins":[{"index":0,"count":5,"confidence":0.22328122854232788,"empirical_correctness":0.0,"signed_gap":0.22328122854232788,"absolute_gap":0.22328122854232788,"min_confidence":0.2133185714483261,"max_confidence":0.23346449434757233},{"index":1,"count":6,"confidence":0.2498639722665151,"empirical_correctness":0.0,"signed_gap":0.2498639722665151,"absolute_gap":0.2498639722665151,"min_confidence":0.23687754571437836,"max_confidence":0.2637186348438263},{"index":2,"count":5,"confidence":0.27329697608947756,"empirical_correctness":0.0,"signed_gap":0.27329697608947756,"absolute_gap":0.27329697608947756,"min_confidence":0.26420363783836365,"max_confidence":0.28470274806022644},{"index":3,"count":6,"confidence":0.3178056428829829,"empirical_correctness":0.16666666666666666,"signed_gap":0.15113897621631625,"absolute_gap":0.15113897621631625,"min_confidence":0.29621973633766174,"max_confidence":0.3380500376224518},{"index":4,"count":6,"confidence":0.3511861264705658,"empirical_correctness":0.3333333333333333,"signed_gap":0.01785279313723248,"absolute_gap":0.01785279313723248,"min_confidence":0.3431278169155121,"max_confidence":0.36603087186813354},{"index":5,"count":5,"confidence":0.39002777338027955,"empirical_correctness":0.4,"signed_gap":-0.00997222661972047,"absolute_gap":0.00997222661972047,"min_confidence":0.3681107759475708,"max_confidence":0.41203364729881287},{"index":6,"count":6,"confidence":0.4271775633096695,"empirical_correctness":0.5,"signed_gap":-0.0728224366903305,"absolute_gap":0.0728224366903305,"min_confidence":0.41472217440605164,"max_confidence":0.43705591559410095},{"index":7,"count":5,"confidence":0.4584287405014038,"empirical_correctness":0.2,"signed_gap":0.2584287405014038,"absolute_gap":0.2584287405014038,"min_confidence":0.4529012143611908,"max_confidence":0.46352794766426086},{"index":8,"count":6,"confidence":0.49807311594486237,"empirical_correctness":0.0,"signed_gap":0.49807311594486237,"absolute_gap":0.49807311594486237,"min_confidence":0.466327965259552,"max_confidence":0.5701020956039429},{"index":9,"count":6,"confidence":0.6525378028551737,"empirical_correctness":1.0,"signed_gap":-0.3474621971448263,"absolute_gap":0.3474621971448263,"min_confidence":0.5850179195404053,"max_confidence":0.7242468595504761}]}},"calibration_singleton_labels":{"equal_width":{"n":56,"bin_count":10,"ece":0.16472209404621804,"signed_mean_gap":0.11973283812403679,"absolute_mean_gap":0.11973283812403679,"bins":[{"index":0,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.0,"upper":0.1,"upper_inclusive":false},{"index":1,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.1,"upper":0.2,"upper_inclusive":false},{"index":2,"count":17,"confidence":0.2516643878291635,"empirical_correctness":0.0,"signed_gap":0.2516643878291635,"absolute_gap":0.2516643878291635,"lower":0.2,"upper":0.3,"upper_inclusive":false},{"index":3,"count":14,"confidence":0.3461874042238508,"empirical_correctness":0.2857142857142857,"signed_gap":0.060473118509565105,"absolute_gap":0.060473118509565105,"lower":0.3,"upper":0.4,"upper_inclusive":false},{"index":4,"count":17,"confidence":0.44569968826630535,"empirical_correctness":0.29411764705882354,"signed_gap":0.1515820412074818,"absolute_gap":0.1515820412074818,"lower":0.4,"upper":0.5,"upper_inclusive":false},{"index":5,"count":4,"confidence":0.5657312870025635,"empirical_correctness":0.5,"signed_gap":0.06573128700256348,"absolute_gap":0.06573128700256348,"lower":0.5,"upper":0.6,"upper_inclusive":false},{"index":6,"count":2,"confidence":0.6562583148479462,"empirical_correctness":1.0,"signed_gap":-0.34374168515205383,"absolute_gap":0.34374168515205383,"lower":0.6,"upper":0.7,"upper_inclusive":false},{"index":7,"count":2,"confidence":0.7138921022415161,"empirical_correctness":1.0,"signed_gap":-0.2861078977584839,"absolute_gap":0.2861078977584839,"lower":0.7,"upper":0.8,"upper_inclusive":false},{"index":8,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.8,"upper":0.9,"upper_inclusive":false},{"index":9,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.9,"upper":1.0,"upper_inclusive":true}]},"equal_mass":{"n":56,"bin_count":10,"ece":0.21157458584223474,"signed_mean_gap":0.11973283812403679,"absolute_mean_gap":0.11973283812403679,"bins":[{"index":0,"count":5,"confidence":0.22328122854232788,"empirical_correctness":0.0,"signed_gap":0.22328122854232788,"absolute_gap":0.22328122854232788,"min_confidence":0.2133185714483261,"max_confidence":0.23346449434757233},{"index":1,"count":6,"confidence":0.2498639722665151,"empirical_correctness":0.0,"signed_gap":0.2498639722665151,"absolute_gap":0.2498639722665151,"min_confidence":0.23687754571437836,"max_confidence":0.2637186348438263},{"index":2,"count":5,"confidence":0.27329697608947756,"empirical_correctness":0.0,"signed_gap":0.27329697608947756,"absolute_gap":0.27329697608947756,"min_confidence":0.26420363783836365,"max_confidence":0.28470274806022644},{"index":3,"count":6,"confidence":0.3178056428829829,"empirical_correctness":0.16666666666666666,"signed_gap":0.15113897621631625,"absolute_gap":0.15113897621631625,"min_confidence":0.29621973633766174,"max_confidence":0.3380500376224518},{"index":4,"count":6,"confidence":0.3511861264705658,"empirical_correctness":0.3333333333333333,"signed_gap":0.01785279313723248,"absolute_gap":0.01785279313723248,"min_confidence":0.3431278169155121,"max_confidence":0.36603087186813354},{"index":5,"count":5,"confidence":0.39002777338027955,"empirical_correctness":0.4,"signed_gap":-0.00997222661972047,"absolute_gap":0.00997222661972047,"min_confidence":0.3681107759475708,"max_confidence":0.41203364729881287},{"index":6,"count":6,"confidence":0.4271775633096695,"empirical_correctness":0.5,"signed_gap":-0.0728224366903305,"absolute_gap":0.0728224366903305,"min_confidence":0.41472217440605164,"max_confidence":0.43705591559410095},{"index":7,"count":5,"confidence":0.4584287405014038,"empirical_correctness":0.2,"signed_gap":0.2584287405014038,"absolute_gap":0.2584287405014038,"min_confidence":0.4529012143611908,"max_confidence":0.46352794766426086},{"index":8,"count":6,"confidence":0.49807311594486237,"empirical_correctness":0.0,"signed_gap":0.49807311594486237,"absolute_gap":0.49807311594486237,"min_confidence":0.466327965259552,"max_confidence":0.5701020956039429},{"index":9,"count":6,"confidence":0.6525378028551737,"empirical_correctness":1.0,"signed_gap":-0.3474621971448263,"absolute_gap":0.3474621971448263,"min_confidence":0.5850179195404053,"max_confidence":0.7242468595504761}]}},"latency_ms":{"n":56,"median":58.76794992946088,"p95":60.74334983713925},"per_action":{"keep":{"n":8,"raw":{"n":8,"acceptable":8,"unsafe":0,"review":0,"automated_count":8,"automated_acceptable":8,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":1.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.6755924351161198,"upper":1.0,"confidence_level":0.95}},"effective":{"n":8,"acceptable":5,"unsafe":0,"review":3,"automated_count":5,"automated_acceptable":5,"automated_unsafe":0,"unnecessary_reviews":3,"accuracy":0.625,"unsafe_rate":0.0,"review_rate":0.375,"automated_coverage":0.625,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.375,"accuracy_wilson_95":{"lower":0.30574239460262737,"upper":0.8631557141764026,"confidence_level":0.95}}},"drop":{"n":22,"raw":{"n":22,"acceptable":0,"unsafe":22,"review":0,"automated_count":22,"automated_acceptable":0,"automated_unsafe":22,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":1.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":1.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.14865487460862087,"confidence_level":0.95}},"effective":{"n":22,"acceptable":0,"unsafe":0,"review":22,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":22,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.14865487460862087,"confidence_level":0.95}}},"mask":{"n":8,"raw":{"n":8,"acceptable":7,"unsafe":1,"review":0,"automated_count":8,"automated_acceptable":7,"automated_unsafe":1,"unnecessary_reviews":0,"accuracy":0.875,"unsafe_rate":0.125,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.875,"conditional_unsafe":0.125,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.5291118177871464,"upper":0.9775825085499433,"confidence_level":0.95}},"effective":{"n":8,"acceptable":0,"unsafe":0,"review":8,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":8,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.32440756488388023,"confidence_level":0.95}}},"pseudonymize":{"n":8,"raw":{"n":8,"acceptable":0,"unsafe":8,"review":0,"automated_count":8,"automated_acceptable":0,"automated_unsafe":8,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":1.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":1.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.32440756488388023,"confidence_level":0.95}},"effective":{"n":8,"acceptable":0,"unsafe":0,"review":8,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":8,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.32440756488388023,"confidence_level":0.95}}},"generalize":{"n":8,"raw":{"n":8,"acceptable":0,"unsafe":8,"review":0,"automated_count":8,"automated_acceptable":0,"automated_unsafe":8,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":1.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":1.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.32440756488388023,"confidence_level":0.95}},"effective":{"n":8,"acceptable":0,"unsafe":0,"review":8,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":8,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.32440756488388023,"confidence_level":0.95}}},"review":{"n":2,"raw":{"n":2,"acceptable":0,"unsafe":2,"review":0,"automated_count":2,"automated_acceptable":0,"automated_unsafe":2,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":1.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":1.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.6576197724933469,"confidence_level":0.95}},"effective":{"n":2,"acceptable":2,"unsafe":0,"review":2,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":1.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.34238022750665303,"upper":1.0,"confidence_level":0.95}}}}},"challenge56-compact_no_samples":{"n":56,"raw":{"n":56,"acceptable":15,"unsafe":41,"review":0,"automated_count":56,"automated_acceptable":15,"automated_unsafe":41,"unnecessary_reviews":0,"accuracy":0.26785714285714285,"unsafe_rate":0.7321428571428571,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.26785714285714285,"conditional_unsafe":0.7321428571428571,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.16957305418501728,"upper":0.39594555929155156,"confidence_level":0.95}},"effective":{"n":56,"acceptable":7,"unsafe":0,"review":51,"automated_count":5,"automated_acceptable":5,"automated_unsafe":0,"unnecessary_reviews":49,"accuracy":0.125,"unsafe_rate":0.0,"review_rate":0.9107142857142857,"automated_coverage":0.08928571428571429,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.875,"accuracy_wilson_95":{"lower":0.06189068695717122,"upper":0.2362547655819014,"confidence_level":0.95}},"confusion":{"raw":{"keep":{"keep":8,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0},"drop":{"keep":22,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0},"mask":{"keep":1,"drop":0,"mask":7,"pseudonymize":0,"generalize":0,"review":0},"pseudonymize":{"keep":8,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0},"generalize":{"keep":8,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0},"review":{"keep":2,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0}},"effective":{"keep":{"keep":5,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":3},"drop":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":22},"mask":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":8},"pseudonymize":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":8},"generalize":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":8},"review":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":2}}},"probability_metrics":{"singleton_n":56,"multiacceptable_n":0,"multiacceptable_coverage":0.0,"nll":1.7360263427126232,"nll_zero_mass_n":0,"brier":0.8081227685568704,"acceptable_set_n":56,"acceptable_set_nll":1.7360263427126232,"acceptable_set_zero_mass_n":0},"calibration_all_labels":{"equal_width":{"n":56,"bin_count":10,"ece":0.16449051829321043,"signed_mean_gap":0.11351081675716809,"absolute_mean_gap":0.11351081675716809,"bins":[{"index":0,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.0,"upper":0.1,"upper_inclusive":false},{"index":1,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.1,"upper":0.2,"upper_inclusive":false},{"index":2,"count":10,"confidence":0.27937335073947905,"empirical_correctness":0.1,"signed_gap":0.17937335073947905,"absolute_gap":0.17937335073947905,"lower":0.2,"upper":0.3,"upper_inclusive":false},{"index":3,"count":22,"confidence":0.34888656301931903,"empirical_correctness":0.22727272727272727,"signed_gap":0.12161383574659176,"absolute_gap":0.12161383574659176,"lower":0.3,"upper":0.4,"upper_inclusive":false},{"index":4,"count":19,"confidence":0.43762102566267314,"empirical_correctness":0.2631578947368421,"signed_gap":0.17446313092583104,"absolute_gap":0.17446313092583104,"lower":0.4,"upper":0.5,"upper_inclusive":false},{"index":5,"count":5,"confidence":0.5145136713981628,"empirical_correctness":0.8,"signed_gap":-0.2854863286018372,"absolute_gap":0.2854863286018372,"lower":0.5,"upper":0.6,"upper_inclusive":false},{"index":6,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.6,"upper":0.7,"upper_inclusive":false},{"index":7,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.7,"upper":0.8,"upper_inclusive":false},{"index":8,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.8,"upper":0.9,"upper_inclusive":false},{"index":9,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.9,"upper":1.0,"upper_inclusive":true}]},"equal_mass":{"n":56,"bin_count":10,"ece":0.19181735334651812,"signed_mean_gap":0.11351081675716809,"absolute_mean_gap":0.11351081675716809,"bins":[{"index":0,"count":5,"confidence":0.2733811378479004,"empirical_correctness":0.2,"signed_gap":0.0733811378479004,"absolute_gap":0.0733811378479004,"min_confidence":0.26963385939598083,"max_confidence":0.2760254442691803},{"index":1,"count":6,"confidence":0.28865693509578705,"empirical_correctness":0.16666666666666666,"signed_gap":0.12199026842912039,"absolute_gap":0.12199026842912039,"min_confidence":0.2816216051578522,"max_confidence":0.3051137924194336},{"index":2,"count":5,"confidence":0.31695390939712526,"empirical_correctness":0.0,"signed_gap":0.31695390939712526,"absolute_gap":0.31695390939712526,"min_confidence":0.30709707736968994,"max_confidence":0.32538798451423645},{"index":3,"count":6,"confidence":0.3460551897684733,"empirical_correctness":0.3333333333333333,"signed_gap":0.012721856435140011,"absolute_gap":0.012721856435140011,"min_confidence":0.3386393189430237,"max_confidence":0.35376662015914917},{"index":4,"count":6,"confidence":0.3626822680234909,"empirical_correctness":0.16666666666666666,"signed_gap":0.19601560135682425,"absolute_gap":0.19601560135682425,"min_confidence":0.3589257001876831,"max_confidence":0.3698523938655853},{"index":5,"count":5,"confidence":0.3868449866771698,"empirical_correctness":0.4,"signed_gap":-0.0131550133228302,"absolute_gap":0.0131550133228302,"min_confidence":0.374497652053833,"max_confidence":0.4010286331176758},{"index":6,"count":6,"confidence":0.4099065065383911,"empirical_correctness":0.0,"signed_gap":0.4099065065383911,"absolute_gap":0.4099065065383911,"min_confidence":0.4054449498653412,"max_confidence":0.4137423038482666},{"index":7,"count":5,"confidence":0.4307416260242462,"empirical_correctness":0.0,"signed_gap":0.4307416260242462,"absolute_gap":0.4307416260242462,"min_confidence":0.41435348987579346,"max_confidence":0.4464613199234009},{"index":8,"count":6,"confidence":0.4675792157649994,"empirical_correctness":0.5,"signed_gap":-0.03242078423500061,"absolute_gap":0.03242078423500061,"min_confidence":0.448864221572876,"max_confidence":0.4903547167778015},{"index":9,"count":6,"confidence":0.511286124587059,"empirical_correctness":0.8333333333333334,"signed_gap":-0.32204720874627435,"absolute_gap":0.32204720874627435,"min_confidence":0.4951483905315399,"max_confidence":0.5205215215682983}]}},"calibration_singleton_labels":{"equal_width":{"n":56,"bin_count":10,"ece":0.16449051829321043,"signed_mean_gap":0.11351081675716809,"absolute_mean_gap":0.11351081675716809,"bins":[{"index":0,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.0,"upper":0.1,"upper_inclusive":false},{"index":1,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.1,"upper":0.2,"upper_inclusive":false},{"index":2,"count":10,"confidence":0.27937335073947905,"empirical_correctness":0.1,"signed_gap":0.17937335073947905,"absolute_gap":0.17937335073947905,"lower":0.2,"upper":0.3,"upper_inclusive":false},{"index":3,"count":22,"confidence":0.34888656301931903,"empirical_correctness":0.22727272727272727,"signed_gap":0.12161383574659176,"absolute_gap":0.12161383574659176,"lower":0.3,"upper":0.4,"upper_inclusive":false},{"index":4,"count":19,"confidence":0.43762102566267314,"empirical_correctness":0.2631578947368421,"signed_gap":0.17446313092583104,"absolute_gap":0.17446313092583104,"lower":0.4,"upper":0.5,"upper_inclusive":false},{"index":5,"count":5,"confidence":0.5145136713981628,"empirical_correctness":0.8,"signed_gap":-0.2854863286018372,"absolute_gap":0.2854863286018372,"lower":0.5,"upper":0.6,"upper_inclusive":false},{"index":6,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.6,"upper":0.7,"upper_inclusive":false},{"index":7,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.7,"upper":0.8,"upper_inclusive":false},{"index":8,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.8,"upper":0.9,"upper_inclusive":false},{"index":9,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.9,"upper":1.0,"upper_inclusive":true}]},"equal_mass":{"n":56,"bin_count":10,"ece":0.19181735334651812,"signed_mean_gap":0.11351081675716809,"absolute_mean_gap":0.11351081675716809,"bins":[{"index":0,"count":5,"confidence":0.2733811378479004,"empirical_correctness":0.2,"signed_gap":0.0733811378479004,"absolute_gap":0.0733811378479004,"min_confidence":0.26963385939598083,"max_confidence":0.2760254442691803},{"index":1,"count":6,"confidence":0.28865693509578705,"empirical_correctness":0.16666666666666666,"signed_gap":0.12199026842912039,"absolute_gap":0.12199026842912039,"min_confidence":0.2816216051578522,"max_confidence":0.3051137924194336},{"index":2,"count":5,"confidence":0.31695390939712526,"empirical_correctness":0.0,"signed_gap":0.31695390939712526,"absolute_gap":0.31695390939712526,"min_confidence":0.30709707736968994,"max_confidence":0.32538798451423645},{"index":3,"count":6,"confidence":0.3460551897684733,"empirical_correctness":0.3333333333333333,"signed_gap":0.012721856435140011,"absolute_gap":0.012721856435140011,"min_confidence":0.3386393189430237,"max_confidence":0.35376662015914917},{"index":4,"count":6,"confidence":0.3626822680234909,"empirical_correctness":0.16666666666666666,"signed_gap":0.19601560135682425,"absolute_gap":0.19601560135682425,"min_confidence":0.3589257001876831,"max_confidence":0.3698523938655853},{"index":5,"count":5,"confidence":0.3868449866771698,"empirical_correctness":0.4,"signed_gap":-0.0131550133228302,"absolute_gap":0.0131550133228302,"min_confidence":0.374497652053833,"max_confidence":0.4010286331176758},{"index":6,"count":6,"confidence":0.4099065065383911,"empirical_correctness":0.0,"signed_gap":0.4099065065383911,"absolute_gap":0.4099065065383911,"min_confidence":0.4054449498653412,"max_confidence":0.4137423038482666},{"index":7,"count":5,"confidence":0.4307416260242462,"empirical_correctness":0.0,"signed_gap":0.4307416260242462,"absolute_gap":0.4307416260242462,"min_confidence":0.41435348987579346,"max_confidence":0.4464613199234009},{"index":8,"count":6,"confidence":0.4675792157649994,"empirical_correctness":0.5,"signed_gap":-0.03242078423500061,"absolute_gap":0.03242078423500061,"min_confidence":0.448864221572876,"max_confidence":0.4903547167778015},{"index":9,"count":6,"confidence":0.511286124587059,"empirical_correctness":0.8333333333333334,"signed_gap":-0.32204720874627435,"absolute_gap":0.32204720874627435,"min_confidence":0.4951483905315399,"max_confidence":0.5205215215682983}]}},"latency_ms":{"n":56,"median":68.10260005295277,"p95":69.79295000201091},"per_action":{"keep":{"n":8,"raw":{"n":8,"acceptable":8,"unsafe":0,"review":0,"automated_count":8,"automated_acceptable":8,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":1.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.6755924351161198,"upper":1.0,"confidence_level":0.95}},"effective":{"n":8,"acceptable":5,"unsafe":0,"review":3,"automated_count":5,"automated_acceptable":5,"automated_unsafe":0,"unnecessary_reviews":3,"accuracy":0.625,"unsafe_rate":0.0,"review_rate":0.375,"automated_coverage":0.625,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.375,"accuracy_wilson_95":{"lower":0.30574239460262737,"upper":0.8631557141764026,"confidence_level":0.95}}},"drop":{"n":22,"raw":{"n":22,"acceptable":0,"unsafe":22,"review":0,"automated_count":22,"automated_acceptable":0,"automated_unsafe":22,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":1.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":1.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.14865487460862087,"confidence_level":0.95}},"effective":{"n":22,"acceptable":0,"unsafe":0,"review":22,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":22,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.14865487460862087,"confidence_level":0.95}}},"mask":{"n":8,"raw":{"n":8,"acceptable":7,"unsafe":1,"review":0,"automated_count":8,"automated_acceptable":7,"automated_unsafe":1,"unnecessary_reviews":0,"accuracy":0.875,"unsafe_rate":0.125,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.875,"conditional_unsafe":0.125,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.5291118177871464,"upper":0.9775825085499433,"confidence_level":0.95}},"effective":{"n":8,"acceptable":0,"unsafe":0,"review":8,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":8,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.32440756488388023,"confidence_level":0.95}}},"pseudonymize":{"n":8,"raw":{"n":8,"acceptable":0,"unsafe":8,"review":0,"automated_count":8,"automated_acceptable":0,"automated_unsafe":8,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":1.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":1.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.32440756488388023,"confidence_level":0.95}},"effective":{"n":8,"acceptable":0,"unsafe":0,"review":8,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":8,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.32440756488388023,"confidence_level":0.95}}},"generalize":{"n":8,"raw":{"n":8,"acceptable":0,"unsafe":8,"review":0,"automated_count":8,"automated_acceptable":0,"automated_unsafe":8,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":1.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":1.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.32440756488388023,"confidence_level":0.95}},"effective":{"n":8,"acceptable":0,"unsafe":0,"review":8,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":8,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.32440756488388023,"confidence_level":0.95}}},"review":{"n":2,"raw":{"n":2,"acceptable":0,"unsafe":2,"review":0,"automated_count":2,"automated_acceptable":0,"automated_unsafe":2,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":1.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":1.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.6576197724933469,"confidence_level":0.95}},"effective":{"n":2,"acceptable":2,"unsafe":0,"review":2,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":1.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.34238022750665303,"upper":1.0,"confidence_level":0.95}}}}}},"composite":{"n":198,"scope":"descriptive sum across four correlated panels","raw":{"n":198,"acceptable":57,"unsafe":139,"review":2,"automated_count":196,"automated_acceptable":56,"automated_unsafe":139,"unnecessary_reviews":1,"accuracy":0.2878787878787879,"unsafe_rate":0.702020202020202,"review_rate":0.010101010101010102,"automated_coverage":0.98989898989899,"conditional_correctness":0.2857142857142857,"conditional_unsafe":0.7091836734693877,"unnecessary_review_rate":0.005050505050505051},"effective":{"n":198,"acceptable":28,"unsafe":0,"review":178,"automated_count":20,"automated_acceptable":14,"automated_unsafe":0,"unnecessary_reviews":164,"accuracy":0.1414141414141414,"unsafe_rate":0.0,"review_rate":0.898989898989899,"automated_coverage":0.10101010101010101,"conditional_correctness":0.7,"conditional_unsafe":0.0,"unnecessary_review_rate":0.8282828282828283}}},{"id":"3p3r--sentinel-laya-fp32","platform":"Local","model_id":"3p3r/sentinel-laya","revision":"3efebe799179a16c805eab272bd727000fea6c07","loading":"float32 / none","scoring_kind":"Native decision readout","settings":{"dtype":"float32","quantization":"none","max_tokens":512,"head_max_tokens":256,"temperature":1.2,"autocast":false,"truncation":false,"cpu_offload":false,"guard_threshold":0.45,"seed":0,"tf32":false},"scoring_metadata":{"backend":"laya_native_local_builtin","readout":"native_mask_slot_scorer_numpy_float32_softmax","probability_source":"native_choice_unrounded_published_temperature_not_task_calibrated","latency_scope":"synchronized serial complete encoder+head+act forward,input transfer and native readout; tokenization excluded","native_disclosures":["Stock installed torch attention in place of ONNX dynamic equivalent; no custom kernels or repo imports.","FP32/no-autocast explicit precision: convert every saved F16/BF16/F32 tensor toFP32; no quantization.","Unrounded native probability; guard uses winner p, not entropy-concentration confidence.","Act head fully loaded and evaluated; not used to alter six-action answer.","Native choice:6-10 bucket else choice scalar0, T=1.2. No task fitting, priors correction, prompt change or finetuning here."],"equal_mass_convention":"rank bins with floor boundaries at floor(i*n/k), k=min(10,n//2); minimum two rows"},"load_ms":null,"peak_allocated_bytes":1748759552,"peak_reserved_bytes":1870659584,"panels":{"diagnostic43-no_samples":{"n":43,"raw":{"n":43,"acceptable":19,"unsafe":5,"review":0,"automated_count":43,"automated_acceptable":19,"automated_unsafe":5,"unnecessary_reviews":0,"accuracy":0.4418604651162791,"unsafe_rate":0.11627906976744186,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.4418604651162791,"conditional_unsafe":0.11627906976744186,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.30433309839170514,"upper":0.5889238554717591,"confidence_level":0.95}},"effective":{"n":43,"acceptable":16,"unsafe":0,"review":16,"automated_count":27,"automated_acceptable":15,"automated_unsafe":0,"unnecessary_reviews":15,"accuracy":0.37209302325581395,"unsafe_rate":0.0,"review_rate":0.37209302325581395,"automated_coverage":0.627906976744186,"conditional_correctness":0.5555555555555556,"conditional_unsafe":0.0,"unnecessary_review_rate":0.3488372093023256,"accuracy_wilson_95":{"lower":0.24376339009650572,"upper":0.5214019084031153,"confidence_level":0.95}},"confusion":{"raw":{"keep":{"keep":1,"drop":11,"mask":0,"pseudonymize":0,"generalize":0,"review":0},"drop":{"keep":1,"drop":14,"mask":0,"pseudonymize":1,"generalize":0,"review":0},"mask":{"keep":0,"drop":2,"mask":0,"pseudonymize":0,"generalize":0,"review":0},"pseudonymize":{"keep":0,"drop":2,"mask":0,"pseudonymize":0,"generalize":0,"review":0},"generalize":{"keep":2,"drop":4,"mask":0,"pseudonymize":0,"generalize":0,"review":0},"review":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0}},"effective":{"keep":{"keep":0,"drop":8,"mask":0,"pseudonymize":0,"generalize":0,"review":4},"drop":{"keep":0,"drop":11,"mask":0,"pseudonymize":0,"generalize":0,"review":5},"mask":{"keep":0,"drop":2,"mask":0,"pseudonymize":0,"generalize":0,"review":0},"pseudonymize":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":2},"generalize":{"keep":0,"drop":2,"mask":0,"pseudonymize":0,"generalize":0,"review":4},"review":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0}}},"probability_metrics":{"singleton_n":38,"multiacceptable_n":5,"multiacceptable_coverage":0.11627906976744186,"nll":1.7028591533371316,"nll_zero_mass_n":0,"brier":0.7818131928385681,"acceptable_set_n":43,"acceptable_set_nll":1.540888016092677,"acceptable_set_zero_mass_n":0},"calibration_all_labels":{"equal_width":{"n":43,"bin_count":10,"ece":0.12968219400838366,"signed_mean_gap":0.011911588699318633,"absolute_mean_gap":0.011911588699318633,"bins":[{"index":0,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.0,"upper":0.1,"upper_inclusive":false},{"index":1,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.1,"upper":0.2,"upper_inclusive":false},{"index":2,"count":6,"confidence":0.27326036741336185,"empirical_correctness":0.16666666666666666,"signed_gap":0.10659370074669519,"absolute_gap":0.10659370074669519,"lower":0.2,"upper":0.3,"upper_inclusive":false},{"index":3,"count":4,"confidence":0.3566790446639061,"empirical_correctness":0.0,"signed_gap":0.3566790446639061,"absolute_gap":0.3566790446639061,"lower":0.3,"upper":0.4,"upper_inclusive":false},{"index":4,"count":22,"confidence":0.4589680108157071,"empirical_correctness":0.5,"signed_gap":-0.041031989184292905,"absolute_gap":0.041031989184292905,"lower":0.4,"upper":0.5,"upper_inclusive":false},{"index":5,"count":8,"confidence":0.5463294684886932,"empirical_correctness":0.75,"signed_gap":-0.20367053151130676,"absolute_gap":0.20367053151130676,"lower":0.5,"upper":0.6,"upper_inclusive":false},{"index":6,"count":3,"confidence":0.6593293150266012,"empirical_correctness":0.3333333333333333,"signed_gap":0.3259959816932679,"absolute_gap":0.3259959816932679,"lower":0.6,"upper":0.7,"upper_inclusive":false},{"index":7,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.7,"upper":0.8,"upper_inclusive":false},{"index":8,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.8,"upper":0.9,"upper_inclusive":false},{"index":9,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.9,"upper":1.0,"upper_inclusive":true}]},"equal_mass":{"n":43,"bin_count":10,"ece":0.12217728585697883,"signed_mean_gap":0.011911588699318654,"absolute_mean_gap":0.011911588699318654,"bins":[{"index":0,"count":4,"confidence":0.26553573086857796,"empirical_correctness":0.0,"signed_gap":0.26553573086857796,"absolute_gap":0.26553573086857796,"min_confidence":0.24148674309253693,"max_confidence":0.2800491154193878},{"index":1,"count":4,"confidence":0.3090035691857338,"empirical_correctness":0.25,"signed_gap":0.059003569185733795,"absolute_gap":0.059003569185733795,"min_confidence":0.2807239294052124,"max_confidence":0.3461991548538208},{"index":2,"count":4,"confidence":0.3947373256087303,"empirical_correctness":0.25,"signed_gap":0.14473732560873032,"absolute_gap":0.14473732560873032,"min_confidence":0.3724912106990814,"max_confidence":0.40585845708847046},{"index":3,"count":5,"confidence":0.42895281314849854,"empirical_correctness":0.6,"signed_gap":-0.17104718685150144,"absolute_gap":0.17104718685150144,"min_confidence":0.4166775345802307,"max_confidence":0.4518482983112335},{"index":4,"count":4,"confidence":0.4586286246776581,"empirical_correctness":0.25,"signed_gap":0.20862862467765808,"absolute_gap":0.20862862467765808,"min_confidence":0.4527783989906311,"max_confidence":0.4640515148639679},{"index":5,"count":4,"confidence":0.4703371301293373,"empirical_correctness":0.5,"signed_gap":-0.02966286987066269,"absolute_gap":0.02966286987066269,"min_confidence":0.4663178026676178,"max_confidence":0.4762413799762726},{"index":6,"count":5,"confidence":0.4867241382598877,"empirical_correctness":0.6,"signed_gap":-0.11327586174011228,"absolute_gap":0.11327586174011228,"min_confidence":0.4832504391670227,"max_confidence":0.4899959862232208},{"index":7,"count":4,"confidence":0.49952277541160583,"empirical_correctness":0.5,"signed_gap":-0.00047722458839416504,"absolute_gap":0.00047722458839416504,"min_confidence":0.4931185841560364,"max_confidence":0.5037742853164673},{"index":8,"count":4,"confidence":0.5428657829761505,"empirical_correctness":0.75,"signed_gap":-0.2071342170238495,"absolute_gap":0.2071342170238495,"min_confidence":0.5268810987472534,"max_confidence":0.5816894173622131},{"index":9,"count":5,"confidence":0.6342579603195191,"empirical_correctness":0.6,"signed_gap":0.03425796031951911,"absolute_gap":0.03425796031951911,"min_confidence":0.5962063670158386,"max_confidence":0.6946988105773926}]}},"calibration_singleton_labels":{"equal_width":{"n":38,"bin_count":10,"ece":0.12077729521613374,"signed_mean_gap":0.05968308017442103,"absolute_mean_gap":0.05968308017442103,"bins":[{"index":0,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.0,"upper":0.1,"upper_inclusive":false},{"index":1,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.1,"upper":0.2,"upper_inclusive":false},{"index":2,"count":5,"confidence":0.2719026178121567,"empirical_correctness":0.2,"signed_gap":0.07190261781215668,"absolute_gap":0.07190261781215668,"lower":0.2,"upper":0.3,"upper_inclusive":false},{"index":3,"count":4,"confidence":0.3566790446639061,"empirical_correctness":0.0,"signed_gap":0.3566790446639061,"absolute_gap":0.3566790446639061,"lower":0.3,"upper":0.4,"upper_inclusive":false},{"index":4,"count":19,"confidence":0.45602789050654363,"empirical_correctness":0.42105263157894735,"signed_gap":0.03497525892759629,"absolute_gap":0.03497525892759629,"lower":0.4,"upper":0.5,"upper_inclusive":false},{"index":5,"count":7,"confidence":0.5484585591724941,"empirical_correctness":0.7142857142857143,"signed_gap":-0.16582715511322021,"absolute_gap":0.16582715511322021,"lower":0.5,"upper":0.6,"upper_inclusive":false},{"index":6,"count":3,"confidence":0.6593293150266012,"empirical_correctness":0.3333333333333333,"signed_gap":0.3259959816932679,"absolute_gap":0.3259959816932679,"lower":0.6,"upper":0.7,"upper_inclusive":false},{"index":7,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.7,"upper":0.8,"upper_inclusive":false},{"index":8,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.8,"upper":0.9,"upper_inclusive":false},{"index":9,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.9,"upper":1.0,"upper_inclusive":true}]},"equal_mass":{"n":38,"bin_count":10,"ece":0.14761807612682643,"signed_mean_gap":0.05968308017442101,"absolute_mean_gap":0.05968308017442101,"bins":[{"index":0,"count":3,"confidence":0.260697936018308,"empirical_correctness":0.0,"signed_gap":0.260697936018308,"absolute_gap":0.260697936018308,"min_confidence":0.24148674309253693,"max_confidence":0.2797011435031891},{"index":1,"count":4,"confidence":0.3090035691857338,"empirical_correctness":0.25,"signed_gap":0.059003569185733795,"absolute_gap":0.059003569185733795,"min_confidence":0.2807239294052124,"max_confidence":0.3461991548538208},{"index":2,"count":4,"confidence":0.3947373256087303,"empirical_correctness":0.25,"signed_gap":0.14473732560873032,"absolute_gap":0.14473732560873032,"min_confidence":0.3724912106990814,"max_confidence":0.40585845708847046},{"index":3,"count":4,"confidence":0.4232289418578148,"empirical_correctness":0.5,"signed_gap":-0.07677105814218521,"absolute_gap":0.07677105814218521,"min_confidence":0.4166775345802307,"max_confidence":0.43303194642066956},{"index":4,"count":4,"confidence":0.4586286246776581,"empirical_correctness":0.25,"signed_gap":0.20862862467765808,"absolute_gap":0.20862862467765808,"min_confidence":0.4527783989906311,"max_confidence":0.4640515148639679},{"index":5,"count":3,"confidence":0.46836904684702557,"empirical_correctness":0.6666666666666666,"signed_gap":-0.19829761981964106,"absolute_gap":0.19829761981964106,"min_confidence":0.4663178026676178,"max_confidence":0.4697791635990143},{"index":6,"count":4,"confidence":0.483016662299633,"empirical_correctness":0.25,"signed_gap":0.23301666229963303,"absolute_gap":0.23301666229963303,"min_confidence":0.4762413799762726,"max_confidence":0.4882325232028961},{"index":7,"count":4,"confidence":0.49874212592840195,"empirical_correctness":0.5,"signed_gap":-0.001257874071598053,"absolute_gap":0.001257874071598053,"min_confidence":0.4899959862232208,"max_confidence":0.5037742853164673},{"index":8,"count":4,"confidence":0.5590609163045883,"empirical_correctness":0.75,"signed_gap":-0.19093908369541168,"absolute_gap":0.19093908369541168,"min_confidence":0.5268810987472534,"max_confidence":0.5962063670158386},{"index":9,"count":4,"confidence":0.6437708586454391,"empirical_correctness":0.5,"signed_gap":0.14377085864543915,"absolute_gap":0.14377085864543915,"min_confidence":0.5970954895019531,"max_confidence":0.6946988105773926}]}},"latency_ms":{"n":43,"median":66.67850003577769,"p95":67.74143993388861},"per_action":{"keep":{"n":12,"raw":{"n":12,"acceptable":1,"unsafe":0,"review":0,"automated_count":12,"automated_acceptable":1,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":0.08333333333333333,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.08333333333333333,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.014865094404917095,"upper":0.35387991114111694,"confidence_level":0.95}},"effective":{"n":12,"acceptable":0,"unsafe":0,"review":4,"automated_count":8,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":4,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":0.3333333333333333,"automated_coverage":0.6666666666666666,"conditional_correctness":0.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.3333333333333333,"accuracy_wilson_95":{"lower":2.7755575615628914e-17,"upper":0.24249400665524085,"confidence_level":0.95}}},"drop":{"n":16,"raw":{"n":16,"acceptable":14,"unsafe":2,"review":0,"automated_count":16,"automated_acceptable":14,"automated_unsafe":2,"unnecessary_reviews":0,"accuracy":0.875,"unsafe_rate":0.125,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.875,"conditional_unsafe":0.125,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.639771727342413,"upper":0.9650225122567595,"confidence_level":0.95}},"effective":{"n":16,"acceptable":11,"unsafe":0,"review":5,"automated_count":11,"automated_acceptable":11,"automated_unsafe":0,"unnecessary_reviews":5,"accuracy":0.6875,"unsafe_rate":0.0,"review_rate":0.3125,"automated_coverage":0.6875,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.3125,"accuracy_wilson_95":{"lower":0.44404355834740666,"upper":0.8583535614521796,"confidence_level":0.95}}},"mask":{"n":2,"raw":{"n":2,"acceptable":0,"unsafe":0,"review":0,"automated_count":2,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.6576197724933469,"confidence_level":0.95}},"effective":{"n":2,"acceptable":0,"unsafe":0,"review":0,"automated_count":2,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.6576197724933469,"confidence_level":0.95}}},"pseudonymize":{"n":2,"raw":{"n":2,"acceptable":0,"unsafe":0,"review":0,"automated_count":2,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.6576197724933469,"confidence_level":0.95}},"effective":{"n":2,"acceptable":0,"unsafe":0,"review":2,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":2,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.6576197724933469,"confidence_level":0.95}}},"generalize":{"n":6,"raw":{"n":6,"acceptable":0,"unsafe":2,"review":0,"automated_count":6,"automated_acceptable":0,"automated_unsafe":2,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":0.3333333333333333,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":0.3333333333333333,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":2.7755575615628914e-17,"upper":0.3903342879021653,"confidence_level":0.95}},"effective":{"n":6,"acceptable":0,"unsafe":0,"review":4,"automated_count":2,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":4,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":0.6666666666666666,"automated_coverage":0.3333333333333333,"conditional_correctness":0.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.6666666666666666,"accuracy_wilson_95":{"lower":2.7755575615628914e-17,"upper":0.3903342879021653,"confidence_level":0.95}}},"review":{"n":0,"raw":{"n":0,"acceptable":0,"unsafe":0,"review":0,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":null,"unsafe_rate":null,"review_rate":null,"automated_coverage":null,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":null},"effective":{"n":0,"acceptable":0,"unsafe":0,"review":0,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":null,"unsafe_rate":null,"review_rate":null,"automated_coverage":null,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":null}}}},"diagnostic43-compact_no_samples":{"n":43,"raw":{"n":43,"acceptable":17,"unsafe":10,"review":0,"automated_count":43,"automated_acceptable":17,"automated_unsafe":10,"unnecessary_reviews":0,"accuracy":0.3953488372093023,"unsafe_rate":0.23255813953488372,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.3953488372093023,"conditional_unsafe":0.23255813953488372,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.2636531475829744,"upper":0.5442093693712609,"confidence_level":0.95}},"effective":{"n":43,"acceptable":5,"unsafe":0,"review":43,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":38,"accuracy":0.11627906976744186,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":0.8837209302325582,"accuracy_wilson_95":{"lower":0.05070427739804166,"upper":0.24479161810082128,"confidence_level":0.95}},"confusion":{"raw":{"keep":{"keep":4,"drop":0,"mask":8,"pseudonymize":0,"generalize":0,"review":0},"drop":{"keep":4,"drop":7,"mask":5,"pseudonymize":0,"generalize":0,"review":0},"mask":{"keep":0,"drop":0,"mask":2,"pseudonymize":0,"generalize":0,"review":0},"pseudonymize":{"keep":2,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0},"generalize":{"keep":3,"drop":1,"mask":2,"pseudonymize":0,"generalize":0,"review":0},"review":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0}},"effective":{"keep":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":12},"drop":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":16},"mask":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":2},"pseudonymize":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":2},"generalize":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":6},"review":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0}}},"probability_metrics":{"singleton_n":38,"multiacceptable_n":5,"multiacceptable_coverage":0.11627906976744186,"nll":1.5603717351847497,"nll_zero_mass_n":0,"brier":0.7585964387745139,"acceptable_set_n":43,"acceptable_set_nll":1.4413918565353931,"acceptable_set_zero_mass_n":0},"calibration_all_labels":{"equal_width":{"n":43,"bin_count":10,"ece":0.07891950773638347,"signed_mean_gap":-0.07891950773638347,"absolute_mean_gap":0.07891950773638347,"bins":[{"index":0,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.0,"upper":0.1,"upper_inclusive":false},{"index":1,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.1,"upper":0.2,"upper_inclusive":false},{"index":2,"count":16,"confidence":0.2831320073455572,"empirical_correctness":0.4375,"signed_gap":-0.1543679926544428,"absolute_gap":0.1543679926544428,"lower":0.2,"upper":0.3,"upper_inclusive":false},{"index":3,"count":27,"confidence":0.33616107591876276,"empirical_correctness":0.37037037037037035,"signed_gap":-0.034209294451607586,"absolute_gap":0.034209294451607586,"lower":0.3,"upper":0.4,"upper_inclusive":false},{"index":4,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.4,"upper":0.5,"upper_inclusive":false},{"index":5,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.5,"upper":0.6,"upper_inclusive":false},{"index":6,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.6,"upper":0.7,"upper_inclusive":false},{"index":7,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.7,"upper":0.8,"upper_inclusive":false},{"index":8,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.8,"upper":0.9,"upper_inclusive":false},{"index":9,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.9,"upper":1.0,"upper_inclusive":true}]},"equal_mass":{"n":43,"bin_count":10,"ece":0.243740928727527,"signed_mean_gap":-0.07891950773638348,"absolute_mean_gap":0.07891950773638348,"bins":[{"index":0,"count":4,"confidence":0.26743775606155396,"empirical_correctness":0.25,"signed_gap":0.017437756061553955,"absolute_gap":0.017437756061553955,"min_confidence":0.26123395562171936,"max_confidence":0.27557694911956787},{"index":1,"count":4,"confidence":0.28134188055992126,"empirical_correctness":0.75,"signed_gap":-0.46865811944007874,"absolute_gap":0.46865811944007874,"min_confidence":0.28059816360473633,"max_confidence":0.2830214500427246},{"index":2,"count":4,"confidence":0.286803737282753,"empirical_correctness":0.25,"signed_gap":0.03680373728275299,"absolute_gap":0.03680373728275299,"min_confidence":0.28365960717201233,"max_confidence":0.2907921075820923},{"index":3,"count":5,"confidence":0.2976440727710724,"empirical_correctness":0.4,"signed_gap":-0.10235592722892761,"absolute_gap":0.10235592722892761,"min_confidence":0.29446345567703247,"max_confidence":0.3004417419433594},{"index":4,"count":4,"confidence":0.3091326877474785,"empirical_correctness":0.5,"signed_gap":-0.19086731225252151,"absolute_gap":0.19086731225252151,"min_confidence":0.3014293909072876,"max_confidence":0.3183530867099762},{"index":5,"count":4,"confidence":0.32199203968048096,"empirical_correctness":0.25,"signed_gap":0.07199203968048096,"absolute_gap":0.07199203968048096,"min_confidence":0.3193485736846924,"max_confidence":0.3271808922290802},{"index":6,"count":5,"confidence":0.333583277463913,"empirical_correctness":0.0,"signed_gap":0.333583277463913,"absolute_gap":0.333583277463913,"min_confidence":0.32962051033973694,"max_confidence":0.33685508370399475},{"index":7,"count":4,"confidence":0.3427025079727173,"empirical_correctness":0.0,"signed_gap":0.3427025079727173,"absolute_gap":0.3427025079727173,"min_confidence":0.3400880694389343,"max_confidence":0.34457772970199585},{"index":8,"count":4,"confidence":0.34928061813116074,"empirical_correctness":0.5,"signed_gap":-0.15071938186883926,"absolute_gap":0.15071938186883926,"min_confidence":0.3447355329990387,"max_confidence":0.3551277220249176},{"index":9,"count":5,"confidence":0.36311190128326415,"empirical_correctness":1.0,"signed_gap":-0.6368880987167358,"absolute_gap":0.6368880987167358,"min_confidence":0.35703521966934204,"max_confidence":0.37712883949279785}]}},"calibration_singleton_labels":{"equal_width":{"n":38,"bin_count":10,"ece":0.029353428828088862,"signed_mean_gap":-0.025854223652889852,"absolute_mean_gap":0.025854223652889852,"bins":[{"index":0,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.0,"upper":0.1,"upper_inclusive":false},{"index":1,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.1,"upper":0.2,"upper_inclusive":false},{"index":2,"count":14,"confidence":0.28221818591867176,"empirical_correctness":0.35714285714285715,"signed_gap":-0.07492467122418539,"absolute_gap":0.07492467122418539,"lower":0.2,"upper":0.3,"upper_inclusive":false},{"index":3,"count":24,"confidence":0.33610353743036586,"empirical_correctness":0.3333333333333333,"signed_gap":0.002770204097032547,"absolute_gap":0.002770204097032547,"lower":0.3,"upper":0.4,"upper_inclusive":false},{"index":4,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.4,"upper":0.5,"upper_inclusive":false},{"index":5,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.5,"upper":0.6,"upper_inclusive":false},{"index":6,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.6,"upper":0.7,"upper_inclusive":false},{"index":7,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.7,"upper":0.8,"upper_inclusive":false},{"index":8,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.8,"upper":0.9,"upper_inclusive":false},{"index":9,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.9,"upper":1.0,"upper_inclusive":true}]},"equal_mass":{"n":38,"bin_count":10,"ece":0.23704513280015244,"signed_mean_gap":-0.025854223652889852,"absolute_mean_gap":0.025854223652889852,"bins":[{"index":0,"count":3,"confidence":0.26472469170888263,"empirical_correctness":0.3333333333333333,"signed_gap":-0.06860864162445068,"absolute_gap":0.06860864162445068,"min_confidence":0.26123395562171936,"max_confidence":0.27022361755371094},{"index":1,"count":4,"confidence":0.28008657693862915,"empirical_correctness":0.5,"signed_gap":-0.21991342306137085,"absolute_gap":0.21991342306137085,"min_confidence":0.27557694911956787,"max_confidence":0.2830214500427246},{"index":2,"count":4,"confidence":0.286803737282753,"empirical_correctness":0.25,"signed_gap":0.03680373728275299,"absolute_gap":0.03680373728275299,"min_confidence":0.28365960717201233,"max_confidence":0.2907921075820923},{"index":3,"count":4,"confidence":0.2974402531981468,"empirical_correctness":0.25,"signed_gap":0.04744025319814682,"absolute_gap":0.04744025319814682,"min_confidence":0.29446345567703247,"max_confidence":0.3004417419433594},{"index":4,"count":4,"confidence":0.30938155949115753,"empirical_correctness":0.5,"signed_gap":-0.19061844050884247,"absolute_gap":0.19061844050884247,"min_confidence":0.3014293909072876,"max_confidence":0.3193485736846924},{"index":5,"count":3,"confidence":0.32287319501241046,"empirical_correctness":0.0,"signed_gap":0.32287319501241046,"absolute_gap":0.32287319501241046,"min_confidence":0.3201994299888611,"max_confidence":0.3271808922290802},{"index":6,"count":4,"confidence":0.33405542373657227,"empirical_correctness":0.0,"signed_gap":0.33405542373657227,"absolute_gap":0.33405542373657227,"min_confidence":0.32962051033973694,"max_confidence":0.33685508370399475},{"index":7,"count":4,"confidence":0.3427025079727173,"empirical_correctness":0.0,"signed_gap":0.3427025079727173,"absolute_gap":0.3427025079727173,"min_confidence":0.3400880694389343,"max_confidence":0.34457772970199585},{"index":8,"count":4,"confidence":0.34928061813116074,"empirical_correctness":0.5,"signed_gap":-0.15071938186883926,"absolute_gap":0.15071938186883926,"min_confidence":0.3447355329990387,"max_confidence":0.3551277220249176},{"index":9,"count":4,"confidence":0.36393578350543976,"empirical_correctness":1.0,"signed_gap":-0.6360642164945602,"absolute_gap":0.6360642164945602,"min_confidence":0.35703521966934204,"max_confidence":0.37712883949279785}]}},"latency_ms":{"n":43,"median":77.09219981916249,"p95":79.48938005138189},"per_action":{"keep":{"n":12,"raw":{"n":12,"acceptable":4,"unsafe":0,"review":0,"automated_count":12,"automated_acceptable":4,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":0.3333333333333333,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.3333333333333333,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.13812009109121315,"upper":0.6093779111272004,"confidence_level":0.95}},"effective":{"n":12,"acceptable":0,"unsafe":0,"review":12,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":12,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":2.7755575615628914e-17,"upper":0.24249400665524085,"confidence_level":0.95}}},"drop":{"n":16,"raw":{"n":16,"acceptable":7,"unsafe":4,"review":0,"automated_count":16,"automated_acceptable":7,"automated_unsafe":4,"unnecessary_reviews":0,"accuracy":0.4375,"unsafe_rate":0.25,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.4375,"conditional_unsafe":0.25,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.23098652405492354,"upper":0.6682144360118811,"confidence_level":0.95}},"effective":{"n":16,"acceptable":0,"unsafe":0,"review":16,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":16,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.1936076805344365,"confidence_level":0.95}}},"mask":{"n":2,"raw":{"n":2,"acceptable":2,"unsafe":0,"review":0,"automated_count":2,"automated_acceptable":2,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":1.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.34238022750665303,"upper":1.0,"confidence_level":0.95}},"effective":{"n":2,"acceptable":0,"unsafe":0,"review":2,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":2,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.6576197724933469,"confidence_level":0.95}}},"pseudonymize":{"n":2,"raw":{"n":2,"acceptable":0,"unsafe":2,"review":0,"automated_count":2,"automated_acceptable":0,"automated_unsafe":2,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":1.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":1.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.6576197724933469,"confidence_level":0.95}},"effective":{"n":2,"acceptable":0,"unsafe":0,"review":2,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":2,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.6576197724933469,"confidence_level":0.95}}},"generalize":{"n":6,"raw":{"n":6,"acceptable":0,"unsafe":3,"review":0,"automated_count":6,"automated_acceptable":0,"automated_unsafe":3,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":0.5,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":0.5,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":2.7755575615628914e-17,"upper":0.3903342879021653,"confidence_level":0.95}},"effective":{"n":6,"acceptable":0,"unsafe":0,"review":6,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":6,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":2.7755575615628914e-17,"upper":0.3903342879021653,"confidence_level":0.95}}},"review":{"n":0,"raw":{"n":0,"acceptable":0,"unsafe":0,"review":0,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":null,"unsafe_rate":null,"review_rate":null,"automated_coverage":null,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":null},"effective":{"n":0,"acceptable":0,"unsafe":0,"review":0,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":null,"unsafe_rate":null,"review_rate":null,"automated_coverage":null,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":null}}}},"challenge56-no_samples":{"n":56,"raw":{"n":56,"acceptable":29,"unsafe":16,"review":0,"automated_count":56,"automated_acceptable":29,"automated_unsafe":16,"unnecessary_reviews":0,"accuracy":0.5178571428571429,"unsafe_rate":0.2857142857142857,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.5178571428571429,"conditional_unsafe":0.2857142857142857,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.39010383924176983,"upper":0.6433178058754173,"confidence_level":0.95}},"effective":{"n":56,"acceptable":13,"unsafe":0,"review":41,"automated_count":15,"automated_acceptable":13,"automated_unsafe":0,"unnecessary_reviews":41,"accuracy":0.23214285714285715,"unsafe_rate":0.0,"review_rate":0.7321428571428571,"automated_coverage":0.26785714285714285,"conditional_correctness":0.8666666666666667,"conditional_unsafe":0.0,"unnecessary_review_rate":0.7321428571428571,"accuracy_wilson_95":{"lower":0.1409937910920862,"upper":0.35768153215010856,"confidence_level":0.95}},"confusion":{"raw":{"keep":{"keep":8,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0},"drop":{"keep":8,"drop":13,"mask":1,"pseudonymize":0,"generalize":0,"review":0},"mask":{"keep":0,"drop":0,"mask":8,"pseudonymize":0,"generalize":0,"review":0},"pseudonymize":{"keep":0,"drop":0,"mask":8,"pseudonymize":0,"generalize":0,"review":0},"generalize":{"keep":8,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0},"review":{"keep":0,"drop":0,"mask":2,"pseudonymize":0,"generalize":0,"review":0}},"effective":{"keep":{"keep":5,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":3},"drop":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":22},"mask":{"keep":0,"drop":0,"mask":8,"pseudonymize":0,"generalize":0,"review":0},"pseudonymize":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":8},"generalize":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":8},"review":{"keep":0,"drop":0,"mask":2,"pseudonymize":0,"generalize":0,"review":0}}},"probability_metrics":{"singleton_n":56,"multiacceptable_n":0,"multiacceptable_coverage":0.0,"nll":1.3261361734266583,"nll_zero_mass_n":0,"brier":0.6277596728974405,"acceptable_set_n":56,"acceptable_set_nll":1.3261361734266583,"acceptable_set_zero_mass_n":0},"calibration_all_labels":{"equal_width":{"n":56,"bin_count":10,"ece":0.13328999520412513,"signed_mean_gap":-0.13328999520412513,"absolute_mean_gap":0.13328999520412513,"bins":[{"index":0,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.0,"upper":0.1,"upper_inclusive":false},{"index":1,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.1,"upper":0.2,"upper_inclusive":false},{"index":2,"count":16,"confidence":0.25136232282966375,"empirical_correctness":0.375,"signed_gap":-0.12363767717033625,"absolute_gap":0.12363767717033625,"lower":0.2,"upper":0.3,"upper_inclusive":false},{"index":3,"count":20,"confidence":0.3359400391578674,"empirical_correctness":0.35,"signed_gap":-0.014059960842132568,"absolute_gap":0.014059960842132568,"lower":0.3,"upper":0.4,"upper_inclusive":false},{"index":4,"count":6,"confidence":0.4420020480950673,"empirical_correctness":0.6666666666666666,"signed_gap":-0.2246646185715993,"absolute_gap":0.2246646185715993,"lower":0.4,"upper":0.5,"upper_inclusive":false},{"index":5,"count":9,"confidence":0.5572725931803385,"empirical_correctness":0.7777777777777778,"signed_gap":-0.22050518459743929,"absolute_gap":0.22050518459743929,"lower":0.5,"upper":0.6,"upper_inclusive":false},{"index":6,"count":5,"confidence":0.6255393385887146,"empirical_correctness":1.0,"signed_gap":-0.3744606614112854,"absolute_gap":0.3744606614112854,"lower":0.6,"upper":0.7,"upper_inclusive":false},{"index":7,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.7,"upper":0.8,"upper_inclusive":false},{"index":8,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.8,"upper":0.9,"upper_inclusive":false},{"index":9,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.9,"upper":1.0,"upper_inclusive":true}]},"equal_mass":{"n":56,"bin_count":10,"ece":0.25718202468539986,"signed_mean_gap":-0.13328999520412513,"absolute_mean_gap":0.13328999520412513,"bins":[{"index":0,"count":5,"confidence":0.22561496496200562,"empirical_correctness":0.0,"signed_gap":0.22561496496200562,"absolute_gap":0.22561496496200562,"min_confidence":0.20858989655971527,"max_confidence":0.23391275107860565},{"index":1,"count":6,"confidence":0.24941513190666834,"empirical_correctness":0.6666666666666666,"signed_gap":-0.4172515347599983,"absolute_gap":0.4172515347599983,"min_confidence":0.23440338671207428,"max_confidence":0.2549154460430145},{"index":2,"count":5,"confidence":0.27944630980491636,"empirical_correctness":0.4,"signed_gap":-0.12055369019508366,"absolute_gap":0.12055369019508366,"min_confidence":0.25883159041404724,"max_confidence":0.29885998368263245},{"index":3,"count":6,"confidence":0.3136574824651082,"empirical_correctness":0.5,"signed_gap":-0.18634251753489178,"absolute_gap":0.18634251753489178,"min_confidence":0.30016541481018066,"max_confidence":0.32098647952079773},{"index":4,"count":6,"confidence":0.3272889703512192,"empirical_correctness":0.0,"signed_gap":0.3272889703512192,"absolute_gap":0.3272889703512192,"min_confidence":0.3242875933647156,"max_confidence":0.32950669527053833},{"index":5,"count":5,"confidence":0.3452047348022461,"empirical_correctness":0.6,"signed_gap":-0.25479526519775386,"absolute_gap":0.25479526519775386,"min_confidence":0.32970693707466125,"max_confidence":0.3589657247066498},{"index":6,"count":6,"confidence":0.3961946964263916,"empirical_correctness":0.3333333333333333,"signed_gap":0.06286136309305829,"absolute_gap":0.06286136309305829,"min_confidence":0.3689633905887604,"max_confidence":0.41739127039909363},{"index":7,"count":5,"confidence":0.4924538195133209,"empirical_correctness":1.0,"signed_gap":-0.5075461804866791,"absolute_gap":0.5075461804866791,"min_confidence":0.42841073870658875,"max_confidence":0.5377936363220215},{"index":8,"count":6,"confidence":0.5633061627546946,"empirical_correctness":0.6666666666666666,"signed_gap":-0.10336050391197205,"absolute_gap":0.10336050391197205,"min_confidence":0.5390297174453735,"max_confidence":0.5831838846206665},{"index":9,"count":6,"confidence":0.6204977432886759,"empirical_correctness":1.0,"signed_gap":-0.3795022567113241,"absolute_gap":0.3795022567113241,"min_confidence":0.5952897667884827,"max_confidence":0.6408166289329529}]}},"calibration_singleton_labels":{"equal_width":{"n":56,"bin_count":10,"ece":0.13328999520412513,"signed_mean_gap":-0.13328999520412513,"absolute_mean_gap":0.13328999520412513,"bins":[{"index":0,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.0,"upper":0.1,"upper_inclusive":false},{"index":1,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.1,"upper":0.2,"upper_inclusive":false},{"index":2,"count":16,"confidence":0.25136232282966375,"empirical_correctness":0.375,"signed_gap":-0.12363767717033625,"absolute_gap":0.12363767717033625,"lower":0.2,"upper":0.3,"upper_inclusive":false},{"index":3,"count":20,"confidence":0.3359400391578674,"empirical_correctness":0.35,"signed_gap":-0.014059960842132568,"absolute_gap":0.014059960842132568,"lower":0.3,"upper":0.4,"upper_inclusive":false},{"index":4,"count":6,"confidence":0.4420020480950673,"empirical_correctness":0.6666666666666666,"signed_gap":-0.2246646185715993,"absolute_gap":0.2246646185715993,"lower":0.4,"upper":0.5,"upper_inclusive":false},{"index":5,"count":9,"confidence":0.5572725931803385,"empirical_correctness":0.7777777777777778,"signed_gap":-0.22050518459743929,"absolute_gap":0.22050518459743929,"lower":0.5,"upper":0.6,"upper_inclusive":false},{"index":6,"count":5,"confidence":0.6255393385887146,"empirical_correctness":1.0,"signed_gap":-0.3744606614112854,"absolute_gap":0.3744606614112854,"lower":0.6,"upper":0.7,"upper_inclusive":false},{"index":7,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.7,"upper":0.8,"upper_inclusive":false},{"index":8,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.8,"upper":0.9,"upper_inclusive":false},{"index":9,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.9,"upper":1.0,"upper_inclusive":true}]},"equal_mass":{"n":56,"bin_count":10,"ece":0.25718202468539986,"signed_mean_gap":-0.13328999520412513,"absolute_mean_gap":0.13328999520412513,"bins":[{"index":0,"count":5,"confidence":0.22561496496200562,"empirical_correctness":0.0,"signed_gap":0.22561496496200562,"absolute_gap":0.22561496496200562,"min_confidence":0.20858989655971527,"max_confidence":0.23391275107860565},{"index":1,"count":6,"confidence":0.24941513190666834,"empirical_correctness":0.6666666666666666,"signed_gap":-0.4172515347599983,"absolute_gap":0.4172515347599983,"min_confidence":0.23440338671207428,"max_confidence":0.2549154460430145},{"index":2,"count":5,"confidence":0.27944630980491636,"empirical_correctness":0.4,"signed_gap":-0.12055369019508366,"absolute_gap":0.12055369019508366,"min_confidence":0.25883159041404724,"max_confidence":0.29885998368263245},{"index":3,"count":6,"confidence":0.3136574824651082,"empirical_correctness":0.5,"signed_gap":-0.18634251753489178,"absolute_gap":0.18634251753489178,"min_confidence":0.30016541481018066,"max_confidence":0.32098647952079773},{"index":4,"count":6,"confidence":0.3272889703512192,"empirical_correctness":0.0,"signed_gap":0.3272889703512192,"absolute_gap":0.3272889703512192,"min_confidence":0.3242875933647156,"max_confidence":0.32950669527053833},{"index":5,"count":5,"confidence":0.3452047348022461,"empirical_correctness":0.6,"signed_gap":-0.25479526519775386,"absolute_gap":0.25479526519775386,"min_confidence":0.32970693707466125,"max_confidence":0.3589657247066498},{"index":6,"count":6,"confidence":0.3961946964263916,"empirical_correctness":0.3333333333333333,"signed_gap":0.06286136309305829,"absolute_gap":0.06286136309305829,"min_confidence":0.3689633905887604,"max_confidence":0.41739127039909363},{"index":7,"count":5,"confidence":0.4924538195133209,"empirical_correctness":1.0,"signed_gap":-0.5075461804866791,"absolute_gap":0.5075461804866791,"min_confidence":0.42841073870658875,"max_confidence":0.5377936363220215},{"index":8,"count":6,"confidence":0.5633061627546946,"empirical_correctness":0.6666666666666666,"signed_gap":-0.10336050391197205,"absolute_gap":0.10336050391197205,"min_confidence":0.5390297174453735,"max_confidence":0.5831838846206665},{"index":9,"count":6,"confidence":0.6204977432886759,"empirical_correctness":1.0,"signed_gap":-0.3795022567113241,"absolute_gap":0.3795022567113241,"min_confidence":0.5952897667884827,"max_confidence":0.6408166289329529}]}},"latency_ms":{"n":56,"median":58.06389998178929,"p95":60.57540007168427},"per_action":{"keep":{"n":8,"raw":{"n":8,"acceptable":8,"unsafe":0,"review":0,"automated_count":8,"automated_acceptable":8,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":1.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.6755924351161198,"upper":1.0,"confidence_level":0.95}},"effective":{"n":8,"acceptable":5,"unsafe":0,"review":3,"automated_count":5,"automated_acceptable":5,"automated_unsafe":0,"unnecessary_reviews":3,"accuracy":0.625,"unsafe_rate":0.0,"review_rate":0.375,"automated_coverage":0.625,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.375,"accuracy_wilson_95":{"lower":0.30574239460262737,"upper":0.8631557141764026,"confidence_level":0.95}}},"drop":{"n":22,"raw":{"n":22,"acceptable":13,"unsafe":8,"review":0,"automated_count":22,"automated_acceptable":13,"automated_unsafe":8,"unnecessary_reviews":0,"accuracy":0.5909090909090909,"unsafe_rate":0.36363636363636365,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.5909090909090909,"conditional_unsafe":0.36363636363636365,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.3873481847631513,"upper":0.7674418380352814,"confidence_level":0.95}},"effective":{"n":22,"acceptable":0,"unsafe":0,"review":22,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":22,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.14865487460862087,"confidence_level":0.95}}},"mask":{"n":8,"raw":{"n":8,"acceptable":8,"unsafe":0,"review":0,"automated_count":8,"automated_acceptable":8,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":1.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.6755924351161198,"upper":1.0,"confidence_level":0.95}},"effective":{"n":8,"acceptable":8,"unsafe":0,"review":0,"automated_count":8,"automated_acceptable":8,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":1.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.6755924351161198,"upper":1.0,"confidence_level":0.95}}},"pseudonymize":{"n":8,"raw":{"n":8,"acceptable":0,"unsafe":0,"review":0,"automated_count":8,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.32440756488388023,"confidence_level":0.95}},"effective":{"n":8,"acceptable":0,"unsafe":0,"review":8,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":8,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.32440756488388023,"confidence_level":0.95}}},"generalize":{"n":8,"raw":{"n":8,"acceptable":0,"unsafe":8,"review":0,"automated_count":8,"automated_acceptable":0,"automated_unsafe":8,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":1.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":1.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.32440756488388023,"confidence_level":0.95}},"effective":{"n":8,"acceptable":0,"unsafe":0,"review":8,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":8,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.32440756488388023,"confidence_level":0.95}}},"review":{"n":2,"raw":{"n":2,"acceptable":0,"unsafe":0,"review":0,"automated_count":2,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.6576197724933469,"confidence_level":0.95}},"effective":{"n":2,"acceptable":0,"unsafe":0,"review":0,"automated_count":2,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.6576197724933469,"confidence_level":0.95}}}}},"challenge56-compact_no_samples":{"n":56,"raw":{"n":56,"acceptable":14,"unsafe":28,"review":0,"automated_count":56,"automated_acceptable":14,"automated_unsafe":28,"unnecessary_reviews":0,"accuracy":0.25,"unsafe_rate":0.5,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.25,"conditional_unsafe":0.5,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.15517054688270684,"upper":0.376926421476675,"confidence_level":0.95}},"effective":{"n":56,"acceptable":11,"unsafe":0,"review":47,"automated_count":9,"automated_acceptable":9,"automated_unsafe":0,"unnecessary_reviews":45,"accuracy":0.19642857142857142,"unsafe_rate":0.0,"review_rate":0.8392857142857143,"automated_coverage":0.16071428571428573,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.8035714285714286,"accuracy_wilson_95":{"lower":0.11338595674323658,"upper":0.31844607626458404,"confidence_level":0.95}},"confusion":{"raw":{"keep":{"keep":6,"drop":0,"mask":2,"pseudonymize":0,"generalize":0,"review":0},"drop":{"keep":20,"drop":0,"mask":2,"pseudonymize":0,"generalize":0,"review":0},"mask":{"keep":0,"drop":0,"mask":8,"pseudonymize":0,"generalize":0,"review":0},"pseudonymize":{"keep":0,"drop":0,"mask":8,"pseudonymize":0,"generalize":0,"review":0},"generalize":{"keep":8,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0},"review":{"keep":0,"drop":0,"mask":2,"pseudonymize":0,"generalize":0,"review":0}},"effective":{"keep":{"keep":1,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":7},"drop":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":22},"mask":{"keep":0,"drop":0,"mask":8,"pseudonymize":0,"generalize":0,"review":0},"pseudonymize":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":8},"generalize":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":8},"review":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":2}}},"probability_metrics":{"singleton_n":56,"multiacceptable_n":0,"multiacceptable_coverage":0.0,"nll":1.5383435715448694,"nll_zero_mass_n":0,"brier":0.731814500111939,"acceptable_set_n":56,"acceptable_set_nll":1.5383435715448694,"acceptable_set_zero_mass_n":0},"calibration_all_labels":{"equal_width":{"n":56,"bin_count":10,"ece":0.28176795318722725,"signed_mean_gap":0.10402677900024823,"absolute_mean_gap":0.10402677900024823,"bins":[{"index":0,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.0,"upper":0.1,"upper_inclusive":false},{"index":1,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.1,"upper":0.2,"upper_inclusive":false},{"index":2,"count":24,"confidence":0.27038734778761864,"empirical_correctness":0.0,"signed_gap":0.27038734778761864,"absolute_gap":0.27038734778761864,"lower":0.2,"upper":0.3,"upper_inclusive":false},{"index":3,"count":13,"confidence":0.33176585802665126,"empirical_correctness":0.0,"signed_gap":0.33176585802665126,"absolute_gap":0.33176585802665126,"lower":0.3,"upper":0.4,"upper_inclusive":false},{"index":4,"count":12,"confidence":0.4418666909138362,"empirical_correctness":0.5833333333333334,"signed_gap":-0.1414666424194972,"absolute_gap":0.1414666424194972,"lower":0.4,"upper":0.5,"upper_inclusive":false},{"index":5,"count":7,"confidence":0.5315495473997933,"empirical_correctness":1.0,"signed_gap":-0.4684504526002067,"absolute_gap":0.4684504526002067,"lower":0.5,"upper":0.6,"upper_inclusive":false},{"index":6,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.6,"upper":0.7,"upper_inclusive":false},{"index":7,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.7,"upper":0.8,"upper_inclusive":false},{"index":8,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.8,"upper":0.9,"upper_inclusive":false},{"index":9,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.9,"upper":1.0,"upper_inclusive":true}]},"equal_mass":{"n":56,"bin_count":10,"ece":0.3177080319396087,"signed_mean_gap":0.10402677900024823,"absolute_mean_gap":0.10402677900024823,"bins":[{"index":0,"count":5,"confidence":0.2503814518451691,"empirical_correctness":0.0,"signed_gap":0.2503814518451691,"absolute_gap":0.2503814518451691,"min_confidence":0.23836177587509155,"max_confidence":0.25748124718666077},{"index":1,"count":6,"confidence":0.26295000314712524,"empirical_correctness":0.0,"signed_gap":0.26295000314712524,"absolute_gap":0.26295000314712524,"min_confidence":0.257655531167984,"max_confidence":0.2676897644996643},{"index":2,"count":5,"confidence":0.2713229119777679,"empirical_correctness":0.0,"signed_gap":0.2713229119777679,"absolute_gap":0.2713229119777679,"min_confidence":0.26845046877861023,"max_confidence":0.27671703696250916},{"index":3,"count":6,"confidence":0.2845223546028137,"empirical_correctness":0.0,"signed_gap":0.2845223546028137,"absolute_gap":0.2845223546028137,"min_confidence":0.281251460313797,"max_confidence":0.2870544493198395},{"index":4,"count":6,"confidence":0.30246814092000324,"empirical_correctness":0.0,"signed_gap":0.30246814092000324,"absolute_gap":0.30246814092000324,"min_confidence":0.2974095940589905,"max_confidence":0.308126837015152},{"index":5,"count":5,"confidence":0.3170983076095581,"empirical_correctness":0.0,"signed_gap":0.3170983076095581,"absolute_gap":0.3170983076095581,"min_confidence":0.31013497710227966,"max_confidence":0.330527126789093},{"index":6,"count":6,"confidence":0.3899950335423152,"empirical_correctness":0.0,"signed_gap":0.3899950335423152,"absolute_gap":0.3899950335423152,"min_confidence":0.3691217005252838,"max_confidence":0.42420271039009094},{"index":7,"count":5,"confidence":0.4349896311759949,"empirical_correctness":0.4,"signed_gap":0.03498963117599485,"absolute_gap":0.03498963117599485,"min_confidence":0.4325534701347351,"max_confidence":0.43790403008461},{"index":8,"count":6,"confidence":0.46679604053497314,"empirical_correctness":1.0,"signed_gap":-0.5332039594650269,"absolute_gap":0.5332039594650269,"min_confidence":0.4383338987827301,"max_confidence":0.5046981573104858},{"index":9,"count":6,"confidence":0.5360247790813446,"empirical_correctness":1.0,"signed_gap":-0.4639752209186554,"absolute_gap":0.4639752209186554,"min_confidence":0.5137881636619568,"max_confidence":0.5661833882331848}]}},"calibration_singleton_labels":{"equal_width":{"n":56,"bin_count":10,"ece":0.28176795318722725,"signed_mean_gap":0.10402677900024823,"absolute_mean_gap":0.10402677900024823,"bins":[{"index":0,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.0,"upper":0.1,"upper_inclusive":false},{"index":1,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.1,"upper":0.2,"upper_inclusive":false},{"index":2,"count":24,"confidence":0.27038734778761864,"empirical_correctness":0.0,"signed_gap":0.27038734778761864,"absolute_gap":0.27038734778761864,"lower":0.2,"upper":0.3,"upper_inclusive":false},{"index":3,"count":13,"confidence":0.33176585802665126,"empirical_correctness":0.0,"signed_gap":0.33176585802665126,"absolute_gap":0.33176585802665126,"lower":0.3,"upper":0.4,"upper_inclusive":false},{"index":4,"count":12,"confidence":0.4418666909138362,"empirical_correctness":0.5833333333333334,"signed_gap":-0.1414666424194972,"absolute_gap":0.1414666424194972,"lower":0.4,"upper":0.5,"upper_inclusive":false},{"index":5,"count":7,"confidence":0.5315495473997933,"empirical_correctness":1.0,"signed_gap":-0.4684504526002067,"absolute_gap":0.4684504526002067,"lower":0.5,"upper":0.6,"upper_inclusive":false},{"index":6,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.6,"upper":0.7,"upper_inclusive":false},{"index":7,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.7,"upper":0.8,"upper_inclusive":false},{"index":8,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.8,"upper":0.9,"upper_inclusive":false},{"index":9,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.9,"upper":1.0,"upper_inclusive":true}]},"equal_mass":{"n":56,"bin_count":10,"ece":0.3177080319396087,"signed_mean_gap":0.10402677900024823,"absolute_mean_gap":0.10402677900024823,"bins":[{"index":0,"count":5,"confidence":0.2503814518451691,"empirical_correctness":0.0,"signed_gap":0.2503814518451691,"absolute_gap":0.2503814518451691,"min_confidence":0.23836177587509155,"max_confidence":0.25748124718666077},{"index":1,"count":6,"confidence":0.26295000314712524,"empirical_correctness":0.0,"signed_gap":0.26295000314712524,"absolute_gap":0.26295000314712524,"min_confidence":0.257655531167984,"max_confidence":0.2676897644996643},{"index":2,"count":5,"confidence":0.2713229119777679,"empirical_correctness":0.0,"signed_gap":0.2713229119777679,"absolute_gap":0.2713229119777679,"min_confidence":0.26845046877861023,"max_confidence":0.27671703696250916},{"index":3,"count":6,"confidence":0.2845223546028137,"empirical_correctness":0.0,"signed_gap":0.2845223546028137,"absolute_gap":0.2845223546028137,"min_confidence":0.281251460313797,"max_confidence":0.2870544493198395},{"index":4,"count":6,"confidence":0.30246814092000324,"empirical_correctness":0.0,"signed_gap":0.30246814092000324,"absolute_gap":0.30246814092000324,"min_confidence":0.2974095940589905,"max_confidence":0.308126837015152},{"index":5,"count":5,"confidence":0.3170983076095581,"empirical_correctness":0.0,"signed_gap":0.3170983076095581,"absolute_gap":0.3170983076095581,"min_confidence":0.31013497710227966,"max_confidence":0.330527126789093},{"index":6,"count":6,"confidence":0.3899950335423152,"empirical_correctness":0.0,"signed_gap":0.3899950335423152,"absolute_gap":0.3899950335423152,"min_confidence":0.3691217005252838,"max_confidence":0.42420271039009094},{"index":7,"count":5,"confidence":0.4349896311759949,"empirical_correctness":0.4,"signed_gap":0.03498963117599485,"absolute_gap":0.03498963117599485,"min_confidence":0.4325534701347351,"max_confidence":0.43790403008461},{"index":8,"count":6,"confidence":0.46679604053497314,"empirical_correctness":1.0,"signed_gap":-0.5332039594650269,"absolute_gap":0.5332039594650269,"min_confidence":0.4383338987827301,"max_confidence":0.5046981573104858},{"index":9,"count":6,"confidence":0.5360247790813446,"empirical_correctness":1.0,"signed_gap":-0.4639752209186554,"absolute_gap":0.4639752209186554,"min_confidence":0.5137881636619568,"max_confidence":0.5661833882331848}]}},"latency_ms":{"n":56,"median":67.36389989964664,"p95":68.72740003746003},"per_action":{"keep":{"n":8,"raw":{"n":8,"acceptable":6,"unsafe":0,"review":0,"automated_count":8,"automated_acceptable":6,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":0.75,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.75,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.40927543031016883,"upper":0.9285207872478909,"confidence_level":0.95}},"effective":{"n":8,"acceptable":1,"unsafe":0,"review":7,"automated_count":1,"automated_acceptable":1,"automated_unsafe":0,"unnecessary_reviews":7,"accuracy":0.125,"unsafe_rate":0.0,"review_rate":0.875,"automated_coverage":0.125,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.875,"accuracy_wilson_95":{"lower":0.02241749145005667,"upper":0.4708881822128535,"confidence_level":0.95}}},"drop":{"n":22,"raw":{"n":22,"acceptable":0,"unsafe":20,"review":0,"automated_count":22,"automated_acceptable":0,"automated_unsafe":20,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":0.9090909090909091,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":0.9090909090909091,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.14865487460862087,"confidence_level":0.95}},"effective":{"n":22,"acceptable":0,"unsafe":0,"review":22,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":22,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.14865487460862087,"confidence_level":0.95}}},"mask":{"n":8,"raw":{"n":8,"acceptable":8,"unsafe":0,"review":0,"automated_count":8,"automated_acceptable":8,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":1.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.6755924351161198,"upper":1.0,"confidence_level":0.95}},"effective":{"n":8,"acceptable":8,"unsafe":0,"review":0,"automated_count":8,"automated_acceptable":8,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":1.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.6755924351161198,"upper":1.0,"confidence_level":0.95}}},"pseudonymize":{"n":8,"raw":{"n":8,"acceptable":0,"unsafe":0,"review":0,"automated_count":8,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.32440756488388023,"confidence_level":0.95}},"effective":{"n":8,"acceptable":0,"unsafe":0,"review":8,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":8,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.32440756488388023,"confidence_level":0.95}}},"generalize":{"n":8,"raw":{"n":8,"acceptable":0,"unsafe":8,"review":0,"automated_count":8,"automated_acceptable":0,"automated_unsafe":8,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":1.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":1.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.32440756488388023,"confidence_level":0.95}},"effective":{"n":8,"acceptable":0,"unsafe":0,"review":8,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":8,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.32440756488388023,"confidence_level":0.95}}},"review":{"n":2,"raw":{"n":2,"acceptable":0,"unsafe":0,"review":0,"automated_count":2,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.6576197724933469,"confidence_level":0.95}},"effective":{"n":2,"acceptable":2,"unsafe":0,"review":2,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":1.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.34238022750665303,"upper":1.0,"confidence_level":0.95}}}}}},"composite":{"n":198,"scope":"descriptive sum across four correlated panels","raw":{"n":198,"acceptable":79,"unsafe":59,"review":0,"automated_count":198,"automated_acceptable":79,"automated_unsafe":59,"unnecessary_reviews":0,"accuracy":0.398989898989899,"unsafe_rate":0.29797979797979796,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.398989898989899,"conditional_unsafe":0.29797979797979796,"unnecessary_review_rate":0.0},"effective":{"n":198,"acceptable":45,"unsafe":0,"review":147,"automated_count":51,"automated_acceptable":37,"automated_unsafe":0,"unnecessary_reviews":139,"accuracy":0.22727272727272727,"unsafe_rate":0.0,"review_rate":0.7424242424242424,"automated_coverage":0.25757575757575757,"conditional_correctness":0.7254901960784313,"conditional_unsafe":0.0,"unnecessary_review_rate":0.702020202020202}}},{"id":"Prasanna85--laya-issue-triage-fp32","platform":"Local","model_id":"Prasanna85/laya-issue-triage","revision":"bbfde5c48f340c2331dc81ae43f9e4554cc2da39","loading":"float32 / none","scoring_kind":"Native decision readout","settings":{"dtype":"float32","quantization":"none","max_tokens":1024,"head_max_tokens":256,"temperature":2.6968,"autocast":false,"truncation":false,"cpu_offload":false,"guard_threshold":0.45,"seed":0,"tf32":false},"scoring_metadata":{"backend":"laya_native_local_builtin","readout":"native_mask_slot_scorer_numpy_float32_softmax","probability_source":"native_choice_unrounded_published_temperature_not_task_calibrated","latency_scope":"synchronized serial complete encoder+head+act forward,input transfer and native readout; tokenization excluded","native_disclosures":["Stock installed torch attention in place of ONNX dynamic equivalent; no custom kernels or repo imports.","FP32/no-autocast explicit precision: convert every saved F16/BF16/F32 tensor toFP32; no quantization.","Unrounded native probability; guard uses winner p, not entropy-concentration confidence.","Act head fully loaded and evaluated; not used to alter six-action answer.","Native choice:6-10 bucket else choice scalar0, T=2.6968. No task fitting, priors correction, prompt change or finetuning here."],"equal_mass_convention":"rank bins with floor boundaries at floor(i*n/k), k=min(10,n//2); minimum two rows"},"load_ms":null,"peak_allocated_bytes":1748760576,"peak_reserved_bytes":1870659584,"panels":{"diagnostic43-no_samples":{"n":43,"raw":{"n":43,"acceptable":19,"unsafe":3,"review":0,"automated_count":43,"automated_acceptable":19,"automated_unsafe":3,"unnecessary_reviews":0,"accuracy":0.4418604651162791,"unsafe_rate":0.06976744186046512,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.4418604651162791,"conditional_unsafe":0.06976744186046512,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.30433309839170514,"upper":0.5889238554717591,"confidence_level":0.95}},"effective":{"n":43,"acceptable":12,"unsafe":0,"review":30,"automated_count":13,"automated_acceptable":9,"automated_unsafe":0,"unnecessary_reviews":27,"accuracy":0.27906976744186046,"unsafe_rate":0.0,"review_rate":0.6976744186046512,"automated_coverage":0.3023255813953488,"conditional_correctness":0.6923076923076923,"conditional_unsafe":0.0,"unnecessary_review_rate":0.627906976744186,"accuracy_wilson_95":{"lower":0.16746613786237488,"upper":0.42691028681878873,"confidence_level":0.95}},"confusion":{"raw":{"keep":{"keep":0,"drop":11,"mask":0,"pseudonymize":1,"generalize":0,"review":0},"drop":{"keep":0,"drop":14,"mask":0,"pseudonymize":2,"generalize":0,"review":0},"mask":{"keep":0,"drop":2,"mask":0,"pseudonymize":0,"generalize":0,"review":0},"pseudonymize":{"keep":0,"drop":1,"mask":0,"pseudonymize":1,"generalize":0,"review":0},"generalize":{"keep":0,"drop":5,"mask":0,"pseudonymize":1,"generalize":0,"review":0},"review":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0}},"effective":{"keep":{"keep":0,"drop":3,"mask":0,"pseudonymize":0,"generalize":0,"review":9},"drop":{"keep":0,"drop":7,"mask":0,"pseudonymize":0,"generalize":0,"review":9},"mask":{"keep":0,"drop":1,"mask":0,"pseudonymize":0,"generalize":0,"review":1},"pseudonymize":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":2},"generalize":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":6},"review":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0}}},"probability_metrics":{"singleton_n":38,"multiacceptable_n":5,"multiacceptable_coverage":0.11627906976744186,"nll":1.5361972692531272,"nll_zero_mass_n":0,"brier":0.7247496220247024,"acceptable_set_n":43,"acceptable_set_nll":1.409713209020598,"acceptable_set_zero_mass_n":0},"calibration_all_labels":{"equal_width":{"n":43,"bin_count":10,"ece":0.08696808510048447,"signed_mean_gap":-0.015497465466344054,"absolute_mean_gap":0.015497465466344054,"bins":[{"index":0,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.0,"upper":0.1,"upper_inclusive":false},{"index":1,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.1,"upper":0.2,"upper_inclusive":false},{"index":2,"count":6,"confidence":0.28444864849249524,"empirical_correctness":0.16666666666666666,"signed_gap":0.11778198182582858,"absolute_gap":0.11778198182582858,"lower":0.2,"upper":0.3,"upper_inclusive":false},{"index":3,"count":21,"confidence":0.34581534351621357,"empirical_correctness":0.3333333333333333,"signed_gap":0.012482010182880254,"absolute_gap":0.012482010182880254,"lower":0.3,"upper":0.4,"upper_inclusive":false},{"index":4,"count":4,"confidence":0.43175744265317917,"empirical_correctness":0.75,"signed_gap":-0.31824255734682083,"absolute_gap":0.31824255734682083,"lower":0.4,"upper":0.5,"upper_inclusive":false},{"index":5,"count":3,"confidence":0.5101839900016785,"empirical_correctness":0.6666666666666666,"signed_gap":-0.15648267666498816,"absolute_gap":0.15648267666498816,"lower":0.5,"upper":0.6,"upper_inclusive":false},{"index":6,"count":7,"confidence":0.6525434596197945,"empirical_correctness":0.5714285714285714,"signed_gap":0.08111488819122314,"absolute_gap":0.08111488819122314,"lower":0.6,"upper":0.7,"upper_inclusive":false},{"index":7,"count":2,"confidence":0.7697044610977173,"empirical_correctness":1.0,"signed_gap":-0.23029553890228271,"absolute_gap":0.23029553890228271,"lower":0.7,"upper":0.8,"upper_inclusive":false},{"index":8,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.8,"upper":0.9,"upper_inclusive":false},{"index":9,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.9,"upper":1.0,"upper_inclusive":true}]},"equal_mass":{"n":43,"bin_count":10,"ece":0.19255202038343563,"signed_mean_gap":-0.015497465466344085,"absolute_mean_gap":0.015497465466344085,"bins":[{"index":0,"count":4,"confidence":0.28017712384462357,"empirical_correctness":0.25,"signed_gap":0.030177123844623566,"absolute_gap":0.030177123844623566,"min_confidence":0.26856085658073425,"max_confidence":0.28524544835090637},{"index":1,"count":4,"confidence":0.29955529421567917,"empirical_correctness":0.0,"signed_gap":0.29955529421567917,"absolute_gap":0.29955529421567917,"min_confidence":0.2900787889957428,"max_confidence":0.3062934875488281},{"index":2,"count":4,"confidence":0.32250603288412094,"empirical_correctness":0.75,"signed_gap":-0.42749396711587906,"absolute_gap":0.42749396711587906,"min_confidence":0.3138526678085327,"max_confidence":0.3281707763671875},{"index":3,"count":5,"confidence":0.33207487463951113,"empirical_correctness":0.6,"signed_gap":-0.26792512536048885,"absolute_gap":0.26792512536048885,"min_confidence":0.32908767461776733,"max_confidence":0.3348101079463959},{"index":4,"count":4,"confidence":0.34896302968263626,"empirical_correctness":0.0,"signed_gap":0.34896302968263626,"absolute_gap":0.34896302968263626,"min_confidence":0.33665207028388977,"max_confidence":0.3616120219230652},{"index":5,"count":4,"confidence":0.37731584906578064,"empirical_correctness":0.25,"signed_gap":0.12731584906578064,"absolute_gap":0.12731584906578064,"min_confidence":0.36568331718444824,"max_confidence":0.3887610137462616},{"index":6,"count":5,"confidence":0.4128679811954498,"empirical_correctness":0.4,"signed_gap":0.012867981195449796,"absolute_gap":0.012867981195449796,"min_confidence":0.396849125623703,"max_confidence":0.43741753697395325},{"index":7,"count":4,"confidence":0.49690306186676025,"empirical_correctness":0.75,"signed_gap":-0.25309693813323975,"absolute_gap":0.25309693813323975,"min_confidence":0.4570602774620056,"max_confidence":0.5166032910346985},{"index":8,"count":4,"confidence":0.6295719593763351,"empirical_correctness":0.5,"signed_gap":0.12957195937633514,"absolute_gap":0.12957195937633514,"min_confidence":0.6054526567459106,"max_confidence":0.652060329914093},{"index":9,"count":5,"confidence":0.7177850604057312,"empirical_correctness":0.8,"signed_gap":-0.08221493959426884,"absolute_gap":0.08221493959426884,"min_confidence":0.6729629635810852,"max_confidence":0.788902759552002}]}},"calibration_singleton_labels":{"equal_width":{"n":38,"bin_count":10,"ece":0.12184295450386248,"signed_mean_gap":0.030602885704291507,"absolute_mean_gap":0.030602885704291507,"bins":[{"index":0,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.0,"upper":0.1,"upper_inclusive":false},{"index":1,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.1,"upper":0.2,"upper_inclusive":false},{"index":2,"count":6,"confidence":0.28444864849249524,"empirical_correctness":0.16666666666666666,"signed_gap":0.11778198182582858,"absolute_gap":0.11778198182582858,"lower":0.2,"upper":0.3,"upper_inclusive":false},{"index":3,"count":18,"confidence":0.3487747195694182,"empirical_correctness":0.2777777777777778,"signed_gap":0.0709969417916404,"absolute_gap":0.0709969417916404,"lower":0.3,"upper":0.4,"upper_inclusive":false},{"index":4,"count":4,"confidence":0.43175744265317917,"empirical_correctness":0.75,"signed_gap":-0.31824255734682083,"absolute_gap":0.31824255734682083,"lower":0.4,"upper":0.5,"upper_inclusive":false},{"index":5,"count":2,"confidence":0.5084964334964752,"empirical_correctness":0.5,"signed_gap":0.00849643349647522,"absolute_gap":0.00849643349647522,"lower":0.5,"upper":0.6,"upper_inclusive":false},{"index":6,"count":6,"confidence":0.6491402089595795,"empirical_correctness":0.5,"signed_gap":0.14914020895957947,"absolute_gap":0.14914020895957947,"lower":0.6,"upper":0.7,"upper_inclusive":false},{"index":7,"count":2,"confidence":0.7697044610977173,"empirical_correctness":1.0,"signed_gap":-0.23029553890228271,"absolute_gap":0.23029553890228271,"lower":0.7,"upper":0.8,"upper_inclusive":false},{"index":8,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.8,"upper":0.9,"upper_inclusive":false},{"index":9,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.9,"upper":1.0,"upper_inclusive":true}]},"equal_mass":{"n":38,"bin_count":10,"ece":0.16538230290538386,"signed_mean_gap":0.030602885704291493,"absolute_mean_gap":0.030602885704291493,"bins":[{"index":0,"count":3,"confidence":0.2784876823425293,"empirical_correctness":0.0,"signed_gap":0.2784876823425293,"absolute_gap":0.2784876823425293,"min_confidence":0.26856085658073425,"max_confidence":0.2839091420173645},{"index":1,"count":4,"confidence":0.29429328441619873,"empirical_correctness":0.25,"signed_gap":0.04429328441619873,"absolute_gap":0.04429328441619873,"min_confidence":0.28524544835090637,"max_confidence":0.3059442937374115},{"index":2,"count":4,"confidence":0.3206162378191948,"empirical_correctness":0.5,"signed_gap":-0.1793837621808052,"absolute_gap":0.1793837621808052,"min_confidence":0.3062934875488281,"max_confidence":0.3281707763671875},{"index":3,"count":4,"confidence":0.33232327550649643,"empirical_correctness":0.5,"signed_gap":-0.16767672449350357,"absolute_gap":0.16767672449350357,"min_confidence":0.32908767461776733,"max_confidence":0.3348101079463959},{"index":4,"count":4,"confidence":0.35557302832603455,"empirical_correctness":0.0,"signed_gap":0.35557302832603455,"absolute_gap":0.35557302832603455,"min_confidence":0.33665207028388977,"max_confidence":0.36568331718444824},{"index":5,"count":3,"confidence":0.3811933596928914,"empirical_correctness":0.3333333333333333,"signed_gap":0.047860026359558105,"absolute_gap":0.047860026359558105,"min_confidence":0.37085631489753723,"max_confidence":0.3887610137462616},{"index":6,"count":4,"confidence":0.406730592250824,"empirical_correctness":0.25,"signed_gap":0.15673059225082397,"absolute_gap":0.15673059225082397,"min_confidence":0.396849125623703,"max_confidence":0.4231530427932739},{"index":7,"count":4,"confidence":0.4778676703572273,"empirical_correctness":0.75,"signed_gap":-0.2721323296427727,"absolute_gap":0.2721323296427727,"min_confidence":0.43741753697395325,"max_confidence":0.5166032910346985},{"index":8,"count":4,"confidence":0.6295719593763351,"empirical_correctness":0.5,"signed_gap":0.12957195937633514,"absolute_gap":0.12957195937633514,"min_confidence":0.6054526567459106,"max_confidence":0.652060329914093},{"index":9,"count":4,"confidence":0.7289905846118927,"empirical_correctness":0.75,"signed_gap":-0.0210094153881073,"absolute_gap":0.0210094153881073,"min_confidence":0.6876195669174194,"max_confidence":0.788902759552002}]}},"latency_ms":{"n":43,"median":18.13997799763456,"p95":18.872665974777192},"per_action":{"keep":{"n":12,"raw":{"n":12,"acceptable":0,"unsafe":0,"review":0,"automated_count":12,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":2.7755575615628914e-17,"upper":0.24249400665524085,"confidence_level":0.95}},"effective":{"n":12,"acceptable":0,"unsafe":0,"review":9,"automated_count":3,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":9,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":0.75,"automated_coverage":0.25,"conditional_correctness":0.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.75,"accuracy_wilson_95":{"lower":2.7755575615628914e-17,"upper":0.24249400665524085,"confidence_level":0.95}}},"drop":{"n":16,"raw":{"n":16,"acceptable":14,"unsafe":2,"review":0,"automated_count":16,"automated_acceptable":14,"automated_unsafe":2,"unnecessary_reviews":0,"accuracy":0.875,"unsafe_rate":0.125,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.875,"conditional_unsafe":0.125,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.639771727342413,"upper":0.9650225122567595,"confidence_level":0.95}},"effective":{"n":16,"acceptable":7,"unsafe":0,"review":9,"automated_count":7,"automated_acceptable":7,"automated_unsafe":0,"unnecessary_reviews":9,"accuracy":0.4375,"unsafe_rate":0.0,"review_rate":0.5625,"automated_coverage":0.4375,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.5625,"accuracy_wilson_95":{"lower":0.23098652405492354,"upper":0.6682144360118811,"confidence_level":0.95}}},"mask":{"n":2,"raw":{"n":2,"acceptable":0,"unsafe":0,"review":0,"automated_count":2,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.6576197724933469,"confidence_level":0.95}},"effective":{"n":2,"acceptable":0,"unsafe":0,"review":1,"automated_count":1,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":1,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":0.5,"automated_coverage":0.5,"conditional_correctness":0.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.5,"accuracy_wilson_95":{"lower":0.0,"upper":0.6576197724933469,"confidence_level":0.95}}},"pseudonymize":{"n":2,"raw":{"n":2,"acceptable":1,"unsafe":0,"review":0,"automated_count":2,"automated_acceptable":1,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":0.5,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.5,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.09453120573423074,"upper":0.9054687942657693,"confidence_level":0.95}},"effective":{"n":2,"acceptable":0,"unsafe":0,"review":2,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":2,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.6576197724933469,"confidence_level":0.95}}},"generalize":{"n":6,"raw":{"n":6,"acceptable":0,"unsafe":0,"review":0,"automated_count":6,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":2.7755575615628914e-17,"upper":0.3903342879021653,"confidence_level":0.95}},"effective":{"n":6,"acceptable":0,"unsafe":0,"review":6,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":6,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":2.7755575615628914e-17,"upper":0.3903342879021653,"confidence_level":0.95}}},"review":{"n":0,"raw":{"n":0,"acceptable":0,"unsafe":0,"review":0,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":null,"unsafe_rate":null,"review_rate":null,"automated_coverage":null,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":null},"effective":{"n":0,"acceptable":0,"unsafe":0,"review":0,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":null,"unsafe_rate":null,"review_rate":null,"automated_coverage":null,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":null}}}},"diagnostic43-compact_no_samples":{"n":43,"raw":{"n":43,"acceptable":21,"unsafe":0,"review":0,"automated_count":43,"automated_acceptable":21,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":0.4883720930232558,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.4883720930232558,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.3461744754671859,"upper":0.632476915305507,"confidence_level":0.95}},"effective":{"n":43,"acceptable":9,"unsafe":0,"review":34,"automated_count":9,"automated_acceptable":5,"automated_unsafe":0,"unnecessary_reviews":30,"accuracy":0.20930232558139536,"unsafe_rate":0.0,"review_rate":0.7906976744186046,"automated_coverage":0.20930232558139536,"conditional_correctness":0.5555555555555556,"conditional_unsafe":0.0,"unnecessary_review_rate":0.6976744186046512,"accuracy_wilson_95":{"lower":0.11422834220995602,"upper":0.3520564271073644,"confidence_level":0.95}},"confusion":{"raw":{"keep":{"keep":0,"drop":12,"mask":0,"pseudonymize":0,"generalize":0,"review":0},"drop":{"keep":0,"drop":16,"mask":0,"pseudonymize":0,"generalize":0,"review":0},"mask":{"keep":0,"drop":2,"mask":0,"pseudonymize":0,"generalize":0,"review":0},"pseudonymize":{"keep":0,"drop":2,"mask":0,"pseudonymize":0,"generalize":0,"review":0},"generalize":{"keep":0,"drop":6,"mask":0,"pseudonymize":0,"generalize":0,"review":0},"review":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0}},"effective":{"keep":{"keep":0,"drop":3,"mask":0,"pseudonymize":0,"generalize":0,"review":9},"drop":{"keep":0,"drop":4,"mask":0,"pseudonymize":0,"generalize":0,"review":12},"mask":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":2},"pseudonymize":{"keep":0,"drop":1,"mask":0,"pseudonymize":0,"generalize":0,"review":1},"generalize":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":6},"review":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0}}},"probability_metrics":{"singleton_n":38,"multiacceptable_n":5,"multiacceptable_coverage":0.11627906976744186,"nll":1.4535275734199111,"nll_zero_mass_n":0,"brier":0.7028697517087847,"acceptable_set_n":43,"acceptable_set_nll":1.3350213699959492,"acceptable_set_zero_mass_n":0},"calibration_all_labels":{"equal_width":{"n":43,"bin_count":10,"ece":0.1350502170795618,"signed_mean_gap":-0.11633017104725506,"absolute_mean_gap":0.11633017104725506,"bins":[{"index":0,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.0,"upper":0.1,"upper_inclusive":false},{"index":1,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.1,"upper":0.2,"upper_inclusive":false},{"index":2,"count":14,"confidence":0.27603686281612944,"empirical_correctness":0.2857142857142857,"signed_gap":-0.009677422898156263,"absolute_gap":0.009677422898156263,"lower":0.2,"upper":0.3,"upper_inclusive":false},{"index":3,"count":16,"confidence":0.34293441846966743,"empirical_correctness":0.625,"signed_gap":-0.28206558153033257,"absolute_gap":0.28206558153033257,"lower":0.3,"upper":0.4,"upper_inclusive":false},{"index":4,"count":5,"confidence":0.4433732688426971,"empirical_correctness":0.4,"signed_gap":0.0433732688426971,"absolute_gap":0.0433732688426971,"lower":0.4,"upper":0.5,"upper_inclusive":false},{"index":5,"count":6,"confidence":0.5309357742468516,"empirical_correctness":0.5,"signed_gap":0.030935774246851566,"absolute_gap":0.030935774246851566,"lower":0.5,"upper":0.6,"upper_inclusive":false},{"index":6,"count":2,"confidence":0.6219274401664734,"empirical_correctness":1.0,"signed_gap":-0.3780725598335266,"absolute_gap":0.3780725598335266,"lower":0.6,"upper":0.7,"upper_inclusive":false},{"index":7,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.7,"upper":0.8,"upper_inclusive":false},{"index":8,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.8,"upper":0.9,"upper_inclusive":false},{"index":9,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.9,"upper":1.0,"upper_inclusive":true}]},"equal_mass":{"n":43,"bin_count":10,"ece":0.16954242559366448,"signed_mean_gap":-0.11633017104725506,"absolute_mean_gap":0.11633017104725506,"bins":[{"index":0,"count":4,"confidence":0.25404709577560425,"empirical_correctness":0.0,"signed_gap":0.25404709577560425,"absolute_gap":0.25404709577560425,"min_confidence":0.24670693278312683,"max_confidence":0.2595698833465576},{"index":1,"count":4,"confidence":0.2743627578020096,"empirical_correctness":0.25,"signed_gap":0.024362757802009583,"absolute_gap":0.024362757802009583,"min_confidence":0.2647787928581238,"max_confidence":0.2821277379989624},{"index":2,"count":4,"confidence":0.2893828749656677,"empirical_correctness":0.5,"signed_gap":-0.21061712503433228,"absolute_gap":0.21061712503433228,"min_confidence":0.28266727924346924,"max_confidence":0.2957615852355957},{"index":3,"count":5,"confidence":0.30472440719604493,"empirical_correctness":0.6,"signed_gap":-0.29527559280395504,"absolute_gap":0.29527559280395504,"min_confidence":0.2958363890647888,"max_confidence":0.3195258677005768},{"index":4,"count":4,"confidence":0.3287818804383278,"empirical_correctness":0.75,"signed_gap":-0.4212181195616722,"absolute_gap":0.4212181195616722,"min_confidence":0.32522058486938477,"max_confidence":0.331510066986084},{"index":5,"count":4,"confidence":0.3438778966665268,"empirical_correctness":0.75,"signed_gap":-0.4061221033334732,"absolute_gap":0.4061221033334732,"min_confidence":0.3385568857192993,"max_confidence":0.3479880094528198},{"index":6,"count":5,"confidence":0.3732069432735443,"empirical_correctness":0.4,"signed_gap":-0.02679305672645571,"absolute_gap":0.02679305672645571,"min_confidence":0.3483208417892456,"max_confidence":0.3959559202194214},{"index":7,"count":4,"confidence":0.43273352831602097,"empirical_correctness":0.5,"signed_gap":-0.06726647168397903,"absolute_gap":0.06726647168397903,"min_confidence":0.41961759328842163,"max_confidence":0.44294440746307373},{"index":8,"count":4,"confidence":0.5076060146093369,"empirical_correctness":0.5,"signed_gap":0.007606014609336853,"absolute_gap":0.007606014609336853,"min_confidence":0.48593223094940186,"max_confidence":0.5221551060676575},{"index":9,"count":5,"confidence":0.5769955396652222,"empirical_correctness":0.6,"signed_gap":-0.023004460334777765,"absolute_gap":0.023004460334777765,"min_confidence":0.5297169089317322,"max_confidence":0.6347522139549255}]}},"calibration_singleton_labels":{"equal_width":{"n":38,"bin_count":10,"ece":0.16081346257736806,"signed_mean_gap":-0.04688367796571629,"absolute_mean_gap":0.04688367796571629,"bins":[{"index":0,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.0,"upper":0.1,"upper_inclusive":false},{"index":1,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.1,"upper":0.2,"upper_inclusive":false},{"index":2,"count":12,"confidence":0.27369500199953717,"empirical_correctness":0.16666666666666666,"signed_gap":0.10702833533287051,"absolute_gap":0.10702833533287051,"lower":0.2,"upper":0.3,"upper_inclusive":false},{"index":3,"count":14,"confidence":0.34356424638203215,"empirical_correctness":0.5714285714285714,"signed_gap":-0.22786432504653925,"absolute_gap":0.22786432504653925,"lower":0.3,"upper":0.4,"upper_inclusive":false},{"index":4,"count":5,"confidence":0.4433732688426971,"empirical_correctness":0.4,"signed_gap":0.0433732688426971,"absolute_gap":0.0433732688426971,"lower":0.4,"upper":0.5,"upper_inclusive":false},{"index":5,"count":5,"confidence":0.5326919078826904,"empirical_correctness":0.4,"signed_gap":0.13269190788269036,"absolute_gap":0.13269190788269036,"lower":0.5,"upper":0.6,"upper_inclusive":false},{"index":6,"count":2,"confidence":0.6219274401664734,"empirical_correctness":1.0,"signed_gap":-0.3780725598335266,"absolute_gap":0.3780725598335266,"lower":0.6,"upper":0.7,"upper_inclusive":false},{"index":7,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.7,"upper":0.8,"upper_inclusive":false},{"index":8,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.8,"upper":0.9,"upper_inclusive":false},{"index":9,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.9,"upper":1.0,"upper_inclusive":true}]},"equal_mass":{"n":38,"bin_count":10,"ece":0.1535410198726152,"signed_mean_gap":-0.04688367796571631,"absolute_mean_gap":0.04688367796571631,"bins":[{"index":0,"count":3,"confidence":0.25220616658528644,"empirical_correctness":0.0,"signed_gap":0.25220616658528644,"absolute_gap":0.25220616658528644,"min_confidence":0.24670693278312683,"max_confidence":0.2549712657928467},{"index":1,"count":4,"confidence":0.2687232941389084,"empirical_correctness":0.25,"signed_gap":0.018723294138908386,"absolute_gap":0.018723294138908386,"min_confidence":0.2595698833465576,"max_confidence":0.2753210961818695},{"index":2,"count":4,"confidence":0.289247989654541,"empirical_correctness":0.25,"signed_gap":0.039247989654541016,"absolute_gap":0.039247989654541016,"min_confidence":0.2821277379989624,"max_confidence":0.2957615852355957},{"index":3,"count":4,"confidence":0.30652831494808197,"empirical_correctness":0.5,"signed_gap":-0.19347168505191803,"absolute_gap":0.19347168505191803,"min_confidence":0.2958363890647888,"max_confidence":0.3195258677005768},{"index":4,"count":4,"confidence":0.3311084806919098,"empirical_correctness":0.5,"signed_gap":-0.1688915193080902,"absolute_gap":0.1688915193080902,"min_confidence":0.32522058486938477,"max_confidence":0.3385568857192993},{"index":5,"count":3,"confidence":0.3458249270915985,"empirical_correctness":0.6666666666666666,"signed_gap":-0.3208417395750681,"absolute_gap":0.3208417395750681,"min_confidence":0.3411659300327301,"max_confidence":0.3483208417892456},{"index":6,"count":4,"confidence":0.379428468644619,"empirical_correctness":0.5,"signed_gap":-0.12057153135538101,"absolute_gap":0.12057153135538101,"min_confidence":0.353705495595932,"max_confidence":0.3959559202194214},{"index":7,"count":4,"confidence":0.43273352831602097,"empirical_correctness":0.5,"signed_gap":-0.06726647168397903,"absolute_gap":0.06726647168397903,"min_confidence":0.41961759328842163,"max_confidence":0.44294440746307373},{"index":8,"count":4,"confidence":0.5094964653253555,"empirical_correctness":0.25,"signed_gap":0.25949646532535553,"absolute_gap":0.25949646532535553,"min_confidence":0.48593223094940186,"max_confidence":0.5297169089317322},{"index":9,"count":4,"confidence":0.5888151973485947,"empirical_correctness":0.75,"signed_gap":-0.16118480265140533,"absolute_gap":0.16118480265140533,"min_confidence":0.5349684953689575,"max_confidence":0.6347522139549255}]}},"latency_ms":{"n":43,"median":20.773522031959146,"p95":21.149595984024927},"per_action":{"keep":{"n":12,"raw":{"n":12,"acceptable":0,"unsafe":0,"review":0,"automated_count":12,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":2.7755575615628914e-17,"upper":0.24249400665524085,"confidence_level":0.95}},"effective":{"n":12,"acceptable":0,"unsafe":0,"review":9,"automated_count":3,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":9,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":0.75,"automated_coverage":0.25,"conditional_correctness":0.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.75,"accuracy_wilson_95":{"lower":2.7755575615628914e-17,"upper":0.24249400665524085,"confidence_level":0.95}}},"drop":{"n":16,"raw":{"n":16,"acceptable":16,"unsafe":0,"review":0,"automated_count":16,"automated_acceptable":16,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":1.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.8063923194655637,"upper":1.0,"confidence_level":0.95}},"effective":{"n":16,"acceptable":4,"unsafe":0,"review":12,"automated_count":4,"automated_acceptable":4,"automated_unsafe":0,"unnecessary_reviews":12,"accuracy":0.25,"unsafe_rate":0.0,"review_rate":0.75,"automated_coverage":0.25,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.75,"accuracy_wilson_95":{"lower":0.10182067491213048,"upper":0.49498316535508774,"confidence_level":0.95}}},"mask":{"n":2,"raw":{"n":2,"acceptable":0,"unsafe":0,"review":0,"automated_count":2,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.6576197724933469,"confidence_level":0.95}},"effective":{"n":2,"acceptable":0,"unsafe":0,"review":2,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":2,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.6576197724933469,"confidence_level":0.95}}},"pseudonymize":{"n":2,"raw":{"n":2,"acceptable":0,"unsafe":0,"review":0,"automated_count":2,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.6576197724933469,"confidence_level":0.95}},"effective":{"n":2,"acceptable":0,"unsafe":0,"review":1,"automated_count":1,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":1,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":0.5,"automated_coverage":0.5,"conditional_correctness":0.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.5,"accuracy_wilson_95":{"lower":0.0,"upper":0.6576197724933469,"confidence_level":0.95}}},"generalize":{"n":6,"raw":{"n":6,"acceptable":0,"unsafe":0,"review":0,"automated_count":6,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":2.7755575615628914e-17,"upper":0.3903342879021653,"confidence_level":0.95}},"effective":{"n":6,"acceptable":0,"unsafe":0,"review":6,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":6,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":2.7755575615628914e-17,"upper":0.3903342879021653,"confidence_level":0.95}}},"review":{"n":0,"raw":{"n":0,"acceptable":0,"unsafe":0,"review":0,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":null,"unsafe_rate":null,"review_rate":null,"automated_coverage":null,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":null},"effective":{"n":0,"acceptable":0,"unsafe":0,"review":0,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":null,"unsafe_rate":null,"review_rate":null,"automated_coverage":null,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":null}}}},"challenge56-no_samples":{"n":56,"raw":{"n":56,"acceptable":33,"unsafe":23,"review":0,"automated_count":56,"automated_acceptable":33,"automated_unsafe":23,"unnecessary_reviews":0,"accuracy":0.5892857142857143,"unsafe_rate":0.4107142857142857,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.5892857142857143,"conditional_unsafe":0.4107142857142857,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.4587759684643971,"upper":0.708332257121538,"confidence_level":0.95}},"effective":{"n":56,"acceptable":13,"unsafe":0,"review":45,"automated_count":11,"automated_acceptable":11,"automated_unsafe":0,"unnecessary_reviews":43,"accuracy":0.23214285714285715,"unsafe_rate":0.0,"review_rate":0.8035714285714286,"automated_coverage":0.19642857142857142,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.7678571428571429,"accuracy_wilson_95":{"lower":0.1409937910920862,"upper":0.35768153215010856,"confidence_level":0.95}},"confusion":{"raw":{"keep":{"keep":8,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0},"drop":{"keep":6,"drop":16,"mask":0,"pseudonymize":0,"generalize":0,"review":0},"mask":{"keep":0,"drop":0,"mask":8,"pseudonymize":0,"generalize":0,"review":0},"pseudonymize":{"keep":7,"drop":0,"mask":0,"pseudonymize":1,"generalize":0,"review":0},"generalize":{"keep":8,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0},"review":{"keep":2,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0}},"effective":{"keep":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":8},"drop":{"keep":0,"drop":3,"mask":0,"pseudonymize":0,"generalize":0,"review":19},"mask":{"keep":0,"drop":0,"mask":8,"pseudonymize":0,"generalize":0,"review":0},"pseudonymize":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":8},"generalize":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":8},"review":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":2}}},"probability_metrics":{"singleton_n":56,"multiacceptable_n":0,"multiacceptable_coverage":0.0,"nll":1.3149463549939036,"nll_zero_mass_n":0,"brier":0.6263787537886337,"acceptable_set_n":56,"acceptable_set_nll":1.3149463549939036,"acceptable_set_zero_mass_n":0},"calibration_all_labels":{"equal_width":{"n":56,"bin_count":10,"ece":0.2665241238261972,"signed_mean_gap":-0.2665241238261972,"absolute_mean_gap":0.2665241238261972,"bins":[{"index":0,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.0,"upper":0.1,"upper_inclusive":false},{"index":1,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.1,"upper":0.2,"upper_inclusive":false},{"index":2,"count":35,"confidence":0.2462016352585384,"empirical_correctness":0.45714285714285713,"signed_gap":-0.21094122188431874,"absolute_gap":0.21094122188431874,"lower":0.2,"upper":0.3,"upper_inclusive":false},{"index":3,"count":9,"confidence":0.3244483040438758,"empirical_correctness":0.5555555555555556,"signed_gap":-0.23110725151167977,"absolute_gap":0.23110725151167977,"lower":0.3,"upper":0.4,"upper_inclusive":false},{"index":4,"count":3,"confidence":0.45339547594388324,"empirical_correctness":1.0,"signed_gap":-0.5466045240561168,"absolute_gap":0.5466045240561168,"lower":0.4,"upper":0.5,"upper_inclusive":false},{"index":5,"count":6,"confidence":0.5552060405413309,"empirical_correctness":1.0,"signed_gap":-0.44479395945866906,"absolute_gap":0.44479395945866906,"lower":0.5,"upper":0.6,"upper_inclusive":false},{"index":6,"count":3,"confidence":0.6153781414031982,"empirical_correctness":1.0,"signed_gap":-0.38462185859680176,"absolute_gap":0.38462185859680176,"lower":0.6,"upper":0.7,"upper_inclusive":false},{"index":7,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.7,"upper":0.8,"upper_inclusive":false},{"index":8,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.8,"upper":0.9,"upper_inclusive":false},{"index":9,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.9,"upper":1.0,"upper_inclusive":true}]},"equal_mass":{"n":56,"bin_count":10,"ece":0.3093157240322658,"signed_mean_gap":-0.2665241238261972,"absolute_mean_gap":0.2665241238261972,"bins":[{"index":0,"count":5,"confidence":0.20943330526351928,"empirical_correctness":0.2,"signed_gap":0.009433305263519265,"absolute_gap":0.009433305263519265,"min_confidence":0.20072592794895172,"max_confidence":0.2139780968427658},{"index":1,"count":6,"confidence":0.22272058576345444,"empirical_correctness":0.3333333333333333,"signed_gap":-0.11061274756987888,"absolute_gap":0.11061274756987888,"min_confidence":0.2168116569519043,"max_confidence":0.2270880788564682},{"index":2,"count":5,"confidence":0.2301996558904648,"empirical_correctness":0.0,"signed_gap":0.2301996558904648,"absolute_gap":0.2301996558904648,"min_confidence":0.22778207063674927,"max_confidence":0.23390358686447144},{"index":3,"count":6,"confidence":0.24048703412214914,"empirical_correctness":0.6666666666666666,"signed_gap":-0.4261796325445175,"absolute_gap":0.4261796325445175,"min_confidence":0.23799119889736176,"max_confidence":0.2439853847026825},{"index":4,"count":6,"confidence":0.2707899312178294,"empirical_correctness":0.8333333333333334,"signed_gap":-0.562543402115504,"absolute_gap":0.562543402115504,"min_confidence":0.26206231117248535,"max_confidence":0.28219032287597656},{"index":5,"count":5,"confidence":0.2853514075279236,"empirical_correctness":0.6,"signed_gap":-0.3146485924720764,"absolute_gap":0.3146485924720764,"min_confidence":0.2832484841346741,"max_confidence":0.29077619314193726},{"index":6,"count":6,"confidence":0.3006422470013301,"empirical_correctness":0.3333333333333333,"signed_gap":-0.03269108633200324,"absolute_gap":0.03269108633200324,"min_confidence":0.2908468246459961,"max_confidence":0.3082556128501892},{"index":7,"count":5,"confidence":0.3408662676811218,"empirical_correctness":0.8,"signed_gap":-0.4591337323188782,"absolute_gap":0.4591337323188782,"min_confidence":0.30985167622566223,"max_confidence":0.3831823766231537},{"index":8,"count":6,"confidence":0.4983292867739995,"empirical_correctness":1.0,"signed_gap":-0.5016707132260005,"absolute_gap":0.5016707132260005,"min_confidence":0.42059326171875,"max_confidence":0.5505344271659851},{"index":9,"count":6,"confidence":0.5912635624408722,"empirical_correctness":1.0,"signed_gap":-0.4087364375591278,"absolute_gap":0.4087364375591278,"min_confidence":0.5520449876785278,"max_confidence":0.6195487976074219}]}},"calibration_singleton_labels":{"equal_width":{"n":56,"bin_count":10,"ece":0.2665241238261972,"signed_mean_gap":-0.2665241238261972,"absolute_mean_gap":0.2665241238261972,"bins":[{"index":0,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.0,"upper":0.1,"upper_inclusive":false},{"index":1,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.1,"upper":0.2,"upper_inclusive":false},{"index":2,"count":35,"confidence":0.2462016352585384,"empirical_correctness":0.45714285714285713,"signed_gap":-0.21094122188431874,"absolute_gap":0.21094122188431874,"lower":0.2,"upper":0.3,"upper_inclusive":false},{"index":3,"count":9,"confidence":0.3244483040438758,"empirical_correctness":0.5555555555555556,"signed_gap":-0.23110725151167977,"absolute_gap":0.23110725151167977,"lower":0.3,"upper":0.4,"upper_inclusive":false},{"index":4,"count":3,"confidence":0.45339547594388324,"empirical_correctness":1.0,"signed_gap":-0.5466045240561168,"absolute_gap":0.5466045240561168,"lower":0.4,"upper":0.5,"upper_inclusive":false},{"index":5,"count":6,"confidence":0.5552060405413309,"empirical_correctness":1.0,"signed_gap":-0.44479395945866906,"absolute_gap":0.44479395945866906,"lower":0.5,"upper":0.6,"upper_inclusive":false},{"index":6,"count":3,"confidence":0.6153781414031982,"empirical_correctness":1.0,"signed_gap":-0.38462185859680176,"absolute_gap":0.38462185859680176,"lower":0.6,"upper":0.7,"upper_inclusive":false},{"index":7,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.7,"upper":0.8,"upper_inclusive":false},{"index":8,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.8,"upper":0.9,"upper_inclusive":false},{"index":9,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.9,"upper":1.0,"upper_inclusive":true}]},"equal_mass":{"n":56,"bin_count":10,"ece":0.3093157240322658,"signed_mean_gap":-0.2665241238261972,"absolute_mean_gap":0.2665241238261972,"bins":[{"index":0,"count":5,"confidence":0.20943330526351928,"empirical_correctness":0.2,"signed_gap":0.009433305263519265,"absolute_gap":0.009433305263519265,"min_confidence":0.20072592794895172,"max_confidence":0.2139780968427658},{"index":1,"count":6,"confidence":0.22272058576345444,"empirical_correctness":0.3333333333333333,"signed_gap":-0.11061274756987888,"absolute_gap":0.11061274756987888,"min_confidence":0.2168116569519043,"max_confidence":0.2270880788564682},{"index":2,"count":5,"confidence":0.2301996558904648,"empirical_correctness":0.0,"signed_gap":0.2301996558904648,"absolute_gap":0.2301996558904648,"min_confidence":0.22778207063674927,"max_confidence":0.23390358686447144},{"index":3,"count":6,"confidence":0.24048703412214914,"empirical_correctness":0.6666666666666666,"signed_gap":-0.4261796325445175,"absolute_gap":0.4261796325445175,"min_confidence":0.23799119889736176,"max_confidence":0.2439853847026825},{"index":4,"count":6,"confidence":0.2707899312178294,"empirical_correctness":0.8333333333333334,"signed_gap":-0.562543402115504,"absolute_gap":0.562543402115504,"min_confidence":0.26206231117248535,"max_confidence":0.28219032287597656},{"index":5,"count":5,"confidence":0.2853514075279236,"empirical_correctness":0.6,"signed_gap":-0.3146485924720764,"absolute_gap":0.3146485924720764,"min_confidence":0.2832484841346741,"max_confidence":0.29077619314193726},{"index":6,"count":6,"confidence":0.3006422470013301,"empirical_correctness":0.3333333333333333,"signed_gap":-0.03269108633200324,"absolute_gap":0.03269108633200324,"min_confidence":0.2908468246459961,"max_confidence":0.3082556128501892},{"index":7,"count":5,"confidence":0.3408662676811218,"empirical_correctness":0.8,"signed_gap":-0.4591337323188782,"absolute_gap":0.4591337323188782,"min_confidence":0.30985167622566223,"max_confidence":0.3831823766231537},{"index":8,"count":6,"confidence":0.4983292867739995,"empirical_correctness":1.0,"signed_gap":-0.5016707132260005,"absolute_gap":0.5016707132260005,"min_confidence":0.42059326171875,"max_confidence":0.5505344271659851},{"index":9,"count":6,"confidence":0.5912635624408722,"empirical_correctness":1.0,"signed_gap":-0.4087364375591278,"absolute_gap":0.4087364375591278,"min_confidence":0.5520449876785278,"max_confidence":0.6195487976074219}]}},"latency_ms":{"n":56,"median":15.495856991037726,"p95":15.813289486686699},"per_action":{"keep":{"n":8,"raw":{"n":8,"acceptable":8,"unsafe":0,"review":0,"automated_count":8,"automated_acceptable":8,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":1.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.6755924351161198,"upper":1.0,"confidence_level":0.95}},"effective":{"n":8,"acceptable":0,"unsafe":0,"review":8,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":8,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.32440756488388023,"confidence_level":0.95}}},"drop":{"n":22,"raw":{"n":22,"acceptable":16,"unsafe":6,"review":0,"automated_count":22,"automated_acceptable":16,"automated_unsafe":6,"unnecessary_reviews":0,"accuracy":0.7272727272727273,"unsafe_rate":0.2727272727272727,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.7272727272727273,"conditional_unsafe":0.2727272727272727,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.5184826598759762,"upper":0.8684923971201053,"confidence_level":0.95}},"effective":{"n":22,"acceptable":3,"unsafe":0,"review":19,"automated_count":3,"automated_acceptable":3,"automated_unsafe":0,"unnecessary_reviews":19,"accuracy":0.13636363636363635,"unsafe_rate":0.0,"review_rate":0.8636363636363636,"automated_coverage":0.13636363636363635,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.8636363636363636,"accuracy_wilson_95":{"lower":0.04749003714055253,"upper":0.33334987166571717,"confidence_level":0.95}}},"mask":{"n":8,"raw":{"n":8,"acceptable":8,"unsafe":0,"review":0,"automated_count":8,"automated_acceptable":8,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":1.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.6755924351161198,"upper":1.0,"confidence_level":0.95}},"effective":{"n":8,"acceptable":8,"unsafe":0,"review":0,"automated_count":8,"automated_acceptable":8,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":1.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.6755924351161198,"upper":1.0,"confidence_level":0.95}}},"pseudonymize":{"n":8,"raw":{"n":8,"acceptable":1,"unsafe":7,"review":0,"automated_count":8,"automated_acceptable":1,"automated_unsafe":7,"unnecessary_reviews":0,"accuracy":0.125,"unsafe_rate":0.875,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.125,"conditional_unsafe":0.875,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.02241749145005667,"upper":0.4708881822128535,"confidence_level":0.95}},"effective":{"n":8,"acceptable":0,"unsafe":0,"review":8,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":8,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.32440756488388023,"confidence_level":0.95}}},"generalize":{"n":8,"raw":{"n":8,"acceptable":0,"unsafe":8,"review":0,"automated_count":8,"automated_acceptable":0,"automated_unsafe":8,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":1.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":1.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.32440756488388023,"confidence_level":0.95}},"effective":{"n":8,"acceptable":0,"unsafe":0,"review":8,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":8,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.32440756488388023,"confidence_level":0.95}}},"review":{"n":2,"raw":{"n":2,"acceptable":0,"unsafe":2,"review":0,"automated_count":2,"automated_acceptable":0,"automated_unsafe":2,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":1.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":1.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.6576197724933469,"confidence_level":0.95}},"effective":{"n":2,"acceptable":2,"unsafe":0,"review":2,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":1.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.34238022750665303,"upper":1.0,"confidence_level":0.95}}}}},"challenge56-compact_no_samples":{"n":56,"raw":{"n":56,"acceptable":30,"unsafe":22,"review":0,"automated_count":56,"automated_acceptable":30,"automated_unsafe":22,"unnecessary_reviews":0,"accuracy":0.5357142857142857,"unsafe_rate":0.39285714285714285,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.5357142857142857,"conditional_unsafe":0.39285714285714285,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.40704181935661177,"upper":0.6598014708777623,"confidence_level":0.95}},"effective":{"n":56,"acceptable":6,"unsafe":0,"review":52,"automated_count":4,"automated_acceptable":4,"automated_unsafe":0,"unnecessary_reviews":50,"accuracy":0.10714285714285714,"unsafe_rate":0.0,"review_rate":0.9285714285714286,"automated_coverage":0.07142857142857142,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.8928571428571429,"accuracy_wilson_95":{"lower":0.05003935571117074,"upper":0.21468445171071482,"confidence_level":0.95}},"confusion":{"raw":{"keep":{"keep":8,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0},"drop":{"keep":5,"drop":17,"mask":0,"pseudonymize":0,"generalize":0,"review":0},"mask":{"keep":0,"drop":3,"mask":5,"pseudonymize":0,"generalize":0,"review":0},"pseudonymize":{"keep":7,"drop":1,"mask":0,"pseudonymize":0,"generalize":0,"review":0},"generalize":{"keep":8,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0},"review":{"keep":2,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0}},"effective":{"keep":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":8},"drop":{"keep":0,"drop":3,"mask":0,"pseudonymize":0,"generalize":0,"review":19},"mask":{"keep":0,"drop":0,"mask":1,"pseudonymize":0,"generalize":0,"review":7},"pseudonymize":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":8},"generalize":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":8},"review":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":2}}},"probability_metrics":{"singleton_n":56,"multiacceptable_n":0,"multiacceptable_coverage":0.0,"nll":1.4411908564965834,"nll_zero_mass_n":0,"brier":0.6919913657452543,"acceptable_set_n":56,"acceptable_set_nll":1.4411908564965834,"acceptable_set_zero_mass_n":0},"calibration_all_labels":{"equal_width":{"n":56,"bin_count":10,"ece":0.22471037826367787,"signed_mean_gap":-0.22471037826367787,"absolute_mean_gap":0.22471037826367787,"bins":[{"index":0,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.0,"upper":0.1,"upper_inclusive":false},{"index":1,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.1,"upper":0.2,"upper_inclusive":false},{"index":2,"count":34,"confidence":0.26534026773536906,"empirical_correctness":0.47058823529411764,"signed_gap":-0.20524796755874858,"absolute_gap":0.20524796755874858,"lower":0.2,"upper":0.3,"upper_inclusive":false},{"index":3,"count":17,"confidence":0.3468729397829841,"empirical_correctness":0.5882352941176471,"signed_gap":-0.24136235433466297,"absolute_gap":0.24136235433466297,"lower":0.3,"upper":0.4,"upper_inclusive":false},{"index":4,"count":3,"confidence":0.452316015958786,"empirical_correctness":0.6666666666666666,"signed_gap":-0.21435065070788062,"absolute_gap":0.21435065070788062,"lower":0.4,"upper":0.5,"upper_inclusive":false},{"index":5,"count":2,"confidence":0.5704308450222015,"empirical_correctness":1.0,"signed_gap":-0.42956915497779846,"absolute_gap":0.42956915497779846,"lower":0.5,"upper":0.6,"upper_inclusive":false},{"index":6,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.6,"upper":0.7,"upper_inclusive":false},{"index":7,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.7,"upper":0.8,"upper_inclusive":false},{"index":8,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.8,"upper":0.9,"upper_inclusive":false},{"index":9,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.9,"upper":1.0,"upper_inclusive":true}]},"equal_mass":{"n":56,"bin_count":10,"ece":0.2493066739823137,"signed_mean_gap":-0.22471037826367787,"absolute_mean_gap":0.22471037826367787,"bins":[{"index":0,"count":5,"confidence":0.2229290634393692,"empirical_correctness":0.4,"signed_gap":-0.17707093656063083,"absolute_gap":0.17707093656063083,"min_confidence":0.2119300663471222,"max_confidence":0.2326507717370987},{"index":1,"count":6,"confidence":0.2474815919995308,"empirical_correctness":0.16666666666666666,"signed_gap":0.08081492533286413,"absolute_gap":0.08081492533286413,"min_confidence":0.23486265540122986,"max_confidence":0.2551266849040985},{"index":2,"count":5,"confidence":0.26270377039909365,"empirical_correctness":0.4,"signed_gap":-0.13729622960090637,"absolute_gap":0.13729622960090637,"min_confidence":0.257392019033432,"max_confidence":0.26999619603157043},{"index":3,"count":6,"confidence":0.2744844953219096,"empirical_correctness":0.5,"signed_gap":-0.2255155046780904,"absolute_gap":0.2255155046780904,"min_confidence":0.27052757143974304,"max_confidence":0.27964335680007935},{"index":4,"count":6,"confidence":0.28279117743174237,"empirical_correctness":0.6666666666666666,"signed_gap":-0.38387548923492426,"absolute_gap":0.38387548923492426,"min_confidence":0.2800787389278412,"max_confidence":0.28770309686660767},{"index":5,"count":5,"confidence":0.2930248260498047,"empirical_correctness":0.6,"signed_gap":-0.3069751739501953,"absolute_gap":0.3069751739501953,"min_confidence":0.29002827405929565,"max_confidence":0.29888200759887695},{"index":6,"count":6,"confidence":0.31211761633555096,"empirical_correctness":0.6666666666666666,"signed_gap":-0.35454905033111567,"absolute_gap":0.35454905033111567,"min_confidence":0.29973721504211426,"max_confidence":0.3243959844112396},{"index":7,"count":5,"confidence":0.3473890423774719,"empirical_correctness":0.8,"signed_gap":-0.4526109576225281,"absolute_gap":0.4526109576225281,"min_confidence":0.3392273783683777,"max_confidence":0.3576679527759552},{"index":8,"count":6,"confidence":0.3673011213541031,"empirical_correctness":0.3333333333333333,"signed_gap":0.033967788020769774,"absolute_gap":0.033967788020769774,"min_confidence":0.3583092987537384,"max_confidence":0.37514349818229675},{"index":9,"count":6,"confidence":0.4801548818747203,"empirical_correctness":0.8333333333333334,"signed_gap":-0.3531784514586131,"absolute_gap":0.3531784514586131,"min_confidence":0.3831195533275604,"max_confidence":0.5867141485214233}]}},"calibration_singleton_labels":{"equal_width":{"n":56,"bin_count":10,"ece":0.22471037826367787,"signed_mean_gap":-0.22471037826367787,"absolute_mean_gap":0.22471037826367787,"bins":[{"index":0,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.0,"upper":0.1,"upper_inclusive":false},{"index":1,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.1,"upper":0.2,"upper_inclusive":false},{"index":2,"count":34,"confidence":0.26534026773536906,"empirical_correctness":0.47058823529411764,"signed_gap":-0.20524796755874858,"absolute_gap":0.20524796755874858,"lower":0.2,"upper":0.3,"upper_inclusive":false},{"index":3,"count":17,"confidence":0.3468729397829841,"empirical_correctness":0.5882352941176471,"signed_gap":-0.24136235433466297,"absolute_gap":0.24136235433466297,"lower":0.3,"upper":0.4,"upper_inclusive":false},{"index":4,"count":3,"confidence":0.452316015958786,"empirical_correctness":0.6666666666666666,"signed_gap":-0.21435065070788062,"absolute_gap":0.21435065070788062,"lower":0.4,"upper":0.5,"upper_inclusive":false},{"index":5,"count":2,"confidence":0.5704308450222015,"empirical_correctness":1.0,"signed_gap":-0.42956915497779846,"absolute_gap":0.42956915497779846,"lower":0.5,"upper":0.6,"upper_inclusive":false},{"index":6,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.6,"upper":0.7,"upper_inclusive":false},{"index":7,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.7,"upper":0.8,"upper_inclusive":false},{"index":8,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.8,"upper":0.9,"upper_inclusive":false},{"index":9,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.9,"upper":1.0,"upper_inclusive":true}]},"equal_mass":{"n":56,"bin_count":10,"ece":0.2493066739823137,"signed_mean_gap":-0.22471037826367787,"absolute_mean_gap":0.22471037826367787,"bins":[{"index":0,"count":5,"confidence":0.2229290634393692,"empirical_correctness":0.4,"signed_gap":-0.17707093656063083,"absolute_gap":0.17707093656063083,"min_confidence":0.2119300663471222,"max_confidence":0.2326507717370987},{"index":1,"count":6,"confidence":0.2474815919995308,"empirical_correctness":0.16666666666666666,"signed_gap":0.08081492533286413,"absolute_gap":0.08081492533286413,"min_confidence":0.23486265540122986,"max_confidence":0.2551266849040985},{"index":2,"count":5,"confidence":0.26270377039909365,"empirical_correctness":0.4,"signed_gap":-0.13729622960090637,"absolute_gap":0.13729622960090637,"min_confidence":0.257392019033432,"max_confidence":0.26999619603157043},{"index":3,"count":6,"confidence":0.2744844953219096,"empirical_correctness":0.5,"signed_gap":-0.2255155046780904,"absolute_gap":0.2255155046780904,"min_confidence":0.27052757143974304,"max_confidence":0.27964335680007935},{"index":4,"count":6,"confidence":0.28279117743174237,"empirical_correctness":0.6666666666666666,"signed_gap":-0.38387548923492426,"absolute_gap":0.38387548923492426,"min_confidence":0.2800787389278412,"max_confidence":0.28770309686660767},{"index":5,"count":5,"confidence":0.2930248260498047,"empirical_correctness":0.6,"signed_gap":-0.3069751739501953,"absolute_gap":0.3069751739501953,"min_confidence":0.29002827405929565,"max_confidence":0.29888200759887695},{"index":6,"count":6,"confidence":0.31211761633555096,"empirical_correctness":0.6666666666666666,"signed_gap":-0.35454905033111567,"absolute_gap":0.35454905033111567,"min_confidence":0.29973721504211426,"max_confidence":0.3243959844112396},{"index":7,"count":5,"confidence":0.3473890423774719,"empirical_correctness":0.8,"signed_gap":-0.4526109576225281,"absolute_gap":0.4526109576225281,"min_confidence":0.3392273783683777,"max_confidence":0.3576679527759552},{"index":8,"count":6,"confidence":0.3673011213541031,"empirical_correctness":0.3333333333333333,"signed_gap":0.033967788020769774,"absolute_gap":0.033967788020769774,"min_confidence":0.3583092987537384,"max_confidence":0.37514349818229675},{"index":9,"count":6,"confidence":0.4801548818747203,"empirical_correctness":0.8333333333333334,"signed_gap":-0.3531784514586131,"absolute_gap":0.3531784514586131,"min_confidence":0.3831195533275604,"max_confidence":0.5867141485214233}]}},"latency_ms":{"n":56,"median":18.03169646882452,"p95":18.375024490524083},"per_action":{"keep":{"n":8,"raw":{"n":8,"acceptable":8,"unsafe":0,"review":0,"automated_count":8,"automated_acceptable":8,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":1.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.6755924351161198,"upper":1.0,"confidence_level":0.95}},"effective":{"n":8,"acceptable":0,"unsafe":0,"review":8,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":8,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.32440756488388023,"confidence_level":0.95}}},"drop":{"n":22,"raw":{"n":22,"acceptable":17,"unsafe":5,"review":0,"automated_count":22,"automated_acceptable":17,"automated_unsafe":5,"unnecessary_reviews":0,"accuracy":0.7727272727272727,"unsafe_rate":0.22727272727272727,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.7727272727272727,"conditional_unsafe":0.22727272727272727,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.5656004682477829,"upper":0.8987696001475147,"confidence_level":0.95}},"effective":{"n":22,"acceptable":3,"unsafe":0,"review":19,"automated_count":3,"automated_acceptable":3,"automated_unsafe":0,"unnecessary_reviews":19,"accuracy":0.13636363636363635,"unsafe_rate":0.0,"review_rate":0.8636363636363636,"automated_coverage":0.13636363636363635,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.8636363636363636,"accuracy_wilson_95":{"lower":0.04749003714055253,"upper":0.33334987166571717,"confidence_level":0.95}}},"mask":{"n":8,"raw":{"n":8,"acceptable":5,"unsafe":0,"review":0,"automated_count":8,"automated_acceptable":5,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":0.625,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.625,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.30574239460262737,"upper":0.8631557141764026,"confidence_level":0.95}},"effective":{"n":8,"acceptable":1,"unsafe":0,"review":7,"automated_count":1,"automated_acceptable":1,"automated_unsafe":0,"unnecessary_reviews":7,"accuracy":0.125,"unsafe_rate":0.0,"review_rate":0.875,"automated_coverage":0.125,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.875,"accuracy_wilson_95":{"lower":0.02241749145005667,"upper":0.4708881822128535,"confidence_level":0.95}}},"pseudonymize":{"n":8,"raw":{"n":8,"acceptable":0,"unsafe":7,"review":0,"automated_count":8,"automated_acceptable":0,"automated_unsafe":7,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":0.875,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":0.875,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.32440756488388023,"confidence_level":0.95}},"effective":{"n":8,"acceptable":0,"unsafe":0,"review":8,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":8,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.32440756488388023,"confidence_level":0.95}}},"generalize":{"n":8,"raw":{"n":8,"acceptable":0,"unsafe":8,"review":0,"automated_count":8,"automated_acceptable":0,"automated_unsafe":8,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":1.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":1.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.32440756488388023,"confidence_level":0.95}},"effective":{"n":8,"acceptable":0,"unsafe":0,"review":8,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":8,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.32440756488388023,"confidence_level":0.95}}},"review":{"n":2,"raw":{"n":2,"acceptable":0,"unsafe":2,"review":0,"automated_count":2,"automated_acceptable":0,"automated_unsafe":2,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":1.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":1.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.6576197724933469,"confidence_level":0.95}},"effective":{"n":2,"acceptable":2,"unsafe":0,"review":2,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":1.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.34238022750665303,"upper":1.0,"confidence_level":0.95}}}}}},"composite":{"n":198,"scope":"descriptive sum across four correlated panels","raw":{"n":198,"acceptable":103,"unsafe":48,"review":0,"automated_count":198,"automated_acceptable":103,"automated_unsafe":48,"unnecessary_reviews":0,"accuracy":0.5202020202020202,"unsafe_rate":0.24242424242424243,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.5202020202020202,"conditional_unsafe":0.24242424242424243,"unnecessary_review_rate":0.0},"effective":{"n":198,"acceptable":40,"unsafe":0,"review":161,"automated_count":37,"automated_acceptable":29,"automated_unsafe":0,"unnecessary_reviews":150,"accuracy":0.20202020202020202,"unsafe_rate":0.0,"review_rate":0.8131313131313131,"automated_coverage":0.18686868686868688,"conditional_correctness":0.7837837837837838,"conditional_unsafe":0.0,"unnecessary_review_rate":0.7575757575757576}}},{"id":"elnachto--laya-triage-en-fp32","platform":"Local","model_id":"elnachto/laya-triage-en","revision":"9596364eddd73379a2bd9e6957463d6471540014","loading":"float32 / none","scoring_kind":"Native decision readout","settings":{"dtype":"float32","quantization":"none","max_tokens":512,"head_max_tokens":192,"temperature":0.9659,"autocast":false,"truncation":false,"cpu_offload":false,"guard_threshold":0.45,"seed":0,"tf32":false},"scoring_metadata":{"backend":"laya_native_local_builtin","readout":"native_mask_slot_scorer_numpy_float32_softmax","probability_source":"native_choice_unrounded_published_temperature_not_task_calibrated","latency_scope":"synchronized serial complete encoder+head+act forward,input transfer and native readout; tokenization excluded","native_disclosures":["Stock installed torch attention in place of ONNX dynamic equivalent; no custom kernels or repo imports.","FP32/no-autocast explicit precision: convert every saved F16/BF16/F32 tensor toFP32; no quantization.","Unrounded native probability; guard uses winner p, not entropy-concentration confidence.","Act head fully loaded and evaluated; not used to alter six-action answer.","Native choice:6-10 bucket else choice scalar0, T=0.9659. No task fitting, priors correction, prompt change or finetuning here."],"equal_mass_convention":"rank bins with floor boundaries at floor(i*n/k), k=min(10,n//2); minimum two rows"},"load_ms":null,"peak_allocated_bytes":1748760576,"peak_reserved_bytes":1870659584,"panels":{"diagnostic43-no_samples":{"n":43,"raw":{"n":43,"acceptable":10,"unsafe":14,"review":0,"automated_count":43,"automated_acceptable":10,"automated_unsafe":14,"unnecessary_reviews":0,"accuracy":0.23255813953488372,"unsafe_rate":0.32558139534883723,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.23255813953488372,"conditional_unsafe":0.32558139534883723,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.1315367847938448,"upper":0.37744520297809003,"confidence_level":0.95}},"effective":{"n":43,"acceptable":5,"unsafe":0,"review":43,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":38,"accuracy":0.11627906976744186,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":0.8837209302325582,"accuracy_wilson_95":{"lower":0.05070427739804166,"upper":0.24479161810082128,"confidence_level":0.95}},"confusion":{"raw":{"keep":{"keep":0,"drop":5,"mask":0,"pseudonymize":7,"generalize":0,"review":0},"drop":{"keep":0,"drop":6,"mask":0,"pseudonymize":10,"generalize":0,"review":0},"mask":{"keep":0,"drop":1,"mask":0,"pseudonymize":1,"generalize":0,"review":0},"pseudonymize":{"keep":0,"drop":0,"mask":0,"pseudonymize":2,"generalize":0,"review":0},"generalize":{"keep":0,"drop":1,"mask":0,"pseudonymize":5,"generalize":0,"review":0},"review":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0}},"effective":{"keep":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":12},"drop":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":16},"mask":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":2},"pseudonymize":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":2},"generalize":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":6},"review":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0}}},"probability_metrics":{"singleton_n":38,"multiacceptable_n":5,"multiacceptable_coverage":0.11627906976744186,"nll":1.6827312191706711,"nll_zero_mass_n":0,"brier":0.8100670275458718,"acceptable_set_n":43,"acceptable_set_nll":1.5746531977695093,"acceptable_set_zero_mass_n":0},"calibration_all_labels":{"equal_width":{"n":43,"bin_count":10,"ece":0.10662124531213629,"signed_mean_gap":0.05394321194914884,"absolute_mean_gap":0.05394321194914884,"bins":[{"index":0,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.0,"upper":0.1,"upper_inclusive":false},{"index":1,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.1,"upper":0.2,"upper_inclusive":false},{"index":2,"count":23,"confidence":0.25510531663894653,"empirical_correctness":0.30434782608695654,"signed_gap":-0.04924250944801001,"absolute_gap":0.04924250944801001,"lower":0.2,"upper":0.3,"upper_inclusive":false},{"index":3,"count":20,"confidence":0.3226067915558815,"empirical_correctness":0.15,"signed_gap":0.17260679155588152,"absolute_gap":0.17260679155588152,"lower":0.3,"upper":0.4,"upper_inclusive":false},{"index":4,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.4,"upper":0.5,"upper_inclusive":false},{"index":5,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.5,"upper":0.6,"upper_inclusive":false},{"index":6,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.6,"upper":0.7,"upper_inclusive":false},{"index":7,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.7,"upper":0.8,"upper_inclusive":false},{"index":8,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.8,"upper":0.9,"upper_inclusive":false},{"index":9,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.9,"upper":1.0,"upper_inclusive":true}]},"equal_mass":{"n":43,"bin_count":10,"ece":0.18049331667811372,"signed_mean_gap":0.05394321194914884,"absolute_mean_gap":0.05394321194914884,"bins":[{"index":0,"count":4,"confidence":0.22367118299007416,"empirical_correctness":0.0,"signed_gap":0.22367118299007416,"absolute_gap":0.22367118299007416,"min_confidence":0.21996323764324188,"max_confidence":0.22625480592250824},{"index":1,"count":4,"confidence":0.23303412646055222,"empirical_correctness":0.0,"signed_gap":0.23303412646055222,"absolute_gap":0.23303412646055222,"min_confidence":0.22696200013160706,"max_confidence":0.2393520027399063},{"index":2,"count":4,"confidence":0.24187597259879112,"empirical_correctness":0.5,"signed_gap":-0.2581240274012089,"absolute_gap":0.2581240274012089,"min_confidence":0.24011005461215973,"max_confidence":0.2440555989742279},{"index":3,"count":5,"confidence":0.26233377158641813,"empirical_correctness":0.6,"signed_gap":-0.33766622841358185,"absolute_gap":0.33766622841358185,"min_confidence":0.2459249198436737,"max_confidence":0.28836381435394287},{"index":4,"count":4,"confidence":0.2913263365626335,"empirical_correctness":0.25,"signed_gap":0.041326336562633514,"absolute_gap":0.041326336562633514,"min_confidence":0.2894086241722107,"max_confidence":0.29393258690834045},{"index":5,"count":4,"confidence":0.2991979122161865,"empirical_correctness":0.25,"signed_gap":0.04919791221618652,"absolute_gap":0.04919791221618652,"min_confidence":0.2971711754798889,"max_confidence":0.3006109893321991},{"index":6,"count":5,"confidence":0.30232524275779726,"empirical_correctness":0.2,"signed_gap":0.10232524275779725,"absolute_gap":0.10232524275779725,"min_confidence":0.30082932114601135,"max_confidence":0.3065394163131714},{"index":7,"count":4,"confidence":0.31313391774892807,"empirical_correctness":0.25,"signed_gap":0.06313391774892807,"absolute_gap":0.06313391774892807,"min_confidence":0.310803085565567,"max_confidence":0.31515172123908997},{"index":8,"count":4,"confidence":0.328149639070034,"empirical_correctness":0.0,"signed_gap":0.328149639070034,"absolute_gap":0.328149639070034,"min_confidence":0.32119694352149963,"max_confidence":0.33461856842041016},{"index":9,"count":5,"confidence":0.35494133830070496,"empirical_correctness":0.2,"signed_gap":0.15494133830070494,"absolute_gap":0.15494133830070494,"min_confidence":0.3488442003726959,"max_confidence":0.35781511664390564}]}},"calibration_singleton_labels":{"equal_width":{"n":38,"bin_count":10,"ece":0.08688214146777203,"signed_mean_gap":0.07719771446366061,"absolute_mean_gap":0.07719771446366061,"bins":[{"index":0,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.0,"upper":0.1,"upper_inclusive":false},{"index":1,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.1,"upper":0.2,"upper_inclusive":false},{"index":2,"count":19,"confidence":0.25347346773273066,"empirical_correctness":0.2631578947368421,"signed_gap":-0.009684427004111429,"absolute_gap":0.009684427004111429,"lower":0.2,"upper":0.3,"upper_inclusive":false},{"index":3,"count":19,"confidence":0.3219745927735379,"empirical_correctness":0.15789473684210525,"signed_gap":0.16407985593143265,"absolute_gap":0.16407985593143265,"lower":0.3,"upper":0.4,"upper_inclusive":false},{"index":4,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.4,"upper":0.5,"upper_inclusive":false},{"index":5,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.5,"upper":0.6,"upper_inclusive":false},{"index":6,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.6,"upper":0.7,"upper_inclusive":false},{"index":7,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.7,"upper":0.8,"upper_inclusive":false},{"index":8,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.8,"upper":0.9,"upper_inclusive":false},{"index":9,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.9,"upper":1.0,"upper_inclusive":true}]},"equal_mass":{"n":38,"bin_count":10,"ece":0.18045826373915924,"signed_mean_gap":0.0771977144636606,"absolute_mean_gap":0.0771977144636606,"bins":[{"index":0,"count":3,"confidence":0.22280997534592947,"empirical_correctness":0.0,"signed_gap":0.22280997534592947,"absolute_gap":0.22280997534592947,"min_confidence":0.21996323764324188,"max_confidence":0.22423401474952698},{"index":1,"count":4,"confidence":0.2328573279082775,"empirical_correctness":0.0,"signed_gap":0.2328573279082775,"absolute_gap":0.2328573279082775,"min_confidence":0.22625480592250824,"max_confidence":0.2393520027399063},{"index":2,"count":4,"confidence":0.24187597259879112,"empirical_correctness":0.5,"signed_gap":-0.2581240274012089,"absolute_gap":0.2581240274012089,"min_confidence":0.24011005461215973,"max_confidence":0.2440555989742279},{"index":3,"count":4,"confidence":0.26763641834259033,"empirical_correctness":0.5,"signed_gap":-0.23236358165740967,"absolute_gap":0.23236358165740967,"min_confidence":0.2459249198436737,"max_confidence":0.2894086241722107},{"index":4,"count":4,"confidence":0.2945217713713646,"empirical_correctness":0.25,"signed_gap":0.044521771371364594,"absolute_gap":0.044521771371364594,"min_confidence":0.29049018025398254,"max_confidence":0.29895177483558655},{"index":5,"count":3,"confidence":0.3004993398984273,"empirical_correctness":0.0,"signed_gap":0.3004993398984273,"absolute_gap":0.3004993398984273,"min_confidence":0.30005770921707153,"max_confidence":0.30082932114601135},{"index":6,"count":4,"confidence":0.3026992231607437,"empirical_correctness":0.25,"signed_gap":0.05269922316074371,"absolute_gap":0.05269922316074371,"min_confidence":0.3009672164916992,"max_confidence":0.3065394163131714},{"index":7,"count":4,"confidence":0.31313391774892807,"empirical_correctness":0.25,"signed_gap":0.06313391774892807,"absolute_gap":0.06313391774892807,"min_confidence":0.310803085565567,"max_confidence":0.31515172123908997},{"index":8,"count":4,"confidence":0.33170604705810547,"empirical_correctness":0.0,"signed_gap":0.33170604705810547,"absolute_gap":0.33170604705810547,"min_confidence":0.32119694352149963,"max_confidence":0.3488442003726959},{"index":9,"count":4,"confidence":0.3564656227827072,"empirical_correctness":0.25,"signed_gap":0.10646562278270721,"absolute_gap":0.10646562278270721,"min_confidence":0.3552035689353943,"max_confidence":0.35781511664390564}]}},"latency_ms":{"n":43,"median":67.66149983741343,"p95":71.06630001217127},"per_action":{"keep":{"n":12,"raw":{"n":12,"acceptable":0,"unsafe":0,"review":0,"automated_count":12,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":2.7755575615628914e-17,"upper":0.24249400665524085,"confidence_level":0.95}},"effective":{"n":12,"acceptable":0,"unsafe":0,"review":12,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":12,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":2.7755575615628914e-17,"upper":0.24249400665524085,"confidence_level":0.95}}},"drop":{"n":16,"raw":{"n":16,"acceptable":6,"unsafe":10,"review":0,"automated_count":16,"automated_acceptable":6,"automated_unsafe":10,"unnecessary_reviews":0,"accuracy":0.375,"unsafe_rate":0.625,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.375,"conditional_unsafe":0.625,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.18481232558863633,"upper":0.6135895945449727,"confidence_level":0.95}},"effective":{"n":16,"acceptable":0,"unsafe":0,"review":16,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":16,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.1936076805344365,"confidence_level":0.95}}},"mask":{"n":2,"raw":{"n":2,"acceptable":0,"unsafe":1,"review":0,"automated_count":2,"automated_acceptable":0,"automated_unsafe":1,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":0.5,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":0.5,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.6576197724933469,"confidence_level":0.95}},"effective":{"n":2,"acceptable":0,"unsafe":0,"review":2,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":2,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.6576197724933469,"confidence_level":0.95}}},"pseudonymize":{"n":2,"raw":{"n":2,"acceptable":2,"unsafe":0,"review":0,"automated_count":2,"automated_acceptable":2,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":1.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.34238022750665303,"upper":1.0,"confidence_level":0.95}},"effective":{"n":2,"acceptable":0,"unsafe":0,"review":2,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":2,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.6576197724933469,"confidence_level":0.95}}},"generalize":{"n":6,"raw":{"n":6,"acceptable":0,"unsafe":0,"review":0,"automated_count":6,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":2.7755575615628914e-17,"upper":0.3903342879021653,"confidence_level":0.95}},"effective":{"n":6,"acceptable":0,"unsafe":0,"review":6,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":6,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":2.7755575615628914e-17,"upper":0.3903342879021653,"confidence_level":0.95}}},"review":{"n":0,"raw":{"n":0,"acceptable":0,"unsafe":0,"review":0,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":null,"unsafe_rate":null,"review_rate":null,"automated_coverage":null,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":null},"effective":{"n":0,"acceptable":0,"unsafe":0,"review":0,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":null,"unsafe_rate":null,"review_rate":null,"automated_coverage":null,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":null}}}},"diagnostic43-compact_no_samples":{"n":43,"raw":{"n":43,"acceptable":16,"unsafe":24,"review":0,"automated_count":43,"automated_acceptable":16,"automated_unsafe":24,"unnecessary_reviews":0,"accuracy":0.37209302325581395,"unsafe_rate":0.5581395348837209,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.37209302325581395,"conditional_unsafe":0.5581395348837209,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.24376339009650572,"upper":0.5214019084031153,"confidence_level":0.95}},"effective":{"n":43,"acceptable":6,"unsafe":0,"review":40,"automated_count":3,"automated_acceptable":1,"automated_unsafe":0,"unnecessary_reviews":35,"accuracy":0.13953488372093023,"unsafe_rate":0.0,"review_rate":0.9302325581395349,"automated_coverage":0.06976744186046512,"conditional_correctness":0.3333333333333333,"conditional_unsafe":0.0,"unnecessary_review_rate":0.813953488372093,"accuracy_wilson_95":{"lower":0.06555723427964878,"upper":0.2726358796738286,"confidence_level":0.95}},"confusion":{"raw":{"keep":{"keep":11,"drop":0,"mask":0,"pseudonymize":0,"generalize":1,"review":0},"drop":{"keep":11,"drop":2,"mask":0,"pseudonymize":0,"generalize":3,"review":0},"mask":{"keep":2,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0},"pseudonymize":{"keep":2,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0},"generalize":{"keep":4,"drop":0,"mask":0,"pseudonymize":0,"generalize":2,"review":0},"review":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0}},"effective":{"keep":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":12},"drop":{"keep":0,"drop":0,"mask":2,"pseudonymize":0,"generalize":0,"review":14},"mask":{"keep":0,"drop":0,"mask":1,"pseudonymize":0,"generalize":0,"review":1},"pseudonymize":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":2},"generalize":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":6},"review":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0}}},"probability_metrics":{"singleton_n":38,"multiacceptable_n":5,"multiacceptable_coverage":0.11627906976744186,"nll":1.5792790296486778,"nll_zero_mass_n":0,"brier":0.7661189175170456,"acceptable_set_n":43,"acceptable_set_nll":1.481386425081224,"acceptable_set_zero_mass_n":0},"calibration_all_labels":{"equal_width":{"n":43,"bin_count":10,"ece":0.14195121720779771,"signed_mean_gap":-0.13300949612329172,"absolute_mean_gap":0.13300949612329172,"bins":[{"index":0,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.0,"upper":0.1,"upper_inclusive":false},{"index":1,"count":1,"confidence":0.19224700331687927,"empirical_correctness":0.0,"signed_gap":0.19224700331687927,"absolute_gap":0.19224700331687927,"lower":0.1,"upper":0.2,"upper_inclusive":false},{"index":2,"count":42,"confidence":0.2401986824614661,"empirical_correctness":0.38095238095238093,"signed_gap":-0.14075369849091482,"absolute_gap":0.14075369849091482,"lower":0.2,"upper":0.3,"upper_inclusive":false},{"index":3,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.3,"upper":0.4,"upper_inclusive":false},{"index":4,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.4,"upper":0.5,"upper_inclusive":false},{"index":5,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.5,"upper":0.6,"upper_inclusive":false},{"index":6,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.6,"upper":0.7,"upper_inclusive":false},{"index":7,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.7,"upper":0.8,"upper_inclusive":false},{"index":8,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.8,"upper":0.9,"upper_inclusive":false},{"index":9,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.9,"upper":1.0,"upper_inclusive":true}]},"equal_mass":{"n":43,"bin_count":10,"ece":0.2479623181875362,"signed_mean_gap":-0.13300949612329172,"absolute_mean_gap":0.13300949612329172,"bins":[{"index":0,"count":4,"confidence":0.20861735939979553,"empirical_correctness":0.5,"signed_gap":-0.29138264060020447,"absolute_gap":0.29138264060020447,"min_confidence":0.19224700331687927,"max_confidence":0.21804159879684448},{"index":1,"count":4,"confidence":0.22125877439975739,"empirical_correctness":0.25,"signed_gap":-0.028741225600242615,"absolute_gap":0.028741225600242615,"min_confidence":0.21948350965976715,"max_confidence":0.2227199226617813},{"index":2,"count":4,"confidence":0.2268952801823616,"empirical_correctness":0.0,"signed_gap":0.2268952801823616,"absolute_gap":0.2268952801823616,"min_confidence":0.22327309846878052,"max_confidence":0.22867992520332336},{"index":3,"count":5,"confidence":0.23027382493019105,"empirical_correctness":0.0,"signed_gap":0.23027382493019105,"absolute_gap":0.23027382493019105,"min_confidence":0.22908373177051544,"max_confidence":0.23118604719638824},{"index":4,"count":4,"confidence":0.23460329324007034,"empirical_correctness":0.75,"signed_gap":-0.5153967067599297,"absolute_gap":0.5153967067599297,"min_confidence":0.2325504720211029,"max_confidence":0.23590271174907684},{"index":5,"count":4,"confidence":0.23763063549995422,"empirical_correctness":0.25,"signed_gap":-0.012369364500045776,"absolute_gap":0.012369364500045776,"min_confidence":0.23657234013080597,"max_confidence":0.23853084444999695},{"index":6,"count":5,"confidence":0.2400142729282379,"empirical_correctness":0.6,"signed_gap":-0.3599857270717621,"absolute_gap":0.3599857270717621,"min_confidence":0.23856115341186523,"max_confidence":0.24099555611610413},{"index":7,"count":4,"confidence":0.24524063616991043,"empirical_correctness":0.5,"signed_gap":-0.25475936383008957,"absolute_gap":0.25475936383008957,"min_confidence":0.24244728684425354,"max_confidence":0.2518363893032074},{"index":8,"count":4,"confidence":0.2549079582095146,"empirical_correctness":0.75,"signed_gap":-0.4950920417904854,"absolute_gap":0.4950920417904854,"min_confidence":0.2527838349342346,"max_confidence":0.2583174705505371},{"index":9,"count":5,"confidence":0.2825070858001709,"empirical_correctness":0.2,"signed_gap":0.08250708580017091,"absolute_gap":0.08250708580017091,"min_confidence":0.26111528277397156,"max_confidence":0.29210206866264343}]}},"calibration_singleton_labels":{"equal_width":{"n":38,"bin_count":10,"ece":0.16436997330502462,"signed_mean_gap":-0.1542517099725573,"absolute_mean_gap":0.1542517099725573,"bins":[{"index":0,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.0,"upper":0.1,"upper_inclusive":false},{"index":1,"count":1,"confidence":0.19224700331687927,"empirical_correctness":0.0,"signed_gap":0.19224700331687927,"absolute_gap":0.19224700331687927,"lower":0.1,"upper":0.2,"upper_inclusive":false},{"index":2,"count":37,"confidence":0.24178886534394445,"empirical_correctness":0.40540540540540543,"signed_gap":-0.16361654006146098,"absolute_gap":0.16361654006146098,"lower":0.2,"upper":0.3,"upper_inclusive":false},{"index":3,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.3,"upper":0.4,"upper_inclusive":false},{"index":4,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.4,"upper":0.5,"upper_inclusive":false},{"index":5,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.5,"upper":0.6,"upper_inclusive":false},{"index":6,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.6,"upper":0.7,"upper_inclusive":false},{"index":7,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.7,"upper":0.8,"upper_inclusive":false},{"index":8,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.8,"upper":0.9,"upper_inclusive":false},{"index":9,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.9,"upper":1.0,"upper_inclusive":true}]},"equal_mass":{"n":38,"bin_count":10,"ece":0.24801654682347649,"signed_mean_gap":-0.15425170997255727,"absolute_mean_gap":0.15425170997255727,"bins":[{"index":0,"count":3,"confidence":0.2054759462674459,"empirical_correctness":0.6666666666666666,"signed_gap":-0.4611907203992207,"absolute_gap":0.4611907203992207,"min_confidence":0.19224700331687927,"max_confidence":0.21484215557575226},{"index":1,"count":4,"confidence":0.22089829668402672,"empirical_correctness":0.25,"signed_gap":-0.029101703315973282,"absolute_gap":0.029101703315973282,"min_confidence":0.21804159879684448,"max_confidence":0.2227199226617813},{"index":2,"count":4,"confidence":0.22874082997441292,"empirical_correctness":0.0,"signed_gap":0.22874082997441292,"absolute_gap":0.22874082997441292,"min_confidence":0.22698628902435303,"max_confidence":0.2302514910697937},{"index":3,"count":4,"confidence":0.23232442885637283,"empirical_correctness":0.25,"signed_gap":-0.017675571143627167,"absolute_gap":0.017675571143627167,"min_confidence":0.23112329840660095,"max_confidence":0.23443789780139923},{"index":4,"count":4,"confidence":0.2363400161266327,"empirical_correctness":0.75,"signed_gap":-0.5136599838733673,"absolute_gap":0.5136599838733673,"min_confidence":0.2355220913887024,"max_confidence":0.23736292123794556},{"index":5,"count":3,"confidence":0.23838281134764353,"empirical_correctness":0.0,"signed_gap":0.23838281134764353,"absolute_gap":0.23838281134764353,"min_confidence":0.23805643618106842,"max_confidence":0.23856115341186523},{"index":6,"count":4,"confidence":0.24074048548936844,"empirical_correctness":0.5,"signed_gap":-0.25925951451063156,"absolute_gap":0.25925951451063156,"min_confidence":0.23928435146808624,"max_confidence":0.24244728684425354},{"index":7,"count":4,"confidence":0.2478247731924057,"empirical_correctness":0.75,"signed_gap":-0.5021752268075943,"absolute_gap":0.5021752268075943,"min_confidence":0.24291959404945374,"max_confidence":0.2527838349342346},{"index":8,"count":4,"confidence":0.25699082016944885,"empirical_correctness":0.5,"signed_gap":-0.24300917983055115,"absolute_gap":0.24300917983055115,"min_confidence":0.25386181473731995,"max_confidence":0.26111528277397156},{"index":9,"count":4,"confidence":0.28785503655672073,"empirical_correctness":0.25,"signed_gap":0.037855036556720734,"absolute_gap":0.037855036556720734,"min_confidence":0.2812991440296173,"max_confidence":0.29210206866264343}]}},"latency_ms":{"n":43,"median":78.44200008548796,"p95":80.61565004754812},"per_action":{"keep":{"n":12,"raw":{"n":12,"acceptable":11,"unsafe":0,"review":0,"automated_count":12,"automated_acceptable":11,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":0.9166666666666666,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.9166666666666666,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.646120088858883,"upper":0.9851349055950829,"confidence_level":0.95}},"effective":{"n":12,"acceptable":0,"unsafe":0,"review":12,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":12,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":2.7755575615628914e-17,"upper":0.24249400665524085,"confidence_level":0.95}}},"drop":{"n":16,"raw":{"n":16,"acceptable":2,"unsafe":12,"review":0,"automated_count":16,"automated_acceptable":2,"automated_unsafe":12,"unnecessary_reviews":0,"accuracy":0.125,"unsafe_rate":0.75,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.125,"conditional_unsafe":0.75,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.03497748774324047,"upper":0.36022827265758695,"confidence_level":0.95}},"effective":{"n":16,"acceptable":0,"unsafe":0,"review":14,"automated_count":2,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":14,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":0.875,"automated_coverage":0.125,"conditional_correctness":0.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.875,"accuracy_wilson_95":{"lower":0.0,"upper":0.1936076805344365,"confidence_level":0.95}}},"mask":{"n":2,"raw":{"n":2,"acceptable":0,"unsafe":2,"review":0,"automated_count":2,"automated_acceptable":0,"automated_unsafe":2,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":1.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":1.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.6576197724933469,"confidence_level":0.95}},"effective":{"n":2,"acceptable":1,"unsafe":0,"review":1,"automated_count":1,"automated_acceptable":1,"automated_unsafe":0,"unnecessary_reviews":1,"accuracy":0.5,"unsafe_rate":0.0,"review_rate":0.5,"automated_coverage":0.5,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.5,"accuracy_wilson_95":{"lower":0.09453120573423074,"upper":0.9054687942657693,"confidence_level":0.95}}},"pseudonymize":{"n":2,"raw":{"n":2,"acceptable":0,"unsafe":2,"review":0,"automated_count":2,"automated_acceptable":0,"automated_unsafe":2,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":1.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":1.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.6576197724933469,"confidence_level":0.95}},"effective":{"n":2,"acceptable":0,"unsafe":0,"review":2,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":2,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.6576197724933469,"confidence_level":0.95}}},"generalize":{"n":6,"raw":{"n":6,"acceptable":2,"unsafe":4,"review":0,"automated_count":6,"automated_acceptable":2,"automated_unsafe":4,"unnecessary_reviews":0,"accuracy":0.3333333333333333,"unsafe_rate":0.6666666666666666,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.3333333333333333,"conditional_unsafe":0.6666666666666666,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.09677141110578041,"upper":0.700006684861608,"confidence_level":0.95}},"effective":{"n":6,"acceptable":0,"unsafe":0,"review":6,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":6,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":2.7755575615628914e-17,"upper":0.3903342879021653,"confidence_level":0.95}}},"review":{"n":0,"raw":{"n":0,"acceptable":0,"unsafe":0,"review":0,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":null,"unsafe_rate":null,"review_rate":null,"automated_coverage":null,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":null},"effective":{"n":0,"acceptable":0,"unsafe":0,"review":0,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":null,"unsafe_rate":null,"review_rate":null,"automated_coverage":null,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":null}}}},"challenge56-no_samples":{"n":56,"raw":{"n":56,"acceptable":34,"unsafe":21,"review":0,"automated_count":56,"automated_acceptable":34,"automated_unsafe":21,"unnecessary_reviews":0,"accuracy":0.6071428571428571,"unsafe_rate":0.375,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.6071428571428571,"conditional_unsafe":0.375,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.4763340280042271,"upper":0.7241958426988949,"confidence_level":0.95}},"effective":{"n":56,"acceptable":2,"unsafe":0,"review":56,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":54,"accuracy":0.03571428571428571,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":0.9642857142857143,"accuracy_wilson_95":{"lower":0.00984942514823639,"upper":0.12118780180490107,"confidence_level":0.95}},"confusion":{"raw":{"keep":{"keep":7,"drop":0,"mask":1,"pseudonymize":0,"generalize":0,"review":0},"drop":{"keep":6,"drop":11,"mask":0,"pseudonymize":5,"generalize":0,"review":0},"mask":{"keep":0,"drop":0,"mask":8,"pseudonymize":0,"generalize":0,"review":0},"pseudonymize":{"keep":0,"drop":0,"mask":0,"pseudonymize":8,"generalize":0,"review":0},"generalize":{"keep":8,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0},"review":{"keep":2,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0}},"effective":{"keep":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":8},"drop":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":22},"mask":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":8},"pseudonymize":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":8},"generalize":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":8},"review":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":2}}},"probability_metrics":{"singleton_n":56,"multiacceptable_n":0,"multiacceptable_coverage":0.0,"nll":1.513567640292408,"nll_zero_mass_n":0,"brier":0.7258768934553226,"acceptable_set_n":56,"acceptable_set_nll":1.513567640292408,"acceptable_set_zero_mass_n":0},"calibration_all_labels":{"equal_width":{"n":56,"bin_count":10,"ece":0.33695696773273603,"signed_mean_gap":-0.33695696773273603,"absolute_mean_gap":0.33695696773273603,"bins":[{"index":0,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.0,"upper":0.1,"upper_inclusive":false},{"index":1,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.1,"upper":0.2,"upper_inclusive":false},{"index":2,"count":43,"confidence":0.25206323626429533,"empirical_correctness":0.6046511627906976,"signed_gap":-0.3525879265264023,"absolute_gap":0.3525879265264023,"lower":0.2,"upper":0.3,"upper_inclusive":false},{"index":3,"count":13,"confidence":0.3301300498155447,"empirical_correctness":0.6153846153846154,"signed_gap":-0.2852545655690707,"absolute_gap":0.2852545655690707,"lower":0.3,"upper":0.4,"upper_inclusive":false},{"index":4,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.4,"upper":0.5,"upper_inclusive":false},{"index":5,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.5,"upper":0.6,"upper_inclusive":false},{"index":6,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.6,"upper":0.7,"upper_inclusive":false},{"index":7,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.7,"upper":0.8,"upper_inclusive":false},{"index":8,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.8,"upper":0.9,"upper_inclusive":false},{"index":9,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.9,"upper":1.0,"upper_inclusive":true}]},"equal_mass":{"n":56,"bin_count":10,"ece":0.3750290013849735,"signed_mean_gap":-0.33695696773273603,"absolute_mean_gap":0.33695696773273603,"bins":[{"index":0,"count":5,"confidence":0.2132033884525299,"empirical_correctness":0.0,"signed_gap":0.2132033884525299,"absolute_gap":0.2132033884525299,"min_confidence":0.20649515092372894,"max_confidence":0.21817775070667267},{"index":1,"count":6,"confidence":0.22862038016319275,"empirical_correctness":0.5,"signed_gap":-0.27137961983680725,"absolute_gap":0.27137961983680725,"min_confidence":0.22207018733024597,"max_confidence":0.23213066160678864},{"index":2,"count":5,"confidence":0.24011077582836152,"empirical_correctness":0.8,"signed_gap":-0.5598892241716386,"absolute_gap":0.5598892241716386,"min_confidence":0.23337280750274658,"max_confidence":0.2440747767686844},{"index":3,"count":6,"confidence":0.24710114548603693,"empirical_correctness":0.6666666666666666,"signed_gap":-0.41956552118062973,"absolute_gap":0.41956552118062973,"min_confidence":0.24444356560707092,"max_confidence":0.24951988458633423},{"index":4,"count":6,"confidence":0.2574184189240138,"empirical_correctness":0.5,"signed_gap":-0.2425815810759862,"absolute_gap":0.2425815810759862,"min_confidence":0.2512868344783783,"max_confidence":0.2630562484264374},{"index":5,"count":5,"confidence":0.26848323941230773,"empirical_correctness":0.8,"signed_gap":-0.5315167605876923,"absolute_gap":0.5315167605876923,"min_confidence":0.26508039236068726,"max_confidence":0.2726474404335022},{"index":6,"count":6,"confidence":0.277618611852328,"empirical_correctness":0.6666666666666666,"signed_gap":-0.38904805481433863,"absolute_gap":0.38904805481433863,"min_confidence":0.27330946922302246,"max_confidence":0.2853119671344757},{"index":7,"count":5,"confidence":0.29319268465042114,"empirical_correctness":0.8,"signed_gap":-0.5068073153495789,"absolute_gap":0.5068073153495789,"min_confidence":0.2868402600288391,"max_confidence":0.30078262090682983},{"index":8,"count":6,"confidence":0.31718161205450696,"empirical_correctness":0.3333333333333333,"signed_gap":-0.01615172127882636,"absolute_gap":0.01615172127882636,"min_confidence":0.3088183104991913,"max_confidence":0.32822102308273315},{"index":9,"count":6,"confidence":0.347969725728035,"empirical_correctness":1.0,"signed_gap":-0.652030274271965,"absolute_gap":0.652030274271965,"min_confidence":0.3329911530017853,"max_confidence":0.3609595000743866}]}},"calibration_singleton_labels":{"equal_width":{"n":56,"bin_count":10,"ece":0.33695696773273603,"signed_mean_gap":-0.33695696773273603,"absolute_mean_gap":0.33695696773273603,"bins":[{"index":0,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.0,"upper":0.1,"upper_inclusive":false},{"index":1,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.1,"upper":0.2,"upper_inclusive":false},{"index":2,"count":43,"confidence":0.25206323626429533,"empirical_correctness":0.6046511627906976,"signed_gap":-0.3525879265264023,"absolute_gap":0.3525879265264023,"lower":0.2,"upper":0.3,"upper_inclusive":false},{"index":3,"count":13,"confidence":0.3301300498155447,"empirical_correctness":0.6153846153846154,"signed_gap":-0.2852545655690707,"absolute_gap":0.2852545655690707,"lower":0.3,"upper":0.4,"upper_inclusive":false},{"index":4,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.4,"upper":0.5,"upper_inclusive":false},{"index":5,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.5,"upper":0.6,"upper_inclusive":false},{"index":6,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.6,"upper":0.7,"upper_inclusive":false},{"index":7,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.7,"upper":0.8,"upper_inclusive":false},{"index":8,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.8,"upper":0.9,"upper_inclusive":false},{"index":9,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.9,"upper":1.0,"upper_inclusive":true}]},"equal_mass":{"n":56,"bin_count":10,"ece":0.3750290013849735,"signed_mean_gap":-0.33695696773273603,"absolute_mean_gap":0.33695696773273603,"bins":[{"index":0,"count":5,"confidence":0.2132033884525299,"empirical_correctness":0.0,"signed_gap":0.2132033884525299,"absolute_gap":0.2132033884525299,"min_confidence":0.20649515092372894,"max_confidence":0.21817775070667267},{"index":1,"count":6,"confidence":0.22862038016319275,"empirical_correctness":0.5,"signed_gap":-0.27137961983680725,"absolute_gap":0.27137961983680725,"min_confidence":0.22207018733024597,"max_confidence":0.23213066160678864},{"index":2,"count":5,"confidence":0.24011077582836152,"empirical_correctness":0.8,"signed_gap":-0.5598892241716386,"absolute_gap":0.5598892241716386,"min_confidence":0.23337280750274658,"max_confidence":0.2440747767686844},{"index":3,"count":6,"confidence":0.24710114548603693,"empirical_correctness":0.6666666666666666,"signed_gap":-0.41956552118062973,"absolute_gap":0.41956552118062973,"min_confidence":0.24444356560707092,"max_confidence":0.24951988458633423},{"index":4,"count":6,"confidence":0.2574184189240138,"empirical_correctness":0.5,"signed_gap":-0.2425815810759862,"absolute_gap":0.2425815810759862,"min_confidence":0.2512868344783783,"max_confidence":0.2630562484264374},{"index":5,"count":5,"confidence":0.26848323941230773,"empirical_correctness":0.8,"signed_gap":-0.5315167605876923,"absolute_gap":0.5315167605876923,"min_confidence":0.26508039236068726,"max_confidence":0.2726474404335022},{"index":6,"count":6,"confidence":0.277618611852328,"empirical_correctness":0.6666666666666666,"signed_gap":-0.38904805481433863,"absolute_gap":0.38904805481433863,"min_confidence":0.27330946922302246,"max_confidence":0.2853119671344757},{"index":7,"count":5,"confidence":0.29319268465042114,"empirical_correctness":0.8,"signed_gap":-0.5068073153495789,"absolute_gap":0.5068073153495789,"min_confidence":0.2868402600288391,"max_confidence":0.30078262090682983},{"index":8,"count":6,"confidence":0.31718161205450696,"empirical_correctness":0.3333333333333333,"signed_gap":-0.01615172127882636,"absolute_gap":0.01615172127882636,"min_confidence":0.3088183104991913,"max_confidence":0.32822102308273315},{"index":9,"count":6,"confidence":0.347969725728035,"empirical_correctness":1.0,"signed_gap":-0.652030274271965,"absolute_gap":0.652030274271965,"min_confidence":0.3329911530017853,"max_confidence":0.3609595000743866}]}},"latency_ms":{"n":56,"median":58.88445011805743,"p95":60.38664985680953},"per_action":{"keep":{"n":8,"raw":{"n":8,"acceptable":7,"unsafe":0,"review":0,"automated_count":8,"automated_acceptable":7,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":0.875,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.875,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.5291118177871464,"upper":0.9775825085499433,"confidence_level":0.95}},"effective":{"n":8,"acceptable":0,"unsafe":0,"review":8,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":8,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.32440756488388023,"confidence_level":0.95}}},"drop":{"n":22,"raw":{"n":22,"acceptable":11,"unsafe":11,"review":0,"automated_count":22,"automated_acceptable":11,"automated_unsafe":11,"unnecessary_reviews":0,"accuracy":0.5,"unsafe_rate":0.5,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.5,"conditional_unsafe":0.5,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.3072210627372502,"upper":0.6927789372627497,"confidence_level":0.95}},"effective":{"n":22,"acceptable":0,"unsafe":0,"review":22,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":22,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.14865487460862087,"confidence_level":0.95}}},"mask":{"n":8,"raw":{"n":8,"acceptable":8,"unsafe":0,"review":0,"automated_count":8,"automated_acceptable":8,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":1.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.6755924351161198,"upper":1.0,"confidence_level":0.95}},"effective":{"n":8,"acceptable":0,"unsafe":0,"review":8,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":8,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.32440756488388023,"confidence_level":0.95}}},"pseudonymize":{"n":8,"raw":{"n":8,"acceptable":8,"unsafe":0,"review":0,"automated_count":8,"automated_acceptable":8,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":1.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.6755924351161198,"upper":1.0,"confidence_level":0.95}},"effective":{"n":8,"acceptable":0,"unsafe":0,"review":8,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":8,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.32440756488388023,"confidence_level":0.95}}},"generalize":{"n":8,"raw":{"n":8,"acceptable":0,"unsafe":8,"review":0,"automated_count":8,"automated_acceptable":0,"automated_unsafe":8,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":1.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":1.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.32440756488388023,"confidence_level":0.95}},"effective":{"n":8,"acceptable":0,"unsafe":0,"review":8,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":8,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.32440756488388023,"confidence_level":0.95}}},"review":{"n":2,"raw":{"n":2,"acceptable":0,"unsafe":2,"review":0,"automated_count":2,"automated_acceptable":0,"automated_unsafe":2,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":1.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":1.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.6576197724933469,"confidence_level":0.95}},"effective":{"n":2,"acceptable":2,"unsafe":0,"review":2,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":1.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.34238022750665303,"upper":1.0,"confidence_level":0.95}}}}},"challenge56-compact_no_samples":{"n":56,"raw":{"n":56,"acceptable":16,"unsafe":35,"review":0,"automated_count":56,"automated_acceptable":16,"automated_unsafe":35,"unnecessary_reviews":0,"accuracy":0.2857142857142857,"unsafe_rate":0.625,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.2857142857142857,"conditional_unsafe":0.625,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.18418775143858906,"upper":0.4147525071551667,"confidence_level":0.95}},"effective":{"n":56,"acceptable":2,"unsafe":0,"review":56,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":54,"accuracy":0.03571428571428571,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":0.9642857142857143,"accuracy_wilson_95":{"lower":0.00984942514823639,"upper":0.12118780180490107,"confidence_level":0.95}},"confusion":{"raw":{"keep":{"keep":8,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0},"drop":{"keep":22,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0},"mask":{"keep":0,"drop":0,"mask":8,"pseudonymize":0,"generalize":0,"review":0},"pseudonymize":{"keep":3,"drop":0,"mask":5,"pseudonymize":0,"generalize":0,"review":0},"generalize":{"keep":8,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0},"review":{"keep":2,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0}},"effective":{"keep":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":8},"drop":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":22},"mask":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":8},"pseudonymize":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":8},"generalize":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":8},"review":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":2}}},"probability_metrics":{"singleton_n":56,"multiacceptable_n":0,"multiacceptable_coverage":0.0,"nll":1.6719262465661597,"nll_zero_mass_n":0,"brier":0.7871662280537117,"acceptable_set_n":56,"acceptable_set_nll":1.6719262465661597,"acceptable_set_zero_mass_n":0},"calibration_all_labels":{"equal_width":{"n":56,"bin_count":10,"ece":0.11810695646064621,"signed_mean_gap":-0.022880948547806066,"absolute_mean_gap":0.022880948547806066,"bins":[{"index":0,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.0,"upper":0.1,"upper_inclusive":false},{"index":1,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.1,"upper":0.2,"upper_inclusive":false},{"index":2,"count":50,"confidence":0.2533265644311905,"empirical_correctness":0.2,"signed_gap":0.05332656443119049,"absolute_gap":0.05332656443119049,"lower":0.2,"upper":0.3,"upper_inclusive":false},{"index":3,"count":6,"confidence":0.34205644329388935,"empirical_correctness":1.0,"signed_gap":-0.6579435567061107,"absolute_gap":0.6579435567061107,"lower":0.3,"upper":0.4,"upper_inclusive":false},{"index":4,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.4,"upper":0.5,"upper_inclusive":false},{"index":5,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.5,"upper":0.6,"upper_inclusive":false},{"index":6,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.6,"upper":0.7,"upper_inclusive":false},{"index":7,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.7,"upper":0.8,"upper_inclusive":false},{"index":8,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.8,"upper":0.9,"upper_inclusive":false},{"index":9,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.9,"upper":1.0,"upper_inclusive":true}]},"equal_mass":{"n":56,"bin_count":10,"ece":0.27033237527523724,"signed_mean_gap":-0.022880948547806063,"absolute_mean_gap":0.022880948547806063,"bins":[{"index":0,"count":5,"confidence":0.21762303709983827,"empirical_correctness":0.0,"signed_gap":0.21762303709983827,"absolute_gap":0.21762303709983827,"min_confidence":0.21323160827159882,"max_confidence":0.2224089652299881},{"index":1,"count":6,"confidence":0.22514963895082474,"empirical_correctness":0.0,"signed_gap":0.22514963895082474,"absolute_gap":0.22514963895082474,"min_confidence":0.2228454053401947,"max_confidence":0.23063941299915314},{"index":2,"count":5,"confidence":0.23783985674381256,"empirical_correctness":0.0,"signed_gap":0.23783985674381256,"absolute_gap":0.23783985674381256,"min_confidence":0.2341790646314621,"max_confidence":0.24224057793617249},{"index":3,"count":6,"confidence":0.24600221465031305,"empirical_correctness":0.0,"signed_gap":0.24600221465031305,"absolute_gap":0.24600221465031305,"min_confidence":0.24286428093910217,"max_confidence":0.24913550913333893},{"index":4,"count":6,"confidence":0.25345296909411746,"empirical_correctness":0.16666666666666666,"signed_gap":0.0867863024274508,"absolute_gap":0.0867863024274508,"min_confidence":0.24949756264686584,"max_confidence":0.25727710127830505},{"index":5,"count":5,"confidence":0.26073930859565736,"empirical_correctness":0.0,"signed_gap":0.26073930859565736,"absolute_gap":0.26073930859565736,"min_confidence":0.2581566870212555,"max_confidence":0.26264503598213196},{"index":6,"count":6,"confidence":0.27032514413197833,"empirical_correctness":0.3333333333333333,"signed_gap":-0.06300818920135498,"absolute_gap":0.06300818920135498,"min_confidence":0.26820996403694153,"max_confidence":0.2715466320514679},{"index":7,"count":5,"confidence":0.277142208814621,"empirical_correctness":0.6,"signed_gap":-0.322857791185379,"absolute_gap":0.322857791185379,"min_confidence":0.2744669020175934,"max_confidence":0.2800440490245819},{"index":8,"count":6,"confidence":0.28833772738774616,"empirical_correctness":0.6666666666666666,"signed_gap":-0.3783289392789205,"absolute_gap":0.3783289392789205,"min_confidence":0.2804361581802368,"max_confidence":0.2999461591243744},{"index":9,"count":6,"confidence":0.34205644329388935,"empirical_correctness":1.0,"signed_gap":-0.6579435567061107,"absolute_gap":0.6579435567061107,"min_confidence":0.3258392810821533,"max_confidence":0.3618347644805908}]}},"calibration_singleton_labels":{"equal_width":{"n":56,"bin_count":10,"ece":0.11810695646064621,"signed_mean_gap":-0.022880948547806066,"absolute_mean_gap":0.022880948547806066,"bins":[{"index":0,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.0,"upper":0.1,"upper_inclusive":false},{"index":1,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.1,"upper":0.2,"upper_inclusive":false},{"index":2,"count":50,"confidence":0.2533265644311905,"empirical_correctness":0.2,"signed_gap":0.05332656443119049,"absolute_gap":0.05332656443119049,"lower":0.2,"upper":0.3,"upper_inclusive":false},{"index":3,"count":6,"confidence":0.34205644329388935,"empirical_correctness":1.0,"signed_gap":-0.6579435567061107,"absolute_gap":0.6579435567061107,"lower":0.3,"upper":0.4,"upper_inclusive":false},{"index":4,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.4,"upper":0.5,"upper_inclusive":false},{"index":5,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.5,"upper":0.6,"upper_inclusive":false},{"index":6,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.6,"upper":0.7,"upper_inclusive":false},{"index":7,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.7,"upper":0.8,"upper_inclusive":false},{"index":8,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.8,"upper":0.9,"upper_inclusive":false},{"index":9,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.9,"upper":1.0,"upper_inclusive":true}]},"equal_mass":{"n":56,"bin_count":10,"ece":0.27033237527523724,"signed_mean_gap":-0.022880948547806063,"absolute_mean_gap":0.022880948547806063,"bins":[{"index":0,"count":5,"confidence":0.21762303709983827,"empirical_correctness":0.0,"signed_gap":0.21762303709983827,"absolute_gap":0.21762303709983827,"min_confidence":0.21323160827159882,"max_confidence":0.2224089652299881},{"index":1,"count":6,"confidence":0.22514963895082474,"empirical_correctness":0.0,"signed_gap":0.22514963895082474,"absolute_gap":0.22514963895082474,"min_confidence":0.2228454053401947,"max_confidence":0.23063941299915314},{"index":2,"count":5,"confidence":0.23783985674381256,"empirical_correctness":0.0,"signed_gap":0.23783985674381256,"absolute_gap":0.23783985674381256,"min_confidence":0.2341790646314621,"max_confidence":0.24224057793617249},{"index":3,"count":6,"confidence":0.24600221465031305,"empirical_correctness":0.0,"signed_gap":0.24600221465031305,"absolute_gap":0.24600221465031305,"min_confidence":0.24286428093910217,"max_confidence":0.24913550913333893},{"index":4,"count":6,"confidence":0.25345296909411746,"empirical_correctness":0.16666666666666666,"signed_gap":0.0867863024274508,"absolute_gap":0.0867863024274508,"min_confidence":0.24949756264686584,"max_confidence":0.25727710127830505},{"index":5,"count":5,"confidence":0.26073930859565736,"empirical_correctness":0.0,"signed_gap":0.26073930859565736,"absolute_gap":0.26073930859565736,"min_confidence":0.2581566870212555,"max_confidence":0.26264503598213196},{"index":6,"count":6,"confidence":0.27032514413197833,"empirical_correctness":0.3333333333333333,"signed_gap":-0.06300818920135498,"absolute_gap":0.06300818920135498,"min_confidence":0.26820996403694153,"max_confidence":0.2715466320514679},{"index":7,"count":5,"confidence":0.277142208814621,"empirical_correctness":0.6,"signed_gap":-0.322857791185379,"absolute_gap":0.322857791185379,"min_confidence":0.2744669020175934,"max_confidence":0.2800440490245819},{"index":8,"count":6,"confidence":0.28833772738774616,"empirical_correctness":0.6666666666666666,"signed_gap":-0.3783289392789205,"absolute_gap":0.3783289392789205,"min_confidence":0.2804361581802368,"max_confidence":0.2999461591243744},{"index":9,"count":6,"confidence":0.34205644329388935,"empirical_correctness":1.0,"signed_gap":-0.6579435567061107,"absolute_gap":0.6579435567061107,"min_confidence":0.3258392810821533,"max_confidence":0.3618347644805908}]}},"latency_ms":{"n":56,"median":68.37605009786785,"p95":70.09414996718988},"per_action":{"keep":{"n":8,"raw":{"n":8,"acceptable":8,"unsafe":0,"review":0,"automated_count":8,"automated_acceptable":8,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":1.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.6755924351161198,"upper":1.0,"confidence_level":0.95}},"effective":{"n":8,"acceptable":0,"unsafe":0,"review":8,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":8,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.32440756488388023,"confidence_level":0.95}}},"drop":{"n":22,"raw":{"n":22,"acceptable":0,"unsafe":22,"review":0,"automated_count":22,"automated_acceptable":0,"automated_unsafe":22,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":1.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":1.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.14865487460862087,"confidence_level":0.95}},"effective":{"n":22,"acceptable":0,"unsafe":0,"review":22,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":22,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.14865487460862087,"confidence_level":0.95}}},"mask":{"n":8,"raw":{"n":8,"acceptable":8,"unsafe":0,"review":0,"automated_count":8,"automated_acceptable":8,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":1.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.6755924351161198,"upper":1.0,"confidence_level":0.95}},"effective":{"n":8,"acceptable":0,"unsafe":0,"review":8,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":8,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.32440756488388023,"confidence_level":0.95}}},"pseudonymize":{"n":8,"raw":{"n":8,"acceptable":0,"unsafe":3,"review":0,"automated_count":8,"automated_acceptable":0,"automated_unsafe":3,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":0.375,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":0.375,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.32440756488388023,"confidence_level":0.95}},"effective":{"n":8,"acceptable":0,"unsafe":0,"review":8,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":8,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.32440756488388023,"confidence_level":0.95}}},"generalize":{"n":8,"raw":{"n":8,"acceptable":0,"unsafe":8,"review":0,"automated_count":8,"automated_acceptable":0,"automated_unsafe":8,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":1.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":1.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.32440756488388023,"confidence_level":0.95}},"effective":{"n":8,"acceptable":0,"unsafe":0,"review":8,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":8,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.32440756488388023,"confidence_level":0.95}}},"review":{"n":2,"raw":{"n":2,"acceptable":0,"unsafe":2,"review":0,"automated_count":2,"automated_acceptable":0,"automated_unsafe":2,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":1.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":1.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.6576197724933469,"confidence_level":0.95}},"effective":{"n":2,"acceptable":2,"unsafe":0,"review":2,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":1.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.34238022750665303,"upper":1.0,"confidence_level":0.95}}}}}},"composite":{"n":198,"scope":"descriptive sum across four correlated panels","raw":{"n":198,"acceptable":76,"unsafe":94,"review":0,"automated_count":198,"automated_acceptable":76,"automated_unsafe":94,"unnecessary_reviews":0,"accuracy":0.3838383838383838,"unsafe_rate":0.47474747474747475,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.3838383838383838,"conditional_unsafe":0.47474747474747475,"unnecessary_review_rate":0.0},"effective":{"n":198,"acceptable":15,"unsafe":0,"review":195,"automated_count":3,"automated_acceptable":1,"automated_unsafe":0,"unnecessary_reviews":181,"accuracy":0.07575757575757576,"unsafe_rate":0.0,"review_rate":0.9848484848484849,"automated_coverage":0.015151515151515152,"conditional_correctness":0.3333333333333333,"conditional_unsafe":0.0,"unnecessary_review_rate":0.9141414141414141}}},{"id":"gtm-k--laya-maintenance-decision-assist-fp32","platform":"Local","model_id":"gtm-k/laya-maintenance-decision-assist","revision":"8989e4ed2106315ed15692a9228aee9e509f70ff","loading":"float32 / none","scoring_kind":"Native decision readout","settings":{"dtype":"float32","quantization":"none","max_tokens":512,"head_max_tokens":192,"temperature":1.0,"autocast":false,"truncation":false,"cpu_offload":false,"guard_threshold":0.45,"seed":0,"tf32":false},"scoring_metadata":{"backend":"laya_native_local_builtin","readout":"native_mask_slot_scorer_numpy_float32_softmax","probability_source":"native_choice_unrounded_published_temperature_not_task_calibrated","latency_scope":"synchronized serial complete encoder+head+act forward,input transfer and native readout; tokenization excluded","native_disclosures":["Stock installed torch attention in place of ONNX dynamic equivalent; no custom kernels or repo imports.","FP32/no-autocast explicit precision: convert every saved F16/BF16/F32 tensor toFP32; no quantization.","Unrounded native probability; guard uses winner p, not entropy-concentration confidence.","Act head fully loaded and evaluated; not used to alter six-action answer.","Native choice:6-10 bucket else choice scalar0, T=1.0. No task fitting, priors correction, prompt change or finetuning here."],"equal_mass_convention":"rank bins with floor boundaries at floor(i*n/k), k=min(10,n//2); minimum two rows"},"load_ms":null,"peak_allocated_bytes":1748760576,"peak_reserved_bytes":1870659584,"panels":{"diagnostic43-no_samples":{"n":43,"raw":{"n":43,"acceptable":18,"unsafe":12,"review":0,"automated_count":43,"automated_acceptable":18,"automated_unsafe":12,"unnecessary_reviews":0,"accuracy":0.4186046511627907,"unsafe_rate":0.27906976744186046,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.4186046511627907,"conditional_unsafe":0.27906976744186046,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.28384550927849195,"upper":0.5667142261303578,"confidence_level":0.95}},"effective":{"n":43,"acceptable":16,"unsafe":0,"review":19,"automated_count":24,"automated_acceptable":15,"automated_unsafe":0,"unnecessary_reviews":18,"accuracy":0.37209302325581395,"unsafe_rate":0.0,"review_rate":0.4418604651162791,"automated_coverage":0.5581395348837209,"conditional_correctness":0.625,"conditional_unsafe":0.0,"unnecessary_review_rate":0.4186046511627907,"accuracy_wilson_95":{"lower":0.24376339009650572,"upper":0.5214019084031153,"confidence_level":0.95}},"confusion":{"raw":{"keep":{"keep":4,"drop":8,"mask":0,"pseudonymize":0,"generalize":0,"review":0},"drop":{"keep":7,"drop":9,"mask":0,"pseudonymize":0,"generalize":0,"review":0},"mask":{"keep":0,"drop":2,"mask":0,"pseudonymize":0,"generalize":0,"review":0},"pseudonymize":{"keep":2,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0},"generalize":{"keep":3,"drop":3,"mask":0,"pseudonymize":0,"generalize":0,"review":0},"review":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0}},"effective":{"keep":{"keep":4,"drop":6,"mask":0,"pseudonymize":0,"generalize":0,"review":2},"drop":{"keep":0,"drop":7,"mask":0,"pseudonymize":0,"generalize":0,"review":9},"mask":{"keep":0,"drop":1,"mask":0,"pseudonymize":0,"generalize":0,"review":1},"pseudonymize":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":2},"generalize":{"keep":0,"drop":2,"mask":0,"pseudonymize":0,"generalize":0,"review":4},"review":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0}}},"probability_metrics":{"singleton_n":38,"multiacceptable_n":5,"multiacceptable_coverage":0.11627906976744186,"nll":1.4506450242990585,"nll_zero_mass_n":0,"brier":0.7420349764244828,"acceptable_set_n":43,"acceptable_set_nll":1.3246670605934996,"acceptable_set_zero_mass_n":0},"calibration_all_labels":{"equal_width":{"n":43,"bin_count":10,"ece":0.15078968225523484,"signed_mean_gap":0.1413869760757269,"absolute_mean_gap":0.1413869760757269,"bins":[{"index":0,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.0,"upper":0.1,"upper_inclusive":false},{"index":1,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.1,"upper":0.2,"upper_inclusive":false},{"index":2,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.2,"upper":0.3,"upper_inclusive":false},{"index":3,"count":5,"confidence":0.3595683634281158,"empirical_correctness":0.4,"signed_gap":-0.0404316365718842,"absolute_gap":0.0404316365718842,"lower":0.3,"upper":0.4,"upper_inclusive":false},{"index":4,"count":16,"confidence":0.44003687612712383,"empirical_correctness":0.375,"signed_gap":0.06503687612712383,"absolute_gap":0.06503687612712383,"lower":0.4,"upper":0.5,"upper_inclusive":false},{"index":5,"count":5,"confidence":0.5427898406982422,"empirical_correctness":0.0,"signed_gap":0.5427898406982422,"absolute_gap":0.5427898406982422,"lower":0.5,"upper":0.6,"upper_inclusive":false},{"index":6,"count":6,"confidence":0.6491522689660391,"empirical_correctness":0.3333333333333333,"signed_gap":0.31581893563270574,"absolute_gap":0.31581893563270574,"lower":0.6,"upper":0.7,"upper_inclusive":false},{"index":7,"count":7,"confidence":0.7553073167800903,"empirical_correctness":0.7142857142857143,"signed_gap":0.04102160249437603,"absolute_gap":0.04102160249437603,"lower":0.7,"upper":0.8,"upper_inclusive":false},{"index":8,"count":4,"confidence":0.8362985253334045,"empirical_correctness":0.75,"signed_gap":0.08629852533340454,"absolute_gap":0.08629852533340454,"lower":0.8,"upper":0.9,"upper_inclusive":false},{"index":9,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.9,"upper":1.0,"upper_inclusive":true}]},"equal_mass":{"n":43,"bin_count":10,"ece":0.2828823355741279,"signed_mean_gap":0.1413869760757269,"absolute_mean_gap":0.1413869760757269,"bins":[{"index":0,"count":4,"confidence":0.352477490901947,"empirical_correctness":0.25,"signed_gap":0.10247749090194702,"absolute_gap":0.10247749090194702,"min_confidence":0.3389814794063568,"max_confidence":0.3631928265094757},{"index":1,"count":4,"confidence":0.4015757292509079,"empirical_correctness":0.25,"signed_gap":0.1515757292509079,"absolute_gap":0.1515757292509079,"min_confidence":0.38793185353279114,"max_confidence":0.41130542755126953},{"index":2,"count":4,"confidence":0.414373479783535,"empirical_correctness":0.0,"signed_gap":0.414373479783535,"absolute_gap":0.414373479783535,"min_confidence":0.41209372878074646,"max_confidence":0.41725677251815796},{"index":3,"count":5,"confidence":0.4464362323284149,"empirical_correctness":0.6,"signed_gap":-0.15356376767158508,"absolute_gap":0.15356376767158508,"min_confidence":0.43847882747650146,"max_confidence":0.4552059471607208},{"index":4,"count":4,"confidence":0.48313596844673157,"empirical_correctness":0.75,"signed_gap":-0.26686403155326843,"absolute_gap":0.26686403155326843,"min_confidence":0.47283217310905457,"max_confidence":0.48849591612815857},{"index":5,"count":4,"confidence":0.5306588858366013,"empirical_correctness":0.0,"signed_gap":0.5306588858366013,"absolute_gap":0.5306588858366013,"min_confidence":0.5063644051551819,"max_confidence":0.543055534362793},{"index":6,"count":5,"confidence":0.627795672416687,"empirical_correctness":0.0,"signed_gap":0.627795672416687,"absolute_gap":0.627795672416687,"min_confidence":0.5913136601448059,"max_confidence":0.6535833477973938},{"index":7,"count":4,"confidence":0.6982811838388443,"empirical_correctness":1.0,"signed_gap":-0.3017188161611557,"absolute_gap":0.3017188161611557,"min_confidence":0.6598021388053894,"max_confidence":0.7265298962593079},{"index":8,"count":4,"confidence":0.7620213627815247,"empirical_correctness":0.75,"signed_gap":0.012021362781524658,"absolute_gap":0.012021362781524658,"min_confidence":0.7356992363929749,"max_confidence":0.7907702922821045},{"index":9,"count":5,"confidence":0.8276768088340759,"empirical_correctness":0.6,"signed_gap":0.22767680883407593,"absolute_gap":0.22767680883407593,"min_confidence":0.7931899428367615,"max_confidence":0.8660135269165039}]}},"calibration_singleton_labels":{"equal_width":{"n":38,"bin_count":10,"ece":0.2267933848657106,"signed_mean_gap":0.21615348050468847,"absolute_mean_gap":0.21615348050468847,"bins":[{"index":0,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.0,"upper":0.1,"upper_inclusive":false},{"index":1,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.1,"upper":0.2,"upper_inclusive":false},{"index":2,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.2,"upper":0.3,"upper_inclusive":false},{"index":3,"count":5,"confidence":0.3595683634281158,"empirical_correctness":0.4,"signed_gap":-0.0404316365718842,"absolute_gap":0.0404316365718842,"lower":0.3,"upper":0.4,"upper_inclusive":false},{"index":4,"count":13,"confidence":0.43665404732410723,"empirical_correctness":0.23076923076923078,"signed_gap":0.20588481655487645,"absolute_gap":0.20588481655487645,"lower":0.4,"upper":0.5,"upper_inclusive":false},{"index":5,"count":5,"confidence":0.5427898406982422,"empirical_correctness":0.0,"signed_gap":0.5427898406982422,"absolute_gap":0.5427898406982422,"lower":0.5,"upper":0.6,"upper_inclusive":false},{"index":6,"count":5,"confidence":0.6414933681488038,"empirical_correctness":0.2,"signed_gap":0.44149336814880374,"absolute_gap":0.44149336814880374,"lower":0.6,"upper":0.7,"upper_inclusive":false},{"index":7,"count":7,"confidence":0.7553073167800903,"empirical_correctness":0.7142857142857143,"signed_gap":0.04102160249437603,"absolute_gap":0.04102160249437603,"lower":0.7,"upper":0.8,"upper_inclusive":false},{"index":8,"count":3,"confidence":0.8436401883761088,"empirical_correctness":0.6666666666666666,"signed_gap":0.17697352170944214,"absolute_gap":0.17697352170944214,"lower":0.8,"upper":0.9,"upper_inclusive":false},{"index":9,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.9,"upper":1.0,"upper_inclusive":true}]},"equal_mass":{"n":38,"bin_count":10,"ece":0.23060940598186694,"signed_mean_gap":0.21615348050468847,"absolute_mean_gap":0.21615348050468847,"bins":[{"index":0,"count":3,"confidence":0.3489057123661041,"empirical_correctness":0.3333333333333333,"signed_gap":0.015572379032770811,"absolute_gap":0.015572379032770811,"min_confidence":0.3389814794063568,"max_confidence":0.36077913641929626},{"index":1,"count":4,"confidence":0.38954757899045944,"empirical_correctness":0.25,"signed_gap":0.13954757899045944,"absolute_gap":0.13954757899045944,"min_confidence":0.3631928265094757,"max_confidence":0.4067028760910034},{"index":2,"count":4,"confidence":0.4128856435418129,"empirical_correctness":0.0,"signed_gap":0.4128856435418129,"absolute_gap":0.4128856435418129,"min_confidence":0.41130542755126953,"max_confidence":0.414185494184494},{"index":3,"count":4,"confidence":0.439545676112175,"empirical_correctness":0.25,"signed_gap":0.189545676112175,"absolute_gap":0.189545676112175,"min_confidence":0.41725677251815796,"max_confidence":0.4552059471607208},{"index":4,"count":4,"confidence":0.4915190264582634,"empirical_correctness":0.5,"signed_gap":-0.008480973541736603,"absolute_gap":0.008480973541736603,"min_confidence":0.4829813838005066,"max_confidence":0.5063644051551819},{"index":5,"count":3,"confidence":0.5387570460637411,"empirical_correctness":0.0,"signed_gap":0.5387570460637411,"absolute_gap":0.5387570460637411,"min_confidence":0.5345985889434814,"max_confidence":0.543055534362793},{"index":6,"count":4,"confidence":0.6213487535715103,"empirical_correctness":0.0,"signed_gap":0.6213487535715103,"absolute_gap":0.6213487535715103,"min_confidence":0.5913136601448059,"max_confidence":0.633780300617218},{"index":7,"count":4,"confidence":0.6898153275251389,"empirical_correctness":0.75,"signed_gap":-0.060184672474861145,"absolute_gap":0.060184672474861145,"min_confidence":0.6535833477973938,"max_confidence":0.7265298962593079},{"index":8,"count":4,"confidence":0.7620213627815247,"empirical_correctness":0.75,"signed_gap":0.012021362781524658,"absolute_gap":0.012021362781524658,"min_confidence":0.7356992363929749,"max_confidence":0.7907702922821045},{"index":9,"count":4,"confidence":0.831027626991272,"empirical_correctness":0.5,"signed_gap":0.331027626991272,"absolute_gap":0.331027626991272,"min_confidence":0.7931899428367615,"max_confidence":0.8660135269165039}]}},"latency_ms":{"n":43,"median":18.28541501890868,"p95":18.93992690020241},"per_action":{"keep":{"n":12,"raw":{"n":12,"acceptable":4,"unsafe":0,"review":0,"automated_count":12,"automated_acceptable":4,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":0.3333333333333333,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.3333333333333333,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.13812009109121315,"upper":0.6093779111272004,"confidence_level":0.95}},"effective":{"n":12,"acceptable":4,"unsafe":0,"review":2,"automated_count":10,"automated_acceptable":4,"automated_unsafe":0,"unnecessary_reviews":2,"accuracy":0.3333333333333333,"unsafe_rate":0.0,"review_rate":0.16666666666666666,"automated_coverage":0.8333333333333334,"conditional_correctness":0.4,"conditional_unsafe":0.0,"unnecessary_review_rate":0.16666666666666666,"accuracy_wilson_95":{"lower":0.13812009109121315,"upper":0.6093779111272004,"confidence_level":0.95}}},"drop":{"n":16,"raw":{"n":16,"acceptable":9,"unsafe":7,"review":0,"automated_count":16,"automated_acceptable":9,"automated_unsafe":7,"unnecessary_reviews":0,"accuracy":0.5625,"unsafe_rate":0.4375,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.5625,"conditional_unsafe":0.4375,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.331785563988119,"upper":0.7690134759450765,"confidence_level":0.95}},"effective":{"n":16,"acceptable":7,"unsafe":0,"review":9,"automated_count":7,"automated_acceptable":7,"automated_unsafe":0,"unnecessary_reviews":9,"accuracy":0.4375,"unsafe_rate":0.0,"review_rate":0.5625,"automated_coverage":0.4375,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.5625,"accuracy_wilson_95":{"lower":0.23098652405492354,"upper":0.6682144360118811,"confidence_level":0.95}}},"mask":{"n":2,"raw":{"n":2,"acceptable":0,"unsafe":0,"review":0,"automated_count":2,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.6576197724933469,"confidence_level":0.95}},"effective":{"n":2,"acceptable":0,"unsafe":0,"review":1,"automated_count":1,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":1,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":0.5,"automated_coverage":0.5,"conditional_correctness":0.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.5,"accuracy_wilson_95":{"lower":0.0,"upper":0.6576197724933469,"confidence_level":0.95}}},"pseudonymize":{"n":2,"raw":{"n":2,"acceptable":0,"unsafe":2,"review":0,"automated_count":2,"automated_acceptable":0,"automated_unsafe":2,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":1.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":1.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.6576197724933469,"confidence_level":0.95}},"effective":{"n":2,"acceptable":0,"unsafe":0,"review":2,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":2,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.6576197724933469,"confidence_level":0.95}}},"generalize":{"n":6,"raw":{"n":6,"acceptable":0,"unsafe":3,"review":0,"automated_count":6,"automated_acceptable":0,"automated_unsafe":3,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":0.5,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":0.5,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":2.7755575615628914e-17,"upper":0.3903342879021653,"confidence_level":0.95}},"effective":{"n":6,"acceptable":0,"unsafe":0,"review":4,"automated_count":2,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":4,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":0.6666666666666666,"automated_coverage":0.3333333333333333,"conditional_correctness":0.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.6666666666666666,"accuracy_wilson_95":{"lower":2.7755575615628914e-17,"upper":0.3903342879021653,"confidence_level":0.95}}},"review":{"n":0,"raw":{"n":0,"acceptable":0,"unsafe":0,"review":0,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":null,"unsafe_rate":null,"review_rate":null,"automated_coverage":null,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":null},"effective":{"n":0,"acceptable":0,"unsafe":0,"review":0,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":null,"unsafe_rate":null,"review_rate":null,"automated_coverage":null,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":null}}}},"diagnostic43-compact_no_samples":{"n":43,"raw":{"n":43,"acceptable":14,"unsafe":29,"review":0,"automated_count":43,"automated_acceptable":14,"automated_unsafe":29,"unnecessary_reviews":0,"accuracy":0.32558139534883723,"unsafe_rate":0.6744186046511628,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.32558139534883723,"conditional_unsafe":0.6744186046511628,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.20493297126568708,"upper":0.47483789032470525,"confidence_level":0.95}},"effective":{"n":43,"acceptable":18,"unsafe":3,"review":25,"automated_count":18,"automated_acceptable":13,"automated_unsafe":3,"unnecessary_reviews":20,"accuracy":0.4186046511627907,"unsafe_rate":0.06976744186046512,"review_rate":0.5813953488372093,"automated_coverage":0.4186046511627907,"conditional_correctness":0.7222222222222222,"conditional_unsafe":0.16666666666666666,"unnecessary_review_rate":0.46511627906976744,"accuracy_wilson_95":{"lower":0.28384550927849195,"upper":0.5667142261303578,"confidence_level":0.95}},"confusion":{"raw":{"keep":{"keep":12,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0},"drop":{"keep":14,"drop":2,"mask":0,"pseudonymize":0,"generalize":0,"review":0},"mask":{"keep":2,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0},"pseudonymize":{"keep":2,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0},"generalize":{"keep":6,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0},"review":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0}},"effective":{"keep":{"keep":11,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":1},"drop":{"keep":2,"drop":1,"mask":2,"pseudonymize":0,"generalize":0,"review":11},"mask":{"keep":0,"drop":0,"mask":1,"pseudonymize":0,"generalize":0,"review":1},"pseudonymize":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":2},"generalize":{"keep":1,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":5},"review":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0}}},"probability_metrics":{"singleton_n":38,"multiacceptable_n":5,"multiacceptable_coverage":0.11627906976744186,"nll":1.4931737401584453,"nll_zero_mass_n":0,"brier":0.7705816722891471,"acceptable_set_n":43,"acceptable_set_nll":1.4578625606284938,"acceptable_set_zero_mass_n":0},"calibration_all_labels":{"equal_width":{"n":43,"bin_count":10,"ece":0.27502345761587454,"signed_mean_gap":0.26755105062972667,"absolute_mean_gap":0.26755105062972667,"bins":[{"index":0,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.0,"upper":0.1,"upper_inclusive":false},{"index":1,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.1,"upper":0.2,"upper_inclusive":false},{"index":2,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.2,"upper":0.3,"upper_inclusive":false},{"index":3,"count":2,"confidence":0.3751276731491089,"empirical_correctness":0.0,"signed_gap":0.3751276731491089,"absolute_gap":0.3751276731491089,"lower":0.3,"upper":0.4,"upper_inclusive":false},{"index":4,"count":9,"confidence":0.43859440088272095,"empirical_correctness":0.1111111111111111,"signed_gap":0.32748328977160984,"absolute_gap":0.32748328977160984,"lower":0.4,"upper":0.5,"upper_inclusive":false},{"index":5,"count":9,"confidence":0.5461169613732232,"empirical_correctness":0.3333333333333333,"signed_gap":0.21278362803988987,"absolute_gap":0.21278362803988987,"lower":0.5,"upper":0.6,"upper_inclusive":false},{"index":6,"count":14,"confidence":0.6496130696364811,"empirical_correctness":0.5,"signed_gap":0.14961306963648113,"absolute_gap":0.14961306963648113,"lower":0.6,"upper":0.7,"upper_inclusive":false},{"index":7,"count":8,"confidence":0.7447639182209969,"empirical_correctness":0.25,"signed_gap":0.49476391822099686,"absolute_gap":0.49476391822099686,"lower":0.7,"upper":0.8,"upper_inclusive":false},{"index":8,"count":1,"confidence":0.839343249797821,"empirical_correctness":1.0,"signed_gap":-0.16065675020217896,"absolute_gap":0.16065675020217896,"lower":0.8,"upper":0.9,"upper_inclusive":false},{"index":9,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.9,"upper":1.0,"upper_inclusive":true}]},"equal_mass":{"n":43,"bin_count":10,"ece":0.3407474562179211,"signed_mean_gap":0.26755105062972667,"absolute_mean_gap":0.26755105062972667,"bins":[{"index":0,"count":4,"confidence":0.39392171055078506,"empirical_correctness":0.0,"signed_gap":0.39392171055078506,"absolute_gap":0.39392171055078506,"min_confidence":0.3526097238063812,"max_confidence":0.42122727632522583},{"index":1,"count":4,"confidence":0.42858678847551346,"empirical_correctness":0.25,"signed_gap":0.17858678847551346,"absolute_gap":0.17858678847551346,"min_confidence":0.42374980449676514,"max_confidence":0.4370839595794678},{"index":2,"count":4,"confidence":0.4771803319454193,"empirical_correctness":0.0,"signed_gap":0.4771803319454193,"absolute_gap":0.4771803319454193,"min_confidence":0.4383309781551361,"max_confidence":0.501150369644165},{"index":3,"count":5,"confidence":0.5303650975227356,"empirical_correctness":0.6,"signed_gap":-0.06963490247726434,"absolute_gap":0.06963490247726434,"min_confidence":0.5103604197502136,"max_confidence":0.5714911222457886},{"index":4,"count":4,"confidence":0.5915871858596802,"empirical_correctness":0.0,"signed_gap":0.5915871858596802,"absolute_gap":0.5915871858596802,"min_confidence":0.5841816067695618,"max_confidence":0.6042719483375549},{"index":5,"count":4,"confidence":0.620341882109642,"empirical_correctness":0.75,"signed_gap":-0.12965811789035797,"absolute_gap":0.12965811789035797,"min_confidence":0.6045418977737427,"max_confidence":0.6318666338920593},{"index":6,"count":5,"confidence":0.6586168527603149,"empirical_correctness":0.8,"signed_gap":-0.14138314723968515,"absolute_gap":0.14138314723968515,"min_confidence":0.6458131074905396,"max_confidence":0.6674713492393494},{"index":7,"count":4,"confidence":0.6789648085832596,"empirical_correctness":0.0,"signed_gap":0.6789648085832596,"absolute_gap":0.6789648085832596,"min_confidence":0.6707105040550232,"max_confidence":0.6850706338882446},{"index":8,"count":4,"confidence":0.7175456583499908,"empirical_correctness":0.0,"signed_gap":0.7175456583499908,"absolute_gap":0.7175456583499908,"min_confidence":0.7025032639503479,"max_confidence":0.7337257862091064},{"index":9,"count":5,"confidence":0.7854543924331665,"empirical_correctness":0.6,"signed_gap":0.18545439243316653,"absolute_gap":0.18545439243316653,"min_confidence":0.7550656795501709,"max_confidence":0.839343249797821}]}},"calibration_singleton_labels":{"equal_width":{"n":38,"bin_count":10,"ece":0.23496033094431223,"signed_mean_gap":0.2265047125126186,"absolute_mean_gap":0.2265047125126186,"bins":[{"index":0,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.0,"upper":0.1,"upper_inclusive":false},{"index":1,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.1,"upper":0.2,"upper_inclusive":false},{"index":2,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.2,"upper":0.3,"upper_inclusive":false},{"index":3,"count":1,"confidence":0.3526097238063812,"empirical_correctness":0.0,"signed_gap":0.3526097238063812,"absolute_gap":0.3526097238063812,"lower":0.3,"upper":0.4,"upper_inclusive":false},{"index":4,"count":8,"confidence":0.4334862232208252,"empirical_correctness":0.125,"signed_gap":0.3084862232208252,"absolute_gap":0.3084862232208252,"lower":0.4,"upper":0.5,"upper_inclusive":false},{"index":5,"count":8,"confidence":0.5413588806986809,"empirical_correctness":0.375,"signed_gap":0.16635888069868088,"absolute_gap":0.16635888069868088,"lower":0.5,"upper":0.6,"upper_inclusive":false},{"index":6,"count":14,"confidence":0.6496130696364811,"empirical_correctness":0.5,"signed_gap":0.14961306963648113,"absolute_gap":0.14961306963648113,"lower":0.6,"upper":0.7,"upper_inclusive":false},{"index":7,"count":6,"confidence":0.7536470492680868,"empirical_correctness":0.3333333333333333,"signed_gap":0.4203137159347535,"absolute_gap":0.4203137159347535,"lower":0.7,"upper":0.8,"upper_inclusive":false},{"index":8,"count":1,"confidence":0.839343249797821,"empirical_correctness":1.0,"signed_gap":-0.16065675020217896,"absolute_gap":0.16065675020217896,"lower":0.8,"upper":0.9,"upper_inclusive":false},{"index":9,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.9,"upper":1.0,"upper_inclusive":true}]},"equal_mass":{"n":38,"bin_count":10,"ece":0.29787820656048625,"signed_mean_gap":0.2265047125126186,"absolute_mean_gap":0.2265047125126186,"bins":[{"index":0,"count":3,"confidence":0.39268040657043457,"empirical_correctness":0.0,"signed_gap":0.39268040657043457,"absolute_gap":0.39268040657043457,"min_confidence":0.3526097238063812,"max_confidence":0.42122727632522583},{"index":1,"count":4,"confidence":0.42858678847551346,"empirical_correctness":0.25,"signed_gap":0.17858678847551346,"absolute_gap":0.17858678847551346,"min_confidence":0.42374980449676514,"max_confidence":0.4370839595794678},{"index":2,"count":4,"confidence":0.484905481338501,"empirical_correctness":0.0,"signed_gap":0.484905481338501,"absolute_gap":0.484905481338501,"min_confidence":0.4383309781551361,"max_confidence":0.5103604197502136},{"index":3,"count":4,"confidence":0.5353662669658661,"empirical_correctness":0.75,"signed_gap":-0.2146337330341339,"absolute_gap":0.2146337330341339,"min_confidence":0.5108727812767029,"max_confidence":0.5714911222457886},{"index":4,"count":4,"confidence":0.5966772586107254,"empirical_correctness":0.25,"signed_gap":0.3466772586107254,"absolute_gap":0.3466772586107254,"min_confidence":0.5859089493751526,"max_confidence":0.6045418977737427},{"index":5,"count":3,"confidence":0.6256085435549418,"empirical_correctness":0.6666666666666666,"signed_gap":-0.041058123111724854,"absolute_gap":0.041058123111724854,"min_confidence":0.6172409057617188,"max_confidence":0.6318666338920593},{"index":6,"count":4,"confidence":0.6564032286405563,"empirical_correctness":0.75,"signed_gap":-0.09359677135944366,"absolute_gap":0.09359677135944366,"min_confidence":0.6458131074905396,"max_confidence":0.6673969030380249},{"index":7,"count":4,"confidence":0.6745649874210358,"empirical_correctness":0.25,"signed_gap":0.42456498742103577,"absolute_gap":0.42456498742103577,"min_confidence":0.6674713492393494,"max_confidence":0.6832324266433716},{"index":8,"count":4,"confidence":0.7185224741697311,"empirical_correctness":0.0,"signed_gap":0.7185224741697311,"absolute_gap":0.7185224741697311,"min_confidence":0.6850706338882446,"max_confidence":0.7550656795501709},{"index":9,"count":4,"confidence":0.7930515706539154,"empirical_correctness":0.75,"signed_gap":0.043051570653915405,"absolute_gap":0.043051570653915405,"min_confidence":0.7612488865852356,"max_confidence":0.839343249797821}]}},"latency_ms":{"n":43,"median":21.068511006888002,"p95":21.262407931499183},"per_action":{"keep":{"n":12,"raw":{"n":12,"acceptable":12,"unsafe":0,"review":0,"automated_count":12,"automated_acceptable":12,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":1.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.7575059933447592,"upper":1.0,"confidence_level":0.95}},"effective":{"n":12,"acceptable":11,"unsafe":0,"review":1,"automated_count":11,"automated_acceptable":11,"automated_unsafe":0,"unnecessary_reviews":1,"accuracy":0.9166666666666666,"unsafe_rate":0.0,"review_rate":0.08333333333333333,"automated_coverage":0.9166666666666666,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.08333333333333333,"accuracy_wilson_95":{"lower":0.646120088858883,"upper":0.9851349055950829,"confidence_level":0.95}}},"drop":{"n":16,"raw":{"n":16,"acceptable":2,"unsafe":14,"review":0,"automated_count":16,"automated_acceptable":2,"automated_unsafe":14,"unnecessary_reviews":0,"accuracy":0.125,"unsafe_rate":0.875,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.125,"conditional_unsafe":0.875,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.03497748774324047,"upper":0.36022827265758695,"confidence_level":0.95}},"effective":{"n":16,"acceptable":1,"unsafe":2,"review":11,"automated_count":5,"automated_acceptable":1,"automated_unsafe":2,"unnecessary_reviews":11,"accuracy":0.0625,"unsafe_rate":0.125,"review_rate":0.6875,"automated_coverage":0.3125,"conditional_correctness":0.2,"conditional_unsafe":0.4,"unnecessary_review_rate":0.6875,"accuracy_wilson_95":{"lower":0.01111934476464252,"upper":0.28328737570298945,"confidence_level":0.95}}},"mask":{"n":2,"raw":{"n":2,"acceptable":0,"unsafe":2,"review":0,"automated_count":2,"automated_acceptable":0,"automated_unsafe":2,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":1.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":1.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.6576197724933469,"confidence_level":0.95}},"effective":{"n":2,"acceptable":1,"unsafe":0,"review":1,"automated_count":1,"automated_acceptable":1,"automated_unsafe":0,"unnecessary_reviews":1,"accuracy":0.5,"unsafe_rate":0.0,"review_rate":0.5,"automated_coverage":0.5,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.5,"accuracy_wilson_95":{"lower":0.09453120573423074,"upper":0.9054687942657693,"confidence_level":0.95}}},"pseudonymize":{"n":2,"raw":{"n":2,"acceptable":0,"unsafe":2,"review":0,"automated_count":2,"automated_acceptable":0,"automated_unsafe":2,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":1.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":1.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.6576197724933469,"confidence_level":0.95}},"effective":{"n":2,"acceptable":0,"unsafe":0,"review":2,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":2,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.6576197724933469,"confidence_level":0.95}}},"generalize":{"n":6,"raw":{"n":6,"acceptable":0,"unsafe":6,"review":0,"automated_count":6,"automated_acceptable":0,"automated_unsafe":6,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":1.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":1.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":2.7755575615628914e-17,"upper":0.3903342879021653,"confidence_level":0.95}},"effective":{"n":6,"acceptable":0,"unsafe":1,"review":5,"automated_count":1,"automated_acceptable":0,"automated_unsafe":1,"unnecessary_reviews":5,"accuracy":0.0,"unsafe_rate":0.16666666666666666,"review_rate":0.8333333333333334,"automated_coverage":0.16666666666666666,"conditional_correctness":0.0,"conditional_unsafe":1.0,"unnecessary_review_rate":0.8333333333333334,"accuracy_wilson_95":{"lower":2.7755575615628914e-17,"upper":0.3903342879021653,"confidence_level":0.95}}},"review":{"n":0,"raw":{"n":0,"acceptable":0,"unsafe":0,"review":0,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":null,"unsafe_rate":null,"review_rate":null,"automated_coverage":null,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":null},"effective":{"n":0,"acceptable":0,"unsafe":0,"review":0,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":null,"unsafe_rate":null,"review_rate":null,"automated_coverage":null,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":null}}}},"challenge56-no_samples":{"n":56,"raw":{"n":56,"acceptable":23,"unsafe":32,"review":0,"automated_count":56,"automated_acceptable":23,"automated_unsafe":32,"unnecessary_reviews":0,"accuracy":0.4107142857142857,"unsafe_rate":0.5714285714285714,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.4107142857142857,"conditional_unsafe":0.5714285714285714,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.29166774287846203,"upper":0.541224031535603,"confidence_level":0.95}},"effective":{"n":56,"acceptable":14,"unsafe":0,"review":44,"automated_count":12,"automated_acceptable":12,"automated_unsafe":0,"unnecessary_reviews":42,"accuracy":0.25,"unsafe_rate":0.0,"review_rate":0.7857142857142857,"automated_coverage":0.21428571428571427,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.75,"accuracy_wilson_95":{"lower":0.15517054688270684,"upper":0.376926421476675,"confidence_level":0.95}},"confusion":{"raw":{"keep":{"keep":8,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0},"drop":{"keep":14,"drop":8,"mask":0,"pseudonymize":0,"generalize":0,"review":0},"mask":{"keep":0,"drop":1,"mask":7,"pseudonymize":0,"generalize":0,"review":0},"pseudonymize":{"keep":8,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0},"generalize":{"keep":8,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0},"review":{"keep":2,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0}},"effective":{"keep":{"keep":5,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":3},"drop":{"keep":0,"drop":3,"mask":0,"pseudonymize":0,"generalize":0,"review":19},"mask":{"keep":0,"drop":0,"mask":4,"pseudonymize":0,"generalize":0,"review":4},"pseudonymize":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":8},"generalize":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":8},"review":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":2}}},"probability_metrics":{"singleton_n":56,"multiacceptable_n":0,"multiacceptable_coverage":0.0,"nll":1.2875332139928568,"nll_zero_mass_n":0,"brier":0.6489022049559339,"acceptable_set_n":56,"acceptable_set_nll":1.2875332139928568,"acceptable_set_zero_mass_n":0},"calibration_all_labels":{"equal_width":{"n":56,"bin_count":10,"ece":0.11847918746726853,"signed_mean_gap":0.0635254058454718,"absolute_mean_gap":0.0635254058454718,"bins":[{"index":0,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.0,"upper":0.1,"upper_inclusive":false},{"index":1,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.1,"upper":0.2,"upper_inclusive":false},{"index":2,"count":2,"confidence":0.29958322644233704,"empirical_correctness":0.5,"signed_gap":-0.20041677355766296,"absolute_gap":0.20041677355766296,"lower":0.2,"upper":0.3,"upper_inclusive":false},{"index":3,"count":17,"confidence":0.3573905702899484,"empirical_correctness":0.17647058823529413,"signed_gap":0.1809199820546543,"absolute_gap":0.1809199820546543,"lower":0.3,"upper":0.4,"upper_inclusive":false},{"index":4,"count":17,"confidence":0.4421829300768235,"empirical_correctness":0.35294117647058826,"signed_gap":0.08924175360623526,"absolute_gap":0.08924175360623526,"lower":0.4,"upper":0.5,"upper_inclusive":false},{"index":5,"count":10,"confidence":0.5503379106521606,"empirical_correctness":0.5,"signed_gap":0.050337910652160645,"absolute_gap":0.050337910652160645,"lower":0.5,"upper":0.6,"upper_inclusive":false},{"index":6,"count":8,"confidence":0.6666587442159653,"empirical_correctness":0.75,"signed_gap":-0.08334125578403473,"absolute_gap":0.08334125578403473,"lower":0.6,"upper":0.7,"upper_inclusive":false},{"index":7,"count":2,"confidence":0.7644288539886475,"empirical_correctness":1.0,"signed_gap":-0.23557114601135254,"absolute_gap":0.23557114601135254,"lower":0.7,"upper":0.8,"upper_inclusive":false},{"index":8,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.8,"upper":0.9,"upper_inclusive":false},{"index":9,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.9,"upper":1.0,"upper_inclusive":true}]},"equal_mass":{"n":56,"bin_count":10,"ece":0.15145278349518776,"signed_mean_gap":0.0635254058454718,"absolute_mean_gap":0.0635254058454718,"bins":[{"index":0,"count":5,"confidence":0.31171905994415283,"empirical_correctness":0.2,"signed_gap":0.11171905994415282,"absolute_gap":0.11171905994415282,"min_confidence":0.29936903715133667,"max_confidence":0.32089826464653015},{"index":1,"count":6,"confidence":0.34115639328956604,"empirical_correctness":0.0,"signed_gap":0.34115639328956604,"absolute_gap":0.34115639328956604,"min_confidence":0.3284788131713867,"max_confidence":0.36275091767311096},{"index":2,"count":5,"confidence":0.3761567771434784,"empirical_correctness":0.4,"signed_gap":-0.02384322285652163,"absolute_gap":0.02384322285652163,"min_confidence":0.36295169591903687,"max_confidence":0.3900429606437683},{"index":3,"count":6,"confidence":0.40476534763971966,"empirical_correctness":0.3333333333333333,"signed_gap":0.07143201430638635,"absolute_gap":0.07143201430638635,"min_confidence":0.39278972148895264,"max_confidence":0.41781482100486755},{"index":4,"count":6,"confidence":0.4308553487062454,"empirical_correctness":0.5,"signed_gap":-0.06914465129375458,"absolute_gap":0.06914465129375458,"min_confidence":0.41926541924476624,"max_confidence":0.44462352991104126},{"index":5,"count":5,"confidence":0.45118626952171326,"empirical_correctness":0.4,"signed_gap":0.051186269521713235,"absolute_gap":0.051186269521713235,"min_confidence":0.4460304379463196,"max_confidence":0.4604714810848236},{"index":6,"count":6,"confidence":0.49778586626052856,"empirical_correctness":0.16666666666666666,"signed_gap":0.33111919959386193,"absolute_gap":0.33111919959386193,"min_confidence":0.4653499722480774,"max_confidence":0.5198143720626831},{"index":7,"count":5,"confidence":0.5579952239990235,"empirical_correctness":0.6,"signed_gap":-0.04200477600097652,"absolute_gap":0.04200477600097652,"min_confidence":0.5477362275123596,"max_confidence":0.5678120255470276},{"index":8,"count":6,"confidence":0.6237695018450419,"empirical_correctness":0.5,"signed_gap":0.12376950184504187,"absolute_gap":0.12376950184504187,"min_confidence":0.5704522132873535,"max_confidence":0.6809813380241394},{"index":9,"count":6,"confidence":0.7136902213096619,"empirical_correctness":1.0,"signed_gap":-0.28630977869033813,"absolute_gap":0.28630977869033813,"min_confidence":0.6852757930755615,"max_confidence":0.7909113168716431}]}},"calibration_singleton_labels":{"equal_width":{"n":56,"bin_count":10,"ece":0.11847918746726853,"signed_mean_gap":0.0635254058454718,"absolute_mean_gap":0.0635254058454718,"bins":[{"index":0,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.0,"upper":0.1,"upper_inclusive":false},{"index":1,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.1,"upper":0.2,"upper_inclusive":false},{"index":2,"count":2,"confidence":0.29958322644233704,"empirical_correctness":0.5,"signed_gap":-0.20041677355766296,"absolute_gap":0.20041677355766296,"lower":0.2,"upper":0.3,"upper_inclusive":false},{"index":3,"count":17,"confidence":0.3573905702899484,"empirical_correctness":0.17647058823529413,"signed_gap":0.1809199820546543,"absolute_gap":0.1809199820546543,"lower":0.3,"upper":0.4,"upper_inclusive":false},{"index":4,"count":17,"confidence":0.4421829300768235,"empirical_correctness":0.35294117647058826,"signed_gap":0.08924175360623526,"absolute_gap":0.08924175360623526,"lower":0.4,"upper":0.5,"upper_inclusive":false},{"index":5,"count":10,"confidence":0.5503379106521606,"empirical_correctness":0.5,"signed_gap":0.050337910652160645,"absolute_gap":0.050337910652160645,"lower":0.5,"upper":0.6,"upper_inclusive":false},{"index":6,"count":8,"confidence":0.6666587442159653,"empirical_correctness":0.75,"signed_gap":-0.08334125578403473,"absolute_gap":0.08334125578403473,"lower":0.6,"upper":0.7,"upper_inclusive":false},{"index":7,"count":2,"confidence":0.7644288539886475,"empirical_correctness":1.0,"signed_gap":-0.23557114601135254,"absolute_gap":0.23557114601135254,"lower":0.7,"upper":0.8,"upper_inclusive":false},{"index":8,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.8,"upper":0.9,"upper_inclusive":false},{"index":9,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.9,"upper":1.0,"upper_inclusive":true}]},"equal_mass":{"n":56,"bin_count":10,"ece":0.15145278349518776,"signed_mean_gap":0.0635254058454718,"absolute_mean_gap":0.0635254058454718,"bins":[{"index":0,"count":5,"confidence":0.31171905994415283,"empirical_correctness":0.2,"signed_gap":0.11171905994415282,"absolute_gap":0.11171905994415282,"min_confidence":0.29936903715133667,"max_confidence":0.32089826464653015},{"index":1,"count":6,"confidence":0.34115639328956604,"empirical_correctness":0.0,"signed_gap":0.34115639328956604,"absolute_gap":0.34115639328956604,"min_confidence":0.3284788131713867,"max_confidence":0.36275091767311096},{"index":2,"count":5,"confidence":0.3761567771434784,"empirical_correctness":0.4,"signed_gap":-0.02384322285652163,"absolute_gap":0.02384322285652163,"min_confidence":0.36295169591903687,"max_confidence":0.3900429606437683},{"index":3,"count":6,"confidence":0.40476534763971966,"empirical_correctness":0.3333333333333333,"signed_gap":0.07143201430638635,"absolute_gap":0.07143201430638635,"min_confidence":0.39278972148895264,"max_confidence":0.41781482100486755},{"index":4,"count":6,"confidence":0.4308553487062454,"empirical_correctness":0.5,"signed_gap":-0.06914465129375458,"absolute_gap":0.06914465129375458,"min_confidence":0.41926541924476624,"max_confidence":0.44462352991104126},{"index":5,"count":5,"confidence":0.45118626952171326,"empirical_correctness":0.4,"signed_gap":0.051186269521713235,"absolute_gap":0.051186269521713235,"min_confidence":0.4460304379463196,"max_confidence":0.4604714810848236},{"index":6,"count":6,"confidence":0.49778586626052856,"empirical_correctness":0.16666666666666666,"signed_gap":0.33111919959386193,"absolute_gap":0.33111919959386193,"min_confidence":0.4653499722480774,"max_confidence":0.5198143720626831},{"index":7,"count":5,"confidence":0.5579952239990235,"empirical_correctness":0.6,"signed_gap":-0.04200477600097652,"absolute_gap":0.04200477600097652,"min_confidence":0.5477362275123596,"max_confidence":0.5678120255470276},{"index":8,"count":6,"confidence":0.6237695018450419,"empirical_correctness":0.5,"signed_gap":0.12376950184504187,"absolute_gap":0.12376950184504187,"min_confidence":0.5704522132873535,"max_confidence":0.6809813380241394},{"index":9,"count":6,"confidence":0.7136902213096619,"empirical_correctness":1.0,"signed_gap":-0.28630977869033813,"absolute_gap":0.28630977869033813,"min_confidence":0.6852757930755615,"max_confidence":0.7909113168716431}]}},"latency_ms":{"n":56,"median":15.715360001195222,"p95":16.04956350638531},"per_action":{"keep":{"n":8,"raw":{"n":8,"acceptable":8,"unsafe":0,"review":0,"automated_count":8,"automated_acceptable":8,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":1.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.6755924351161198,"upper":1.0,"confidence_level":0.95}},"effective":{"n":8,"acceptable":5,"unsafe":0,"review":3,"automated_count":5,"automated_acceptable":5,"automated_unsafe":0,"unnecessary_reviews":3,"accuracy":0.625,"unsafe_rate":0.0,"review_rate":0.375,"automated_coverage":0.625,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.375,"accuracy_wilson_95":{"lower":0.30574239460262737,"upper":0.8631557141764026,"confidence_level":0.95}}},"drop":{"n":22,"raw":{"n":22,"acceptable":8,"unsafe":14,"review":0,"automated_count":22,"automated_acceptable":8,"automated_unsafe":14,"unnecessary_reviews":0,"accuracy":0.36363636363636365,"unsafe_rate":0.6363636363636364,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.36363636363636365,"conditional_unsafe":0.6363636363636364,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.19733204670366125,"upper":0.5704829190986899,"confidence_level":0.95}},"effective":{"n":22,"acceptable":3,"unsafe":0,"review":19,"automated_count":3,"automated_acceptable":3,"automated_unsafe":0,"unnecessary_reviews":19,"accuracy":0.13636363636363635,"unsafe_rate":0.0,"review_rate":0.8636363636363636,"automated_coverage":0.13636363636363635,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.8636363636363636,"accuracy_wilson_95":{"lower":0.04749003714055253,"upper":0.33334987166571717,"confidence_level":0.95}}},"mask":{"n":8,"raw":{"n":8,"acceptable":7,"unsafe":0,"review":0,"automated_count":8,"automated_acceptable":7,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":0.875,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.875,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.5291118177871464,"upper":0.9775825085499433,"confidence_level":0.95}},"effective":{"n":8,"acceptable":4,"unsafe":0,"review":4,"automated_count":4,"automated_acceptable":4,"automated_unsafe":0,"unnecessary_reviews":4,"accuracy":0.5,"unsafe_rate":0.0,"review_rate":0.5,"automated_coverage":0.5,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.5,"accuracy_wilson_95":{"lower":0.21521606221387757,"upper":0.7847839377861224,"confidence_level":0.95}}},"pseudonymize":{"n":8,"raw":{"n":8,"acceptable":0,"unsafe":8,"review":0,"automated_count":8,"automated_acceptable":0,"automated_unsafe":8,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":1.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":1.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.32440756488388023,"confidence_level":0.95}},"effective":{"n":8,"acceptable":0,"unsafe":0,"review":8,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":8,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.32440756488388023,"confidence_level":0.95}}},"generalize":{"n":8,"raw":{"n":8,"acceptable":0,"unsafe":8,"review":0,"automated_count":8,"automated_acceptable":0,"automated_unsafe":8,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":1.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":1.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.32440756488388023,"confidence_level":0.95}},"effective":{"n":8,"acceptable":0,"unsafe":0,"review":8,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":8,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.32440756488388023,"confidence_level":0.95}}},"review":{"n":2,"raw":{"n":2,"acceptable":0,"unsafe":2,"review":0,"automated_count":2,"automated_acceptable":0,"automated_unsafe":2,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":1.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":1.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.6576197724933469,"confidence_level":0.95}},"effective":{"n":2,"acceptable":2,"unsafe":0,"review":2,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":1.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.34238022750665303,"upper":1.0,"confidence_level":0.95}}}}},"challenge56-compact_no_samples":{"n":56,"raw":{"n":56,"acceptable":8,"unsafe":48,"review":0,"automated_count":56,"automated_acceptable":8,"automated_unsafe":48,"unnecessary_reviews":0,"accuracy":0.14285714285714285,"unsafe_rate":0.8571428571428571,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.14285714285714285,"conditional_unsafe":0.8571428571428571,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.07420788536920624,"upper":0.25735921228705333,"confidence_level":0.95}},"effective":{"n":56,"acceptable":7,"unsafe":0,"review":51,"automated_count":5,"automated_acceptable":5,"automated_unsafe":0,"unnecessary_reviews":49,"accuracy":0.125,"unsafe_rate":0.0,"review_rate":0.9107142857142857,"automated_coverage":0.08928571428571429,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.875,"accuracy_wilson_95":{"lower":0.06189068695717122,"upper":0.2362547655819014,"confidence_level":0.95}},"confusion":{"raw":{"keep":{"keep":8,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0},"drop":{"keep":22,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0},"mask":{"keep":8,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0},"pseudonymize":{"keep":8,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0},"generalize":{"keep":8,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0},"review":{"keep":2,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0}},"effective":{"keep":{"keep":5,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":3},"drop":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":22},"mask":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":8},"pseudonymize":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":8},"generalize":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":8},"review":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":2}}},"probability_metrics":{"singleton_n":56,"multiacceptable_n":0,"multiacceptable_coverage":0.0,"nll":1.9085383180410453,"nll_zero_mass_n":0,"brier":1.0047988822908496,"acceptable_set_n":56,"acceptable_set_nll":1.9085383180410453,"acceptable_set_zero_mass_n":0},"calibration_all_labels":{"equal_width":{"n":56,"bin_count":10,"ece":0.4753693187875407,"signed_mean_gap":0.4753693187875407,"absolute_mean_gap":0.4753693187875407,"bins":[{"index":0,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.0,"upper":0.1,"upper_inclusive":false},{"index":1,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.1,"upper":0.2,"upper_inclusive":false},{"index":2,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.2,"upper":0.3,"upper_inclusive":false},{"index":3,"count":3,"confidence":0.38283084829648334,"empirical_correctness":0.0,"signed_gap":0.38283084829648334,"absolute_gap":0.38283084829648334,"lower":0.3,"upper":0.4,"upper_inclusive":false},{"index":4,"count":8,"confidence":0.4580278620123863,"empirical_correctness":0.0,"signed_gap":0.4580278620123863,"absolute_gap":0.4580278620123863,"lower":0.4,"upper":0.5,"upper_inclusive":false},{"index":5,"count":12,"confidence":0.5515460620323817,"empirical_correctness":0.0,"signed_gap":0.5515460620323817,"absolute_gap":0.5515460620323817,"lower":0.5,"upper":0.6,"upper_inclusive":false},{"index":6,"count":14,"confidence":0.6576119916779655,"empirical_correctness":0.07142857142857142,"signed_gap":0.5861834202493941,"absolute_gap":0.5861834202493941,"lower":0.6,"upper":0.7,"upper_inclusive":false},{"index":7,"count":19,"confidence":0.7359392517491391,"empirical_correctness":0.3684210526315789,"signed_gap":0.3675181991175602,"absolute_gap":0.3675181991175602,"lower":0.7,"upper":0.8,"upper_inclusive":false},{"index":8,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.8,"upper":0.9,"upper_inclusive":false},{"index":9,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.9,"upper":1.0,"upper_inclusive":true}]},"equal_mass":{"n":56,"bin_count":10,"ece":0.4753693187875407,"signed_mean_gap":0.4753693187875407,"absolute_mean_gap":0.4753693187875407,"bins":[{"index":0,"count":5,"confidence":0.3987440526485443,"empirical_correctness":0.0,"signed_gap":0.3987440526485443,"absolute_gap":0.3987440526485443,"min_confidence":0.3764071464538574,"max_confidence":0.4356235861778259},{"index":1,"count":6,"confidence":0.46983252962430316,"empirical_correctness":0.0,"signed_gap":0.46983252962430316,"absolute_gap":0.46983252962430316,"min_confidence":0.4395979344844818,"max_confidence":0.4939418137073517},{"index":2,"count":5,"confidence":0.5269372463226318,"empirical_correctness":0.0,"signed_gap":0.5269372463226318,"absolute_gap":0.5269372463226318,"min_confidence":0.5043227672576904,"max_confidence":0.5436327457427979},{"index":3,"count":6,"confidence":0.5656672418117523,"empirical_correctness":0.0,"signed_gap":0.5656672418117523,"absolute_gap":0.5656672418117523,"min_confidence":0.5454077124595642,"max_confidence":0.5743294358253479},{"index":4,"count":6,"confidence":0.6127465069293976,"empirical_correctness":0.0,"signed_gap":0.6127465069293976,"absolute_gap":0.6127465069293976,"min_confidence":0.5898630619049072,"max_confidence":0.639616072177887},{"index":5,"count":5,"confidence":0.6705966234207154,"empirical_correctness":0.2,"signed_gap":0.47059662342071534,"absolute_gap":0.47059662342071534,"min_confidence":0.6566546559333801,"max_confidence":0.6838207244873047},{"index":6,"count":6,"confidence":0.6956366896629333,"empirical_correctness":0.0,"signed_gap":0.6956366896629333,"absolute_gap":0.6956366896629333,"min_confidence":0.6841718554496765,"max_confidence":0.7066749334335327},{"index":7,"count":5,"confidence":0.7172696232795716,"empirical_correctness":0.6,"signed_gap":0.11726962327957158,"absolute_gap":0.11726962327957158,"min_confidence":0.7098459601402283,"max_confidence":0.724073052406311},{"index":8,"count":6,"confidence":0.7341059247652689,"empirical_correctness":0.16666666666666666,"signed_gap":0.5674392580986023,"absolute_gap":0.5674392580986023,"min_confidence":0.7259040474891663,"max_confidence":0.740361213684082},{"index":9,"count":6,"confidence":0.7641684611638387,"empirical_correctness":0.5,"signed_gap":0.26416846116383874,"absolute_gap":0.26416846116383874,"min_confidence":0.7525801658630371,"max_confidence":0.7766887545585632}]}},"calibration_singleton_labels":{"equal_width":{"n":56,"bin_count":10,"ece":0.4753693187875407,"signed_mean_gap":0.4753693187875407,"absolute_mean_gap":0.4753693187875407,"bins":[{"index":0,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.0,"upper":0.1,"upper_inclusive":false},{"index":1,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.1,"upper":0.2,"upper_inclusive":false},{"index":2,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.2,"upper":0.3,"upper_inclusive":false},{"index":3,"count":3,"confidence":0.38283084829648334,"empirical_correctness":0.0,"signed_gap":0.38283084829648334,"absolute_gap":0.38283084829648334,"lower":0.3,"upper":0.4,"upper_inclusive":false},{"index":4,"count":8,"confidence":0.4580278620123863,"empirical_correctness":0.0,"signed_gap":0.4580278620123863,"absolute_gap":0.4580278620123863,"lower":0.4,"upper":0.5,"upper_inclusive":false},{"index":5,"count":12,"confidence":0.5515460620323817,"empirical_correctness":0.0,"signed_gap":0.5515460620323817,"absolute_gap":0.5515460620323817,"lower":0.5,"upper":0.6,"upper_inclusive":false},{"index":6,"count":14,"confidence":0.6576119916779655,"empirical_correctness":0.07142857142857142,"signed_gap":0.5861834202493941,"absolute_gap":0.5861834202493941,"lower":0.6,"upper":0.7,"upper_inclusive":false},{"index":7,"count":19,"confidence":0.7359392517491391,"empirical_correctness":0.3684210526315789,"signed_gap":0.3675181991175602,"absolute_gap":0.3675181991175602,"lower":0.7,"upper":0.8,"upper_inclusive":false},{"index":8,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.8,"upper":0.9,"upper_inclusive":false},{"index":9,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.9,"upper":1.0,"upper_inclusive":true}]},"equal_mass":{"n":56,"bin_count":10,"ece":0.4753693187875407,"signed_mean_gap":0.4753693187875407,"absolute_mean_gap":0.4753693187875407,"bins":[{"index":0,"count":5,"confidence":0.3987440526485443,"empirical_correctness":0.0,"signed_gap":0.3987440526485443,"absolute_gap":0.3987440526485443,"min_confidence":0.3764071464538574,"max_confidence":0.4356235861778259},{"index":1,"count":6,"confidence":0.46983252962430316,"empirical_correctness":0.0,"signed_gap":0.46983252962430316,"absolute_gap":0.46983252962430316,"min_confidence":0.4395979344844818,"max_confidence":0.4939418137073517},{"index":2,"count":5,"confidence":0.5269372463226318,"empirical_correctness":0.0,"signed_gap":0.5269372463226318,"absolute_gap":0.5269372463226318,"min_confidence":0.5043227672576904,"max_confidence":0.5436327457427979},{"index":3,"count":6,"confidence":0.5656672418117523,"empirical_correctness":0.0,"signed_gap":0.5656672418117523,"absolute_gap":0.5656672418117523,"min_confidence":0.5454077124595642,"max_confidence":0.5743294358253479},{"index":4,"count":6,"confidence":0.6127465069293976,"empirical_correctness":0.0,"signed_gap":0.6127465069293976,"absolute_gap":0.6127465069293976,"min_confidence":0.5898630619049072,"max_confidence":0.639616072177887},{"index":5,"count":5,"confidence":0.6705966234207154,"empirical_correctness":0.2,"signed_gap":0.47059662342071534,"absolute_gap":0.47059662342071534,"min_confidence":0.6566546559333801,"max_confidence":0.6838207244873047},{"index":6,"count":6,"confidence":0.6956366896629333,"empirical_correctness":0.0,"signed_gap":0.6956366896629333,"absolute_gap":0.6956366896629333,"min_confidence":0.6841718554496765,"max_confidence":0.7066749334335327},{"index":7,"count":5,"confidence":0.7172696232795716,"empirical_correctness":0.6,"signed_gap":0.11726962327957158,"absolute_gap":0.11726962327957158,"min_confidence":0.7098459601402283,"max_confidence":0.724073052406311},{"index":8,"count":6,"confidence":0.7341059247652689,"empirical_correctness":0.16666666666666666,"signed_gap":0.5674392580986023,"absolute_gap":0.5674392580986023,"min_confidence":0.7259040474891663,"max_confidence":0.740361213684082},{"index":9,"count":6,"confidence":0.7641684611638387,"empirical_correctness":0.5,"signed_gap":0.26416846116383874,"absolute_gap":0.26416846116383874,"min_confidence":0.7525801658630371,"max_confidence":0.7766887545585632}]}},"latency_ms":{"n":56,"median":18.363955488894135,"p95":18.808113483828492},"per_action":{"keep":{"n":8,"raw":{"n":8,"acceptable":8,"unsafe":0,"review":0,"automated_count":8,"automated_acceptable":8,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":1.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.6755924351161198,"upper":1.0,"confidence_level":0.95}},"effective":{"n":8,"acceptable":5,"unsafe":0,"review":3,"automated_count":5,"automated_acceptable":5,"automated_unsafe":0,"unnecessary_reviews":3,"accuracy":0.625,"unsafe_rate":0.0,"review_rate":0.375,"automated_coverage":0.625,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.375,"accuracy_wilson_95":{"lower":0.30574239460262737,"upper":0.8631557141764026,"confidence_level":0.95}}},"drop":{"n":22,"raw":{"n":22,"acceptable":0,"unsafe":22,"review":0,"automated_count":22,"automated_acceptable":0,"automated_unsafe":22,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":1.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":1.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.14865487460862087,"confidence_level":0.95}},"effective":{"n":22,"acceptable":0,"unsafe":0,"review":22,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":22,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.14865487460862087,"confidence_level":0.95}}},"mask":{"n":8,"raw":{"n":8,"acceptable":0,"unsafe":8,"review":0,"automated_count":8,"automated_acceptable":0,"automated_unsafe":8,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":1.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":1.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.32440756488388023,"confidence_level":0.95}},"effective":{"n":8,"acceptable":0,"unsafe":0,"review":8,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":8,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.32440756488388023,"confidence_level":0.95}}},"pseudonymize":{"n":8,"raw":{"n":8,"acceptable":0,"unsafe":8,"review":0,"automated_count":8,"automated_acceptable":0,"automated_unsafe":8,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":1.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":1.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.32440756488388023,"confidence_level":0.95}},"effective":{"n":8,"acceptable":0,"unsafe":0,"review":8,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":8,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.32440756488388023,"confidence_level":0.95}}},"generalize":{"n":8,"raw":{"n":8,"acceptable":0,"unsafe":8,"review":0,"automated_count":8,"automated_acceptable":0,"automated_unsafe":8,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":1.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":1.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.32440756488388023,"confidence_level":0.95}},"effective":{"n":8,"acceptable":0,"unsafe":0,"review":8,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":8,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.32440756488388023,"confidence_level":0.95}}},"review":{"n":2,"raw":{"n":2,"acceptable":0,"unsafe":2,"review":0,"automated_count":2,"automated_acceptable":0,"automated_unsafe":2,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":1.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":1.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.6576197724933469,"confidence_level":0.95}},"effective":{"n":2,"acceptable":2,"unsafe":0,"review":2,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":1.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.34238022750665303,"upper":1.0,"confidence_level":0.95}}}}}},"composite":{"n":198,"scope":"descriptive sum across four correlated panels","raw":{"n":198,"acceptable":63,"unsafe":121,"review":0,"automated_count":198,"automated_acceptable":63,"automated_unsafe":121,"unnecessary_reviews":0,"accuracy":0.3181818181818182,"unsafe_rate":0.6111111111111112,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.3181818181818182,"conditional_unsafe":0.6111111111111112,"unnecessary_review_rate":0.0},"effective":{"n":198,"acceptable":55,"unsafe":3,"review":139,"automated_count":59,"automated_acceptable":45,"automated_unsafe":3,"unnecessary_reviews":129,"accuracy":0.2777777777777778,"unsafe_rate":0.015151515151515152,"review_rate":0.702020202020202,"automated_coverage":0.29797979797979796,"conditional_correctness":0.7627118644067796,"conditional_unsafe":0.05084745762711865,"unnecessary_review_rate":0.6515151515151515}}},{"id":"gtm-k--laya-product-decision-assist-fp32","platform":"Local","model_id":"gtm-k/laya-product-decision-assist","revision":"91294410a613e3cc7a80f80402e258df4716d472","loading":"float32 / none","scoring_kind":"Native decision readout","settings":{"dtype":"float32","quantization":"none","max_tokens":512,"head_max_tokens":192,"temperature":1.0,"autocast":false,"truncation":false,"cpu_offload":false,"guard_threshold":0.45,"seed":0,"tf32":false},"scoring_metadata":{"backend":"laya_native_local_builtin","readout":"native_mask_slot_scorer_numpy_float32_softmax","probability_source":"native_choice_unrounded_published_temperature_not_task_calibrated","latency_scope":"synchronized serial complete encoder+head+act forward,input transfer and native readout; tokenization excluded","native_disclosures":["Stock installed torch attention in place of ONNX dynamic equivalent; no custom kernels or repo imports.","FP32/no-autocast explicit precision: convert every saved F16/BF16/F32 tensor toFP32; no quantization.","Unrounded native probability; guard uses winner p, not entropy-concentration confidence.","Act head fully loaded and evaluated; not used to alter six-action answer.","Native choice:6-10 bucket else choice scalar0, T=1.0. No task fitting, priors correction, prompt change or finetuning here."],"equal_mass_convention":"rank bins with floor boundaries at floor(i*n/k), k=min(10,n//2); minimum two rows"},"load_ms":null,"peak_allocated_bytes":1748760576,"peak_reserved_bytes":1870659584,"panels":{"diagnostic43-no_samples":{"n":43,"raw":{"n":43,"acceptable":20,"unsafe":8,"review":0,"automated_count":43,"automated_acceptable":20,"automated_unsafe":8,"unnecessary_reviews":0,"accuracy":0.46511627906976744,"unsafe_rate":0.18604651162790697,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.46511627906976744,"conditional_unsafe":0.18604651162790697,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.32511057704121815,"upper":0.6108435952768602,"confidence_level":0.95}},"effective":{"n":43,"acceptable":20,"unsafe":2,"review":6,"automated_count":37,"automated_acceptable":20,"automated_unsafe":2,"unnecessary_reviews":6,"accuracy":0.46511627906976744,"unsafe_rate":0.046511627906976744,"review_rate":0.13953488372093023,"automated_coverage":0.8604651162790697,"conditional_correctness":0.5405405405405406,"conditional_unsafe":0.05405405405405406,"unnecessary_review_rate":0.13953488372093023,"accuracy_wilson_95":{"lower":0.32511057704121815,"upper":0.6108435952768602,"confidence_level":0.95}},"confusion":{"raw":{"keep":{"keep":4,"drop":8,"mask":0,"pseudonymize":0,"generalize":0,"review":0},"drop":{"keep":5,"drop":11,"mask":0,"pseudonymize":0,"generalize":0,"review":0},"mask":{"keep":0,"drop":2,"mask":0,"pseudonymize":0,"generalize":0,"review":0},"pseudonymize":{"keep":1,"drop":1,"mask":0,"pseudonymize":0,"generalize":0,"review":0},"generalize":{"keep":2,"drop":4,"mask":0,"pseudonymize":0,"generalize":0,"review":0},"review":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0}},"effective":{"keep":{"keep":4,"drop":8,"mask":0,"pseudonymize":0,"generalize":0,"review":0},"drop":{"keep":2,"drop":11,"mask":0,"pseudonymize":0,"generalize":0,"review":3},"mask":{"keep":0,"drop":2,"mask":0,"pseudonymize":0,"generalize":0,"review":0},"pseudonymize":{"keep":0,"drop":1,"mask":0,"pseudonymize":0,"generalize":0,"review":1},"generalize":{"keep":0,"drop":4,"mask":0,"pseudonymize":0,"generalize":0,"review":2},"review":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0}}},"probability_metrics":{"singleton_n":38,"multiacceptable_n":5,"multiacceptable_coverage":0.11627906976744186,"nll":2.030114175281338,"nll_zero_mass_n":0,"brier":0.9209852222388728,"acceptable_set_n":43,"acceptable_set_nll":1.803907640875976,"acceptable_set_zero_mass_n":0},"calibration_all_labels":{"equal_width":{"n":43,"bin_count":10,"ece":0.3397161620993947,"signed_mean_gap":0.3397161620993947,"absolute_mean_gap":0.3397161620993947,"bins":[{"index":0,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.0,"upper":0.1,"upper_inclusive":false},{"index":1,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.1,"upper":0.2,"upper_inclusive":false},{"index":2,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.2,"upper":0.3,"upper_inclusive":false},{"index":3,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.3,"upper":0.4,"upper_inclusive":false},{"index":4,"count":4,"confidence":0.47743383795022964,"empirical_correctness":0.25,"signed_gap":0.22743383795022964,"absolute_gap":0.22743383795022964,"lower":0.4,"upper":0.5,"upper_inclusive":false},{"index":5,"count":3,"confidence":0.5479162931442261,"empirical_correctness":0.0,"signed_gap":0.5479162931442261,"absolute_gap":0.5479162931442261,"lower":0.5,"upper":0.6,"upper_inclusive":false},{"index":6,"count":5,"confidence":0.6487360954284668,"empirical_correctness":0.6,"signed_gap":0.0487360954284668,"absolute_gap":0.0487360954284668,"lower":0.6,"upper":0.7,"upper_inclusive":false},{"index":7,"count":3,"confidence":0.7775517503420512,"empirical_correctness":0.6666666666666666,"signed_gap":0.11088508367538452,"absolute_gap":0.11088508367538452,"lower":0.7,"upper":0.8,"upper_inclusive":false},{"index":8,"count":11,"confidence":0.8359113389795477,"empirical_correctness":0.2727272727272727,"signed_gap":0.563184066252275,"absolute_gap":0.563184066252275,"lower":0.8,"upper":0.9,"upper_inclusive":false},{"index":9,"count":17,"confidence":0.9578206048292273,"empirical_correctness":0.6470588235294118,"signed_gap":0.31076178129981546,"absolute_gap":0.31076178129981546,"lower":0.9,"upper":1.0,"upper_inclusive":true}]},"equal_mass":{"n":43,"bin_count":10,"ece":0.3397161620993947,"signed_mean_gap":0.3397161620993947,"absolute_mean_gap":0.3397161620993947,"bins":[{"index":0,"count":4,"confidence":0.47743383795022964,"empirical_correctness":0.25,"signed_gap":0.22743383795022964,"absolute_gap":0.22743383795022964,"min_confidence":0.4574059844017029,"max_confidence":0.4890917241573334},{"index":1,"count":4,"confidence":0.5636959075927734,"empirical_correctness":0.25,"signed_gap":0.31369590759277344,"absolute_gap":0.31369590759277344,"min_confidence":0.5196629166603088,"max_confidence":0.6110347509384155},{"index":2,"count":4,"confidence":0.6581614315509796,"empirical_correctness":0.5,"signed_gap":0.15816143155097961,"absolute_gap":0.15816143155097961,"min_confidence":0.6117286682128906,"max_confidence":0.6906870603561401},{"index":3,"count":5,"confidence":0.7885961532592773,"empirical_correctness":0.4,"signed_gap":0.3885961532592773,"absolute_gap":0.3885961532592773,"min_confidence":0.7453746199607849,"max_confidence":0.8102662563323975},{"index":4,"count":4,"confidence":0.8191987425088882,"empirical_correctness":0.5,"signed_gap":0.31919874250888824,"absolute_gap":0.31919874250888824,"min_confidence":0.8113222718238831,"max_confidence":0.8276962637901306},{"index":5,"count":4,"confidence":0.8539922535419464,"empirical_correctness":0.25,"signed_gap":0.6039922535419464,"absolute_gap":0.6039922535419464,"min_confidence":0.8313303589820862,"max_confidence":0.8804641366004944},{"index":6,"count":5,"confidence":0.9095805764198304,"empirical_correctness":0.4,"signed_gap":0.5095805764198303,"absolute_gap":0.5095805764198303,"min_confidence":0.8919352293014526,"max_confidence":0.9274294972419739},{"index":7,"count":4,"confidence":0.9552016258239746,"empirical_correctness":0.5,"signed_gap":0.4552016258239746,"absolute_gap":0.4552016258239746,"min_confidence":0.9487600326538086,"max_confidence":0.961968183517456},{"index":8,"count":4,"confidence":0.9736358374357224,"empirical_correctness":0.75,"signed_gap":0.22363583743572235,"absolute_gap":0.22363583743572235,"min_confidence":0.9650530815124512,"max_confidence":0.9773779511451721},{"index":9,"count":5,"confidence":0.9823265552520752,"empirical_correctness":0.8,"signed_gap":0.1823265552520752,"absolute_gap":0.1823265552520752,"min_confidence":0.9786922335624695,"max_confidence":0.9883406162261963}]}},"calibration_singleton_labels":{"equal_width":{"n":38,"bin_count":10,"ece":0.3979346838436629,"signed_mean_gap":0.3979346838436629,"absolute_mean_gap":0.3979346838436629,"bins":[{"index":0,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.0,"upper":0.1,"upper_inclusive":false},{"index":1,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.1,"upper":0.2,"upper_inclusive":false},{"index":2,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.2,"upper":0.3,"upper_inclusive":false},{"index":3,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.3,"upper":0.4,"upper_inclusive":false},{"index":4,"count":4,"confidence":0.47743383795022964,"empirical_correctness":0.25,"signed_gap":0.22743383795022964,"absolute_gap":0.22743383795022964,"lower":0.4,"upper":0.5,"upper_inclusive":false},{"index":5,"count":3,"confidence":0.5479162931442261,"empirical_correctness":0.0,"signed_gap":0.5479162931442261,"absolute_gap":0.5479162931442261,"lower":0.5,"upper":0.6,"upper_inclusive":false},{"index":6,"count":5,"confidence":0.6487360954284668,"empirical_correctness":0.6,"signed_gap":0.0487360954284668,"absolute_gap":0.0487360954284668,"lower":0.6,"upper":0.7,"upper_inclusive":false},{"index":7,"count":2,"confidence":0.7667132318019867,"empirical_correctness":0.5,"signed_gap":0.2667132318019867,"absolute_gap":0.2667132318019867,"lower":0.7,"upper":0.8,"upper_inclusive":false},{"index":8,"count":10,"confidence":0.8363694369792938,"empirical_correctness":0.2,"signed_gap":0.6363694369792938,"absolute_gap":0.6363694369792938,"lower":0.8,"upper":0.9,"upper_inclusive":false},{"index":9,"count":14,"confidence":0.959088031734739,"empirical_correctness":0.5714285714285714,"signed_gap":0.3876594603061676,"absolute_gap":0.3876594603061676,"lower":0.9,"upper":1.0,"upper_inclusive":true}]},"equal_mass":{"n":38,"bin_count":10,"ece":0.3979346838436629,"signed_mean_gap":0.3979346838436629,"absolute_mean_gap":0.3979346838436629,"bins":[{"index":0,"count":3,"confidence":0.47354787588119507,"empirical_correctness":0.3333333333333333,"signed_gap":0.14021454254786175,"absolute_gap":0.14021454254786175,"min_confidence":0.4574059844017029,"max_confidence":0.4839938282966614},{"index":1,"count":4,"confidence":0.5332101508975029,"empirical_correctness":0.0,"signed_gap":0.5332101508975029,"absolute_gap":0.5332101508975029,"min_confidence":0.4890917241573334,"max_confidence":0.5672066807746887},{"index":2,"count":4,"confidence":0.6382483541965485,"empirical_correctness":0.5,"signed_gap":0.13824835419654846,"absolute_gap":0.13824835419654846,"min_confidence":0.6110347509384155,"max_confidence":0.6676661968231201},{"index":3,"count":4,"confidence":0.7560431957244873,"empirical_correctness":0.5,"signed_gap":0.2560431957244873,"absolute_gap":0.2560431957244873,"min_confidence":0.6906870603561401,"max_confidence":0.8000592589378357},{"index":4,"count":4,"confidence":0.814841240644455,"empirical_correctness":0.25,"signed_gap":0.564841240644455,"absolute_gap":0.564841240644455,"min_confidence":0.8102662563323975,"max_confidence":0.8206716179847717},{"index":5,"count":3,"confidence":0.8439569274584452,"empirical_correctness":0.3333333333333333,"signed_gap":0.5106235941251118,"absolute_gap":0.5106235941251118,"min_confidence":0.8276962637901306,"max_confidence":0.8569079637527466},{"index":6,"count":4,"confidence":0.8972412496805191,"empirical_correctness":0.0,"signed_gap":0.8972412496805191,"absolute_gap":0.8972412496805191,"min_confidence":0.8804641366004944,"max_confidence":0.9105536937713623},{"index":7,"count":4,"confidence":0.9465669542551041,"empirical_correctness":0.75,"signed_gap":0.19656695425510406,"absolute_gap":0.19656695425510406,"min_confidence":0.9274294972419739,"max_confidence":0.9586265087127686},{"index":8,"count":4,"confidence":0.9728646129369736,"empirical_correctness":0.5,"signed_gap":0.47286461293697357,"absolute_gap":0.47286461293697357,"min_confidence":0.961968183517456,"max_confidence":0.9773779511451721},{"index":9,"count":4,"confidence":0.9832351356744766,"empirical_correctness":0.75,"signed_gap":0.23323513567447662,"absolute_gap":0.23323513567447662,"min_confidence":0.9797230362892151,"max_confidence":0.9883406162261963}]}},"latency_ms":{"n":43,"median":68.74549994245172,"p95":70.05746008362621},"per_action":{"keep":{"n":12,"raw":{"n":12,"acceptable":4,"unsafe":0,"review":0,"automated_count":12,"automated_acceptable":4,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":0.3333333333333333,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.3333333333333333,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.13812009109121315,"upper":0.6093779111272004,"confidence_level":0.95}},"effective":{"n":12,"acceptable":4,"unsafe":0,"review":0,"automated_count":12,"automated_acceptable":4,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":0.3333333333333333,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.3333333333333333,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.13812009109121315,"upper":0.6093779111272004,"confidence_level":0.95}}},"drop":{"n":16,"raw":{"n":16,"acceptable":11,"unsafe":5,"review":0,"automated_count":16,"automated_acceptable":11,"automated_unsafe":5,"unnecessary_reviews":0,"accuracy":0.6875,"unsafe_rate":0.3125,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.6875,"conditional_unsafe":0.3125,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.44404355834740666,"upper":0.8583535614521796,"confidence_level":0.95}},"effective":{"n":16,"acceptable":11,"unsafe":2,"review":3,"automated_count":13,"automated_acceptable":11,"automated_unsafe":2,"unnecessary_reviews":3,"accuracy":0.6875,"unsafe_rate":0.125,"review_rate":0.1875,"automated_coverage":0.8125,"conditional_correctness":0.8461538461538461,"conditional_unsafe":0.15384615384615385,"unnecessary_review_rate":0.1875,"accuracy_wilson_95":{"lower":0.44404355834740666,"upper":0.8583535614521796,"confidence_level":0.95}}},"mask":{"n":2,"raw":{"n":2,"acceptable":0,"unsafe":0,"review":0,"automated_count":2,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.6576197724933469,"confidence_level":0.95}},"effective":{"n":2,"acceptable":0,"unsafe":0,"review":0,"automated_count":2,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.6576197724933469,"confidence_level":0.95}}},"pseudonymize":{"n":2,"raw":{"n":2,"acceptable":0,"unsafe":1,"review":0,"automated_count":2,"automated_acceptable":0,"automated_unsafe":1,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":0.5,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":0.5,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.6576197724933469,"confidence_level":0.95}},"effective":{"n":2,"acceptable":0,"unsafe":0,"review":1,"automated_count":1,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":1,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":0.5,"automated_coverage":0.5,"conditional_correctness":0.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.5,"accuracy_wilson_95":{"lower":0.0,"upper":0.6576197724933469,"confidence_level":0.95}}},"generalize":{"n":6,"raw":{"n":6,"acceptable":0,"unsafe":2,"review":0,"automated_count":6,"automated_acceptable":0,"automated_unsafe":2,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":0.3333333333333333,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":0.3333333333333333,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":2.7755575615628914e-17,"upper":0.3903342879021653,"confidence_level":0.95}},"effective":{"n":6,"acceptable":0,"unsafe":0,"review":2,"automated_count":4,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":2,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":0.3333333333333333,"automated_coverage":0.6666666666666666,"conditional_correctness":0.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.3333333333333333,"accuracy_wilson_95":{"lower":2.7755575615628914e-17,"upper":0.3903342879021653,"confidence_level":0.95}}},"review":{"n":0,"raw":{"n":0,"acceptable":0,"unsafe":0,"review":0,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":null,"unsafe_rate":null,"review_rate":null,"automated_coverage":null,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":null},"effective":{"n":0,"acceptable":0,"unsafe":0,"review":0,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":null,"unsafe_rate":null,"review_rate":null,"automated_coverage":null,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":null}}}},"diagnostic43-compact_no_samples":{"n":43,"raw":{"n":43,"acceptable":15,"unsafe":28,"review":0,"automated_count":43,"automated_acceptable":15,"automated_unsafe":28,"unnecessary_reviews":0,"accuracy":0.3488372093023256,"unsafe_rate":0.6511627906976745,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.3488372093023256,"conditional_unsafe":0.6511627906976745,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.22418589997352706,"upper":0.49828218007147956,"confidence_level":0.95}},"effective":{"n":43,"acceptable":20,"unsafe":3,"review":23,"automated_count":20,"automated_acceptable":15,"automated_unsafe":3,"unnecessary_reviews":18,"accuracy":0.46511627906976744,"unsafe_rate":0.06976744186046512,"review_rate":0.5348837209302325,"automated_coverage":0.46511627906976744,"conditional_correctness":0.75,"conditional_unsafe":0.15,"unnecessary_review_rate":0.4186046511627907,"accuracy_wilson_95":{"lower":0.32511057704121815,"upper":0.6108435952768602,"confidence_level":0.95}},"confusion":{"raw":{"keep":{"keep":12,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0},"drop":{"keep":13,"drop":3,"mask":0,"pseudonymize":0,"generalize":0,"review":0},"mask":{"keep":2,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0},"pseudonymize":{"keep":2,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0},"generalize":{"keep":6,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0},"review":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0}},"effective":{"keep":{"keep":11,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":1},"drop":{"keep":2,"drop":3,"mask":2,"pseudonymize":0,"generalize":0,"review":9},"mask":{"keep":0,"drop":0,"mask":1,"pseudonymize":0,"generalize":0,"review":1},"pseudonymize":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":2},"generalize":{"keep":1,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":5},"review":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0}}},"probability_metrics":{"singleton_n":38,"multiacceptable_n":5,"multiacceptable_coverage":0.11627906976744186,"nll":2.345883679810035,"nll_zero_mass_n":0,"brier":1.0722183431977972,"acceptable_set_n":43,"acceptable_set_nll":2.420977267630175,"acceptable_set_zero_mass_n":0},"calibration_all_labels":{"equal_width":{"n":43,"bin_count":10,"ece":0.580495120481003,"signed_mean_gap":0.5378656484359918,"absolute_mean_gap":0.5378656484359918,"bins":[{"index":0,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.0,"upper":0.1,"upper_inclusive":false},{"index":1,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.1,"upper":0.2,"upper_inclusive":false},{"index":2,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.2,"upper":0.3,"upper_inclusive":false},{"index":3,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.3,"upper":0.4,"upper_inclusive":false},{"index":4,"count":1,"confidence":0.49763768911361694,"empirical_correctness":1.0,"signed_gap":-0.5023623108863831,"absolute_gap":0.5023623108863831,"lower":0.4,"upper":0.5,"upper_inclusive":false},{"index":5,"count":1,"confidence":0.5858286619186401,"empirical_correctness":1.0,"signed_gap":-0.41417133808135986,"absolute_gap":0.41417133808135986,"lower":0.5,"upper":0.6,"upper_inclusive":false},{"index":6,"count":2,"confidence":0.656978189945221,"empirical_correctness":0.0,"signed_gap":0.656978189945221,"absolute_gap":0.656978189945221,"lower":0.6,"upper":0.7,"upper_inclusive":false},{"index":7,"count":5,"confidence":0.7538739442825317,"empirical_correctness":0.2,"signed_gap":0.5538739442825318,"absolute_gap":0.5538739442825318,"lower":0.7,"upper":0.8,"upper_inclusive":false},{"index":8,"count":6,"confidence":0.8573615650335947,"empirical_correctness":0.5,"signed_gap":0.35736156503359473,"absolute_gap":0.35736156503359473,"lower":0.8,"upper":0.9,"upper_inclusive":false},{"index":9,"count":28,"confidence":0.9577593228646687,"empirical_correctness":0.32142857142857145,"signed_gap":0.6363307514360972,"absolute_gap":0.6363307514360972,"lower":0.9,"upper":1.0,"upper_inclusive":true}]},"equal_mass":{"n":43,"bin_count":10,"ece":0.5378656484359918,"signed_mean_gap":0.5378656484359918,"absolute_mean_gap":0.5378656484359918,"bins":[{"index":0,"count":4,"confidence":0.5993556827306747,"empirical_correctness":0.5,"signed_gap":0.09935568273067474,"absolute_gap":0.09935568273067474,"min_confidence":0.49763768911361694,"max_confidence":0.6966866850852966},{"index":1,"count":4,"confidence":0.7464158236980438,"empirical_correctness":0.25,"signed_gap":0.4964158236980438,"absolute_gap":0.4964158236980438,"min_confidence":0.7318809628486633,"max_confidence":0.760522723197937},{"index":2,"count":4,"confidence":0.8206820040941238,"empirical_correctness":0.25,"signed_gap":0.5706820040941238,"absolute_gap":0.5706820040941238,"min_confidence":0.7837064266204834,"max_confidence":0.8480753898620605},{"index":3,"count":5,"confidence":0.8928443670272828,"empirical_correctness":0.4,"signed_gap":0.49284436702728274,"absolute_gap":0.49284436702728274,"min_confidence":0.8754488825798035,"max_confidence":0.9111051559448242},{"index":4,"count":4,"confidence":0.9222224652767181,"empirical_correctness":0.25,"signed_gap":0.6722224652767181,"absolute_gap":0.6722224652767181,"min_confidence":0.9148352146148682,"max_confidence":0.9297754764556885},{"index":5,"count":4,"confidence":0.9431517124176025,"empirical_correctness":0.25,"signed_gap":0.6931517124176025,"absolute_gap":0.6931517124176025,"min_confidence":0.9350813031196594,"max_confidence":0.9499374628067017},{"index":6,"count":5,"confidence":0.9630810379981994,"empirical_correctness":0.0,"signed_gap":0.9630810379981994,"absolute_gap":0.9630810379981994,"min_confidence":0.9547464847564697,"max_confidence":0.9676712155342102},{"index":7,"count":4,"confidence":0.9699440896511078,"empirical_correctness":0.5,"signed_gap":0.4699440896511078,"absolute_gap":0.4699440896511078,"min_confidence":0.9687654972076416,"max_confidence":0.9706743955612183},{"index":8,"count":4,"confidence":0.9769671410322189,"empirical_correctness":0.5,"signed_gap":0.47696714103221893,"absolute_gap":0.47696714103221893,"min_confidence":0.9727622270584106,"max_confidence":0.9798434376716614},{"index":9,"count":5,"confidence":0.986728036403656,"empirical_correctness":0.6,"signed_gap":0.38672803640365605,"absolute_gap":0.38672803640365605,"min_confidence":0.9845452308654785,"max_confidence":0.9891639947891235}]}},"calibration_singleton_labels":{"equal_width":{"n":38,"bin_count":10,"ece":0.5386765709048823,"signed_mean_gap":0.49043795780131694,"absolute_mean_gap":0.49043795780131694,"bins":[{"index":0,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.0,"upper":0.1,"upper_inclusive":false},{"index":1,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.1,"upper":0.2,"upper_inclusive":false},{"index":2,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.2,"upper":0.3,"upper_inclusive":false},{"index":3,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.3,"upper":0.4,"upper_inclusive":false},{"index":4,"count":1,"confidence":0.49763768911361694,"empirical_correctness":1.0,"signed_gap":-0.5023623108863831,"absolute_gap":0.5023623108863831,"lower":0.4,"upper":0.5,"upper_inclusive":false},{"index":5,"count":1,"confidence":0.5858286619186401,"empirical_correctness":1.0,"signed_gap":-0.41417133808135986,"absolute_gap":0.41417133808135986,"lower":0.5,"upper":0.6,"upper_inclusive":false},{"index":6,"count":2,"confidence":0.656978189945221,"empirical_correctness":0.0,"signed_gap":0.656978189945221,"absolute_gap":0.656978189945221,"lower":0.6,"upper":0.7,"upper_inclusive":false},{"index":7,"count":4,"confidence":0.7464158236980438,"empirical_correctness":0.25,"signed_gap":0.4964158236980438,"absolute_gap":0.4964158236980438,"lower":0.7,"upper":0.8,"upper_inclusive":false},{"index":8,"count":5,"confidence":0.8681880354881286,"empirical_correctness":0.6,"signed_gap":0.26818803548812864,"absolute_gap":0.26818803548812864,"lower":0.8,"upper":0.9,"upper_inclusive":false},{"index":9,"count":25,"confidence":0.956504647731781,"empirical_correctness":0.36,"signed_gap":0.596504647731781,"absolute_gap":0.596504647731781,"lower":0.9,"upper":1.0,"upper_inclusive":true}]},"equal_mass":{"n":38,"bin_count":10,"ece":0.5061886922309273,"signed_mean_gap":0.49043795780131694,"absolute_mean_gap":0.49043795780131694,"bins":[{"index":0,"count":3,"confidence":0.5669120152791342,"empirical_correctness":0.6666666666666666,"signed_gap":-0.09975465138753248,"absolute_gap":0.09975465138753248,"min_confidence":0.49763768911361694,"max_confidence":0.6172696948051453},{"index":1,"count":4,"confidence":0.7304568141698837,"empirical_correctness":0.0,"signed_gap":0.7304568141698837,"absolute_gap":0.7304568141698837,"min_confidence":0.6966866850852966,"max_confidence":0.7540879249572754},{"index":2,"count":4,"confidence":0.8329409956932068,"empirical_correctness":0.5,"signed_gap":0.3329409956932068,"absolute_gap":0.3329409956932068,"min_confidence":0.760522723197937,"max_confidence":0.8754488825798035},{"index":3,"count":4,"confidence":0.8971932381391525,"empirical_correctness":0.5,"signed_gap":0.3971932381391525,"absolute_gap":0.3971932381391525,"min_confidence":0.8809852600097656,"max_confidence":0.9111051559448242},{"index":4,"count":4,"confidence":0.9222224652767181,"empirical_correctness":0.25,"signed_gap":0.6722224652767181,"absolute_gap":0.6722224652767181,"min_confidence":0.9148352146148682,"max_confidence":0.9297754764556885},{"index":5,"count":3,"confidence":0.9408897956212362,"empirical_correctness":0.3333333333333333,"signed_gap":0.6075564622879028,"absolute_gap":0.6075564622879028,"min_confidence":0.9350813031196594,"max_confidence":0.9452455639839172},{"index":6,"count":4,"confidence":0.9622206836938858,"empirical_correctness":0.0,"signed_gap":0.9622206836938858,"absolute_gap":0.9622206836938858,"min_confidence":0.9547464847564697,"max_confidence":0.9676712155342102},{"index":7,"count":4,"confidence":0.9699440896511078,"empirical_correctness":0.5,"signed_gap":0.4699440896511078,"absolute_gap":0.4699440896511078,"min_confidence":0.9687654972076416,"max_confidence":0.9706743955612183},{"index":8,"count":4,"confidence":0.9769671410322189,"empirical_correctness":0.5,"signed_gap":0.47696714103221893,"absolute_gap":0.47696714103221893,"min_confidence":0.9727622270584106,"max_confidence":0.9798434376716614},{"index":9,"count":4,"confidence":0.9863638132810593,"empirical_correctness":0.75,"signed_gap":0.23636381328105927,"absolute_gap":0.23636381328105927,"min_confidence":0.9845452308654785,"max_confidence":0.9891639947891235}]}},"latency_ms":{"n":43,"median":80.18049993552268,"p95":83.10968019068241},"per_action":{"keep":{"n":12,"raw":{"n":12,"acceptable":12,"unsafe":0,"review":0,"automated_count":12,"automated_acceptable":12,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":1.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.7575059933447592,"upper":1.0,"confidence_level":0.95}},"effective":{"n":12,"acceptable":11,"unsafe":0,"review":1,"automated_count":11,"automated_acceptable":11,"automated_unsafe":0,"unnecessary_reviews":1,"accuracy":0.9166666666666666,"unsafe_rate":0.0,"review_rate":0.08333333333333333,"automated_coverage":0.9166666666666666,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.08333333333333333,"accuracy_wilson_95":{"lower":0.646120088858883,"upper":0.9851349055950829,"confidence_level":0.95}}},"drop":{"n":16,"raw":{"n":16,"acceptable":3,"unsafe":13,"review":0,"automated_count":16,"automated_acceptable":3,"automated_unsafe":13,"unnecessary_reviews":0,"accuracy":0.1875,"unsafe_rate":0.8125,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.1875,"conditional_unsafe":0.8125,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.06591599071428142,"upper":0.4300888096197414,"confidence_level":0.95}},"effective":{"n":16,"acceptable":3,"unsafe":2,"review":9,"automated_count":7,"automated_acceptable":3,"automated_unsafe":2,"unnecessary_reviews":9,"accuracy":0.1875,"unsafe_rate":0.125,"review_rate":0.5625,"automated_coverage":0.4375,"conditional_correctness":0.42857142857142855,"conditional_unsafe":0.2857142857142857,"unnecessary_review_rate":0.5625,"accuracy_wilson_95":{"lower":0.06591599071428142,"upper":0.4300888096197414,"confidence_level":0.95}}},"mask":{"n":2,"raw":{"n":2,"acceptable":0,"unsafe":2,"review":0,"automated_count":2,"automated_acceptable":0,"automated_unsafe":2,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":1.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":1.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.6576197724933469,"confidence_level":0.95}},"effective":{"n":2,"acceptable":1,"unsafe":0,"review":1,"automated_count":1,"automated_acceptable":1,"automated_unsafe":0,"unnecessary_reviews":1,"accuracy":0.5,"unsafe_rate":0.0,"review_rate":0.5,"automated_coverage":0.5,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.5,"accuracy_wilson_95":{"lower":0.09453120573423074,"upper":0.9054687942657693,"confidence_level":0.95}}},"pseudonymize":{"n":2,"raw":{"n":2,"acceptable":0,"unsafe":2,"review":0,"automated_count":2,"automated_acceptable":0,"automated_unsafe":2,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":1.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":1.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.6576197724933469,"confidence_level":0.95}},"effective":{"n":2,"acceptable":0,"unsafe":0,"review":2,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":2,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.6576197724933469,"confidence_level":0.95}}},"generalize":{"n":6,"raw":{"n":6,"acceptable":0,"unsafe":6,"review":0,"automated_count":6,"automated_acceptable":0,"automated_unsafe":6,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":1.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":1.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":2.7755575615628914e-17,"upper":0.3903342879021653,"confidence_level":0.95}},"effective":{"n":6,"acceptable":0,"unsafe":1,"review":5,"automated_count":1,"automated_acceptable":0,"automated_unsafe":1,"unnecessary_reviews":5,"accuracy":0.0,"unsafe_rate":0.16666666666666666,"review_rate":0.8333333333333334,"automated_coverage":0.16666666666666666,"conditional_correctness":0.0,"conditional_unsafe":1.0,"unnecessary_review_rate":0.8333333333333334,"accuracy_wilson_95":{"lower":2.7755575615628914e-17,"upper":0.3903342879021653,"confidence_level":0.95}}},"review":{"n":0,"raw":{"n":0,"acceptable":0,"unsafe":0,"review":0,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":null,"unsafe_rate":null,"review_rate":null,"automated_coverage":null,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":null},"effective":{"n":0,"acceptable":0,"unsafe":0,"review":0,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":null,"unsafe_rate":null,"review_rate":null,"automated_coverage":null,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":null}}}},"challenge56-no_samples":{"n":56,"raw":{"n":56,"acceptable":23,"unsafe":27,"review":0,"automated_count":56,"automated_acceptable":23,"automated_unsafe":27,"unnecessary_reviews":0,"accuracy":0.4107142857142857,"unsafe_rate":0.48214285714285715,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.4107142857142857,"conditional_unsafe":0.48214285714285715,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.29166774287846203,"upper":0.541224031535603,"confidence_level":0.95}},"effective":{"n":56,"acceptable":21,"unsafe":0,"review":31,"automated_count":25,"automated_acceptable":19,"automated_unsafe":0,"unnecessary_reviews":29,"accuracy":0.375,"unsafe_rate":0.0,"review_rate":0.5535714285714286,"automated_coverage":0.44642857142857145,"conditional_correctness":0.76,"conditional_unsafe":0.0,"unnecessary_review_rate":0.5178571428571429,"accuracy_wilson_95":{"lower":0.260102143264709,"upper":0.5059463409149819,"confidence_level":0.95}},"confusion":{"raw":{"keep":{"keep":8,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0},"drop":{"keep":13,"drop":9,"mask":0,"pseudonymize":0,"generalize":0,"review":0},"mask":{"keep":0,"drop":2,"mask":6,"pseudonymize":0,"generalize":0,"review":0},"pseudonymize":{"keep":4,"drop":4,"mask":0,"pseudonymize":0,"generalize":0,"review":0},"generalize":{"keep":8,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0},"review":{"keep":2,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0}},"effective":{"keep":{"keep":5,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":3},"drop":{"keep":0,"drop":9,"mask":0,"pseudonymize":0,"generalize":0,"review":13},"mask":{"keep":0,"drop":2,"mask":5,"pseudonymize":0,"generalize":0,"review":1},"pseudonymize":{"keep":0,"drop":4,"mask":0,"pseudonymize":0,"generalize":0,"review":4},"generalize":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":8},"review":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":2}}},"probability_metrics":{"singleton_n":56,"multiacceptable_n":0,"multiacceptable_coverage":0.0,"nll":1.4547017163665146,"nll_zero_mass_n":0,"brier":0.7367628248967966,"acceptable_set_n":56,"acceptable_set_nll":1.4547017163665146,"acceptable_set_zero_mass_n":0},"calibration_all_labels":{"equal_width":{"n":56,"bin_count":10,"ece":0.30156307720712255,"signed_mean_gap":0.30156307720712255,"absolute_mean_gap":0.30156307720712255,"bins":[{"index":0,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.0,"upper":0.1,"upper_inclusive":false},{"index":1,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.1,"upper":0.2,"upper_inclusive":false},{"index":2,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.2,"upper":0.3,"upper_inclusive":false},{"index":3,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.3,"upper":0.4,"upper_inclusive":false},{"index":4,"count":6,"confidence":0.45171065628528595,"empirical_correctness":0.16666666666666666,"signed_gap":0.2850439896186193,"absolute_gap":0.2850439896186193,"lower":0.4,"upper":0.5,"upper_inclusive":false},{"index":5,"count":12,"confidence":0.5389134734869003,"empirical_correctness":0.16666666666666666,"signed_gap":0.3722468068202337,"absolute_gap":0.3722468068202337,"lower":0.5,"upper":0.6,"upper_inclusive":false},{"index":6,"count":8,"confidence":0.6461594104766846,"empirical_correctness":0.375,"signed_gap":0.27115941047668457,"absolute_gap":0.27115941047668457,"lower":0.6,"upper":0.7,"upper_inclusive":false},{"index":7,"count":11,"confidence":0.750518414107236,"empirical_correctness":0.36363636363636365,"signed_gap":0.3868820504708723,"absolute_gap":0.3868820504708723,"lower":0.7,"upper":0.8,"upper_inclusive":false},{"index":8,"count":8,"confidence":0.8537510931491852,"empirical_correctness":0.5,"signed_gap":0.3537510931491852,"absolute_gap":0.3537510931491852,"lower":0.8,"upper":0.9,"upper_inclusive":false},{"index":9,"count":11,"confidence":0.950483647259799,"empirical_correctness":0.8181818181818182,"signed_gap":0.13230182907798071,"absolute_gap":0.13230182907798071,"lower":0.9,"upper":1.0,"upper_inclusive":true}]},"equal_mass":{"n":56,"bin_count":10,"ece":0.30156307720712255,"signed_mean_gap":0.30156307720712255,"absolute_mean_gap":0.30156307720712255,"bins":[{"index":0,"count":5,"confidence":0.4458733081817627,"empirical_correctness":0.2,"signed_gap":0.24587330818176267,"absolute_gap":0.24587330818176267,"min_confidence":0.4305171072483063,"max_confidence":0.46741169691085815},{"index":1,"count":6,"confidence":0.5057760427395502,"empirical_correctness":0.16666666666666666,"signed_gap":0.3391093760728836,"absolute_gap":0.3391093760728836,"min_confidence":0.4808973968029022,"max_confidence":0.5214468836784363},{"index":2,"count":5,"confidence":0.5483115553855896,"empirical_correctness":0.2,"signed_gap":0.3483115553855896,"absolute_gap":0.3483115553855896,"min_confidence":0.52730393409729,"max_confidence":0.5826572179794312},{"index":3,"count":6,"confidence":0.6141501069068909,"empirical_correctness":0.3333333333333333,"signed_gap":0.28081677357355755,"absolute_gap":0.28081677357355755,"min_confidence":0.585608184337616,"max_confidence":0.6373648047447205},{"index":4,"count":6,"confidence":0.6811078389485677,"empirical_correctness":0.16666666666666666,"signed_gap":0.5144411722819011,"absolute_gap":0.5144411722819011,"min_confidence":0.6442112922668457,"max_confidence":0.7204111814498901},{"index":5,"count":5,"confidence":0.7479087114334106,"empirical_correctness":0.2,"signed_gap":0.5479087114334107,"absolute_gap":0.5479087114334107,"min_confidence":0.7358508706092834,"max_confidence":0.7548518180847168},{"index":6,"count":6,"confidence":0.7864059309164683,"empirical_correctness":0.5,"signed_gap":0.28640593091646827,"absolute_gap":0.28640593091646827,"min_confidence":0.7650580406188965,"max_confidence":0.8225422501564026},{"index":7,"count":5,"confidence":0.8608044743537903,"empirical_correctness":0.6,"signed_gap":0.2608044743537903,"absolute_gap":0.2608044743537903,"min_confidence":0.8367965817451477,"max_confidence":0.8720402121543884},{"index":8,"count":6,"confidence":0.923220713933309,"empirical_correctness":0.8333333333333334,"signed_gap":0.08988738059997559,"absolute_gap":0.08988738059997559,"min_confidence":0.8930824398994446,"max_confidence":0.9485576152801514},{"index":9,"count":6,"confidence":0.9681797126928965,"empirical_correctness":0.8333333333333334,"signed_gap":0.13484637935956312,"absolute_gap":0.13484637935956312,"min_confidence":0.9543910622596741,"max_confidence":0.981951892375946}]}},"calibration_singleton_labels":{"equal_width":{"n":56,"bin_count":10,"ece":0.30156307720712255,"signed_mean_gap":0.30156307720712255,"absolute_mean_gap":0.30156307720712255,"bins":[{"index":0,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.0,"upper":0.1,"upper_inclusive":false},{"index":1,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.1,"upper":0.2,"upper_inclusive":false},{"index":2,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.2,"upper":0.3,"upper_inclusive":false},{"index":3,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.3,"upper":0.4,"upper_inclusive":false},{"index":4,"count":6,"confidence":0.45171065628528595,"empirical_correctness":0.16666666666666666,"signed_gap":0.2850439896186193,"absolute_gap":0.2850439896186193,"lower":0.4,"upper":0.5,"upper_inclusive":false},{"index":5,"count":12,"confidence":0.5389134734869003,"empirical_correctness":0.16666666666666666,"signed_gap":0.3722468068202337,"absolute_gap":0.3722468068202337,"lower":0.5,"upper":0.6,"upper_inclusive":false},{"index":6,"count":8,"confidence":0.6461594104766846,"empirical_correctness":0.375,"signed_gap":0.27115941047668457,"absolute_gap":0.27115941047668457,"lower":0.6,"upper":0.7,"upper_inclusive":false},{"index":7,"count":11,"confidence":0.750518414107236,"empirical_correctness":0.36363636363636365,"signed_gap":0.3868820504708723,"absolute_gap":0.3868820504708723,"lower":0.7,"upper":0.8,"upper_inclusive":false},{"index":8,"count":8,"confidence":0.8537510931491852,"empirical_correctness":0.5,"signed_gap":0.3537510931491852,"absolute_gap":0.3537510931491852,"lower":0.8,"upper":0.9,"upper_inclusive":false},{"index":9,"count":11,"confidence":0.950483647259799,"empirical_correctness":0.8181818181818182,"signed_gap":0.13230182907798071,"absolute_gap":0.13230182907798071,"lower":0.9,"upper":1.0,"upper_inclusive":true}]},"equal_mass":{"n":56,"bin_count":10,"ece":0.30156307720712255,"signed_mean_gap":0.30156307720712255,"absolute_mean_gap":0.30156307720712255,"bins":[{"index":0,"count":5,"confidence":0.4458733081817627,"empirical_correctness":0.2,"signed_gap":0.24587330818176267,"absolute_gap":0.24587330818176267,"min_confidence":0.4305171072483063,"max_confidence":0.46741169691085815},{"index":1,"count":6,"confidence":0.5057760427395502,"empirical_correctness":0.16666666666666666,"signed_gap":0.3391093760728836,"absolute_gap":0.3391093760728836,"min_confidence":0.4808973968029022,"max_confidence":0.5214468836784363},{"index":2,"count":5,"confidence":0.5483115553855896,"empirical_correctness":0.2,"signed_gap":0.3483115553855896,"absolute_gap":0.3483115553855896,"min_confidence":0.52730393409729,"max_confidence":0.5826572179794312},{"index":3,"count":6,"confidence":0.6141501069068909,"empirical_correctness":0.3333333333333333,"signed_gap":0.28081677357355755,"absolute_gap":0.28081677357355755,"min_confidence":0.585608184337616,"max_confidence":0.6373648047447205},{"index":4,"count":6,"confidence":0.6811078389485677,"empirical_correctness":0.16666666666666666,"signed_gap":0.5144411722819011,"absolute_gap":0.5144411722819011,"min_confidence":0.6442112922668457,"max_confidence":0.7204111814498901},{"index":5,"count":5,"confidence":0.7479087114334106,"empirical_correctness":0.2,"signed_gap":0.5479087114334107,"absolute_gap":0.5479087114334107,"min_confidence":0.7358508706092834,"max_confidence":0.7548518180847168},{"index":6,"count":6,"confidence":0.7864059309164683,"empirical_correctness":0.5,"signed_gap":0.28640593091646827,"absolute_gap":0.28640593091646827,"min_confidence":0.7650580406188965,"max_confidence":0.8225422501564026},{"index":7,"count":5,"confidence":0.8608044743537903,"empirical_correctness":0.6,"signed_gap":0.2608044743537903,"absolute_gap":0.2608044743537903,"min_confidence":0.8367965817451477,"max_confidence":0.8720402121543884},{"index":8,"count":6,"confidence":0.923220713933309,"empirical_correctness":0.8333333333333334,"signed_gap":0.08988738059997559,"absolute_gap":0.08988738059997559,"min_confidence":0.8930824398994446,"max_confidence":0.9485576152801514},{"index":9,"count":6,"confidence":0.9681797126928965,"empirical_correctness":0.8333333333333334,"signed_gap":0.13484637935956312,"absolute_gap":0.13484637935956312,"min_confidence":0.9543910622596741,"max_confidence":0.981951892375946}]}},"latency_ms":{"n":56,"median":59.75749983917922,"p95":61.84834992745891},"per_action":{"keep":{"n":8,"raw":{"n":8,"acceptable":8,"unsafe":0,"review":0,"automated_count":8,"automated_acceptable":8,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":1.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.6755924351161198,"upper":1.0,"confidence_level":0.95}},"effective":{"n":8,"acceptable":5,"unsafe":0,"review":3,"automated_count":5,"automated_acceptable":5,"automated_unsafe":0,"unnecessary_reviews":3,"accuracy":0.625,"unsafe_rate":0.0,"review_rate":0.375,"automated_coverage":0.625,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.375,"accuracy_wilson_95":{"lower":0.30574239460262737,"upper":0.8631557141764026,"confidence_level":0.95}}},"drop":{"n":22,"raw":{"n":22,"acceptable":9,"unsafe":13,"review":0,"automated_count":22,"automated_acceptable":9,"automated_unsafe":13,"unnecessary_reviews":0,"accuracy":0.4090909090909091,"unsafe_rate":0.5909090909090909,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.4090909090909091,"conditional_unsafe":0.5909090909090909,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.2325581619647187,"upper":0.6126518152368488,"confidence_level":0.95}},"effective":{"n":22,"acceptable":9,"unsafe":0,"review":13,"automated_count":9,"automated_acceptable":9,"automated_unsafe":0,"unnecessary_reviews":13,"accuracy":0.4090909090909091,"unsafe_rate":0.0,"review_rate":0.5909090909090909,"automated_coverage":0.4090909090909091,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.5909090909090909,"accuracy_wilson_95":{"lower":0.2325581619647187,"upper":0.6126518152368488,"confidence_level":0.95}}},"mask":{"n":8,"raw":{"n":8,"acceptable":6,"unsafe":0,"review":0,"automated_count":8,"automated_acceptable":6,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":0.75,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.75,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.40927543031016883,"upper":0.9285207872478909,"confidence_level":0.95}},"effective":{"n":8,"acceptable":5,"unsafe":0,"review":1,"automated_count":7,"automated_acceptable":5,"automated_unsafe":0,"unnecessary_reviews":1,"accuracy":0.625,"unsafe_rate":0.0,"review_rate":0.125,"automated_coverage":0.875,"conditional_correctness":0.7142857142857143,"conditional_unsafe":0.0,"unnecessary_review_rate":0.125,"accuracy_wilson_95":{"lower":0.30574239460262737,"upper":0.8631557141764026,"confidence_level":0.95}}},"pseudonymize":{"n":8,"raw":{"n":8,"acceptable":0,"unsafe":4,"review":0,"automated_count":8,"automated_acceptable":0,"automated_unsafe":4,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":0.5,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":0.5,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.32440756488388023,"confidence_level":0.95}},"effective":{"n":8,"acceptable":0,"unsafe":0,"review":4,"automated_count":4,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":4,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":0.5,"automated_coverage":0.5,"conditional_correctness":0.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.5,"accuracy_wilson_95":{"lower":0.0,"upper":0.32440756488388023,"confidence_level":0.95}}},"generalize":{"n":8,"raw":{"n":8,"acceptable":0,"unsafe":8,"review":0,"automated_count":8,"automated_acceptable":0,"automated_unsafe":8,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":1.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":1.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.32440756488388023,"confidence_level":0.95}},"effective":{"n":8,"acceptable":0,"unsafe":0,"review":8,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":8,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.32440756488388023,"confidence_level":0.95}}},"review":{"n":2,"raw":{"n":2,"acceptable":0,"unsafe":2,"review":0,"automated_count":2,"automated_acceptable":0,"automated_unsafe":2,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":1.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":1.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.6576197724933469,"confidence_level":0.95}},"effective":{"n":2,"acceptable":2,"unsafe":0,"review":2,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":1.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.34238022750665303,"upper":1.0,"confidence_level":0.95}}}}},"challenge56-compact_no_samples":{"n":56,"raw":{"n":56,"acceptable":12,"unsafe":44,"review":0,"automated_count":56,"automated_acceptable":12,"automated_unsafe":44,"unnecessary_reviews":0,"accuracy":0.21428571428571427,"unsafe_rate":0.7857142857142857,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.21428571428571427,"conditional_unsafe":0.7857142857142857,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.1270590424909565,"upper":0.33819463563405117,"confidence_level":0.95}},"effective":{"n":56,"acceptable":11,"unsafe":0,"review":47,"automated_count":9,"automated_acceptable":9,"automated_unsafe":0,"unnecessary_reviews":45,"accuracy":0.19642857142857142,"unsafe_rate":0.0,"review_rate":0.8392857142857143,"automated_coverage":0.16071428571428573,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.8035714285714286,"accuracy_wilson_95":{"lower":0.11338595674323658,"upper":0.31844607626458404,"confidence_level":0.95}},"confusion":{"raw":{"keep":{"keep":8,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0},"drop":{"keep":18,"drop":4,"mask":0,"pseudonymize":0,"generalize":0,"review":0},"mask":{"keep":8,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0},"pseudonymize":{"keep":8,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0},"generalize":{"keep":8,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0},"review":{"keep":2,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0}},"effective":{"keep":{"keep":5,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":3},"drop":{"keep":0,"drop":4,"mask":0,"pseudonymize":0,"generalize":0,"review":18},"mask":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":8},"pseudonymize":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":8},"generalize":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":8},"review":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":2}}},"probability_metrics":{"singleton_n":56,"multiacceptable_n":0,"multiacceptable_coverage":0.0,"nll":3.0600007443335016,"nll_zero_mass_n":0,"brier":1.3373362238609308,"acceptable_set_n":56,"acceptable_set_nll":3.0600007443335016,"acceptable_set_zero_mass_n":0},"calibration_all_labels":{"equal_width":{"n":56,"bin_count":10,"ece":0.6771087699702808,"signed_mean_gap":0.6618546151689121,"absolute_mean_gap":0.6618546151689121,"bins":[{"index":0,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.0,"upper":0.1,"upper_inclusive":false},{"index":1,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.1,"upper":0.2,"upper_inclusive":false},{"index":2,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.2,"upper":0.3,"upper_inclusive":false},{"index":3,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.3,"upper":0.4,"upper_inclusive":false},{"index":4,"count":2,"confidence":0.47148874402046204,"empirical_correctness":0.5,"signed_gap":-0.028511255979537964,"absolute_gap":0.028511255979537964,"lower":0.4,"upper":0.5,"upper_inclusive":false},{"index":5,"count":3,"confidence":0.5920213262240092,"empirical_correctness":0.0,"signed_gap":0.5920213262240092,"absolute_gap":0.5920213262240092,"lower":0.5,"upper":0.6,"upper_inclusive":false},{"index":6,"count":1,"confidence":0.629906177520752,"empirical_correctness":1.0,"signed_gap":-0.37009382247924805,"absolute_gap":0.37009382247924805,"lower":0.6,"upper":0.7,"upper_inclusive":false},{"index":7,"count":6,"confidence":0.7538639108339945,"empirical_correctness":0.16666666666666666,"signed_gap":0.5871972441673279,"absolute_gap":0.5871972441673279,"lower":0.7,"upper":0.8,"upper_inclusive":false},{"index":8,"count":11,"confidence":0.8482577367262407,"empirical_correctness":0.09090909090909091,"signed_gap":0.7573486458171498,"absolute_gap":0.7573486458171498,"lower":0.8,"upper":0.9,"upper_inclusive":false},{"index":9,"count":33,"confidence":0.9654815829161442,"empirical_correctness":0.24242424242424243,"signed_gap":0.7230573404919017,"absolute_gap":0.7230573404919017,"lower":0.9,"upper":1.0,"upper_inclusive":true}]},"equal_mass":{"n":56,"bin_count":10,"ece":0.6618546151689121,"signed_mean_gap":0.6618546151689121,"absolute_mean_gap":0.6618546151689121,"bins":[{"index":0,"count":5,"confidence":0.5438082933425903,"empirical_correctness":0.2,"signed_gap":0.3438082933425903,"absolute_gap":0.3438082933425903,"min_confidence":0.46436813473701477,"max_confidence":0.5963131189346313},{"index":1,"count":6,"confidence":0.7259926994641622,"empirical_correctness":0.3333333333333333,"signed_gap":0.3926593661308289,"absolute_gap":0.3926593661308289,"min_confidence":0.629906177520752,"max_confidence":0.7686060667037964},{"index":2,"count":5,"confidence":0.8094712257385254,"empirical_correctness":0.0,"signed_gap":0.8094712257385254,"absolute_gap":0.8094712257385254,"min_confidence":0.7971334457397461,"max_confidence":0.8225770592689514},{"index":3,"count":6,"confidence":0.8640438814957937,"empirical_correctness":0.16666666666666666,"signed_gap":0.6973772148291271,"absolute_gap":0.6973772148291271,"min_confidence":0.8402976989746094,"max_confidence":0.8868017196655273},{"index":4,"count":6,"confidence":0.9248997271060944,"empirical_correctness":0.0,"signed_gap":0.9248997271060944,"absolute_gap":0.9248997271060944,"min_confidence":0.8963491320610046,"max_confidence":0.9473453164100647},{"index":5,"count":5,"confidence":0.9497984170913696,"empirical_correctness":0.0,"signed_gap":0.9497984170913696,"absolute_gap":0.9497984170913696,"min_confidence":0.9479034543037415,"max_confidence":0.9532163739204407},{"index":6,"count":6,"confidence":0.9607151548067728,"empirical_correctness":0.16666666666666666,"signed_gap":0.7940484881401062,"absolute_gap":0.7940484881401062,"min_confidence":0.9537230134010315,"max_confidence":0.9667998552322388},{"index":7,"count":5,"confidence":0.9762066602706909,"empirical_correctness":0.0,"signed_gap":0.9762066602706909,"absolute_gap":0.9762066602706909,"min_confidence":0.9701721668243408,"max_confidence":0.9798903465270996},{"index":8,"count":6,"confidence":0.9817302723725637,"empirical_correctness":0.5,"signed_gap":0.4817302723725637,"absolute_gap":0.4817302723725637,"min_confidence":0.980356752872467,"max_confidence":0.9831662178039551},{"index":9,"count":6,"confidence":0.987190842628479,"empirical_correctness":0.6666666666666666,"signed_gap":0.3205241759618124,"absolute_gap":0.3205241759618124,"min_confidence":0.9835131764411926,"max_confidence":0.9916144609451294}]}},"calibration_singleton_labels":{"equal_width":{"n":56,"bin_count":10,"ece":0.6771087699702808,"signed_mean_gap":0.6618546151689121,"absolute_mean_gap":0.6618546151689121,"bins":[{"index":0,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.0,"upper":0.1,"upper_inclusive":false},{"index":1,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.1,"upper":0.2,"upper_inclusive":false},{"index":2,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.2,"upper":0.3,"upper_inclusive":false},{"index":3,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.3,"upper":0.4,"upper_inclusive":false},{"index":4,"count":2,"confidence":0.47148874402046204,"empirical_correctness":0.5,"signed_gap":-0.028511255979537964,"absolute_gap":0.028511255979537964,"lower":0.4,"upper":0.5,"upper_inclusive":false},{"index":5,"count":3,"confidence":0.5920213262240092,"empirical_correctness":0.0,"signed_gap":0.5920213262240092,"absolute_gap":0.5920213262240092,"lower":0.5,"upper":0.6,"upper_inclusive":false},{"index":6,"count":1,"confidence":0.629906177520752,"empirical_correctness":1.0,"signed_gap":-0.37009382247924805,"absolute_gap":0.37009382247924805,"lower":0.6,"upper":0.7,"upper_inclusive":false},{"index":7,"count":6,"confidence":0.7538639108339945,"empirical_correctness":0.16666666666666666,"signed_gap":0.5871972441673279,"absolute_gap":0.5871972441673279,"lower":0.7,"upper":0.8,"upper_inclusive":false},{"index":8,"count":11,"confidence":0.8482577367262407,"empirical_correctness":0.09090909090909091,"signed_gap":0.7573486458171498,"absolute_gap":0.7573486458171498,"lower":0.8,"upper":0.9,"upper_inclusive":false},{"index":9,"count":33,"confidence":0.9654815829161442,"empirical_correctness":0.24242424242424243,"signed_gap":0.7230573404919017,"absolute_gap":0.7230573404919017,"lower":0.9,"upper":1.0,"upper_inclusive":true}]},"equal_mass":{"n":56,"bin_count":10,"ece":0.6618546151689121,"signed_mean_gap":0.6618546151689121,"absolute_mean_gap":0.6618546151689121,"bins":[{"index":0,"count":5,"confidence":0.5438082933425903,"empirical_correctness":0.2,"signed_gap":0.3438082933425903,"absolute_gap":0.3438082933425903,"min_confidence":0.46436813473701477,"max_confidence":0.5963131189346313},{"index":1,"count":6,"confidence":0.7259926994641622,"empirical_correctness":0.3333333333333333,"signed_gap":0.3926593661308289,"absolute_gap":0.3926593661308289,"min_confidence":0.629906177520752,"max_confidence":0.7686060667037964},{"index":2,"count":5,"confidence":0.8094712257385254,"empirical_correctness":0.0,"signed_gap":0.8094712257385254,"absolute_gap":0.8094712257385254,"min_confidence":0.7971334457397461,"max_confidence":0.8225770592689514},{"index":3,"count":6,"confidence":0.8640438814957937,"empirical_correctness":0.16666666666666666,"signed_gap":0.6973772148291271,"absolute_gap":0.6973772148291271,"min_confidence":0.8402976989746094,"max_confidence":0.8868017196655273},{"index":4,"count":6,"confidence":0.9248997271060944,"empirical_correctness":0.0,"signed_gap":0.9248997271060944,"absolute_gap":0.9248997271060944,"min_confidence":0.8963491320610046,"max_confidence":0.9473453164100647},{"index":5,"count":5,"confidence":0.9497984170913696,"empirical_correctness":0.0,"signed_gap":0.9497984170913696,"absolute_gap":0.9497984170913696,"min_confidence":0.9479034543037415,"max_confidence":0.9532163739204407},{"index":6,"count":6,"confidence":0.9607151548067728,"empirical_correctness":0.16666666666666666,"signed_gap":0.7940484881401062,"absolute_gap":0.7940484881401062,"min_confidence":0.9537230134010315,"max_confidence":0.9667998552322388},{"index":7,"count":5,"confidence":0.9762066602706909,"empirical_correctness":0.0,"signed_gap":0.9762066602706909,"absolute_gap":0.9762066602706909,"min_confidence":0.9701721668243408,"max_confidence":0.9798903465270996},{"index":8,"count":6,"confidence":0.9817302723725637,"empirical_correctness":0.5,"signed_gap":0.4817302723725637,"absolute_gap":0.4817302723725637,"min_confidence":0.980356752872467,"max_confidence":0.9831662178039551},{"index":9,"count":6,"confidence":0.987190842628479,"empirical_correctness":0.6666666666666666,"signed_gap":0.3205241759618124,"absolute_gap":0.3205241759618124,"min_confidence":0.9835131764411926,"max_confidence":0.9916144609451294}]}},"latency_ms":{"n":56,"median":69.72469994798303,"p95":71.3164500775747},"per_action":{"keep":{"n":8,"raw":{"n":8,"acceptable":8,"unsafe":0,"review":0,"automated_count":8,"automated_acceptable":8,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":1.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.6755924351161198,"upper":1.0,"confidence_level":0.95}},"effective":{"n":8,"acceptable":5,"unsafe":0,"review":3,"automated_count":5,"automated_acceptable":5,"automated_unsafe":0,"unnecessary_reviews":3,"accuracy":0.625,"unsafe_rate":0.0,"review_rate":0.375,"automated_coverage":0.625,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.375,"accuracy_wilson_95":{"lower":0.30574239460262737,"upper":0.8631557141764026,"confidence_level":0.95}}},"drop":{"n":22,"raw":{"n":22,"acceptable":4,"unsafe":18,"review":0,"automated_count":22,"automated_acceptable":4,"automated_unsafe":18,"unnecessary_reviews":0,"accuracy":0.18181818181818182,"unsafe_rate":0.8181818181818182,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.18181818181818182,"conditional_unsafe":0.8181818181818182,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.07306884583422052,"upper":0.38516607437126543,"confidence_level":0.95}},"effective":{"n":22,"acceptable":4,"unsafe":0,"review":18,"automated_count":4,"automated_acceptable":4,"automated_unsafe":0,"unnecessary_reviews":18,"accuracy":0.18181818181818182,"unsafe_rate":0.0,"review_rate":0.8181818181818182,"automated_coverage":0.18181818181818182,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.8181818181818182,"accuracy_wilson_95":{"lower":0.07306884583422052,"upper":0.38516607437126543,"confidence_level":0.95}}},"mask":{"n":8,"raw":{"n":8,"acceptable":0,"unsafe":8,"review":0,"automated_count":8,"automated_acceptable":0,"automated_unsafe":8,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":1.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":1.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.32440756488388023,"confidence_level":0.95}},"effective":{"n":8,"acceptable":0,"unsafe":0,"review":8,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":8,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.32440756488388023,"confidence_level":0.95}}},"pseudonymize":{"n":8,"raw":{"n":8,"acceptable":0,"unsafe":8,"review":0,"automated_count":8,"automated_acceptable":0,"automated_unsafe":8,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":1.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":1.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.32440756488388023,"confidence_level":0.95}},"effective":{"n":8,"acceptable":0,"unsafe":0,"review":8,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":8,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.32440756488388023,"confidence_level":0.95}}},"generalize":{"n":8,"raw":{"n":8,"acceptable":0,"unsafe":8,"review":0,"automated_count":8,"automated_acceptable":0,"automated_unsafe":8,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":1.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":1.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.32440756488388023,"confidence_level":0.95}},"effective":{"n":8,"acceptable":0,"unsafe":0,"review":8,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":8,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.32440756488388023,"confidence_level":0.95}}},"review":{"n":2,"raw":{"n":2,"acceptable":0,"unsafe":2,"review":0,"automated_count":2,"automated_acceptable":0,"automated_unsafe":2,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":1.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":1.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.6576197724933469,"confidence_level":0.95}},"effective":{"n":2,"acceptable":2,"unsafe":0,"review":2,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":1.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.34238022750665303,"upper":1.0,"confidence_level":0.95}}}}}},"composite":{"n":198,"scope":"descriptive sum across four correlated panels","raw":{"n":198,"acceptable":70,"unsafe":107,"review":0,"automated_count":198,"automated_acceptable":70,"automated_unsafe":107,"unnecessary_reviews":0,"accuracy":0.35353535353535354,"unsafe_rate":0.5404040404040404,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.35353535353535354,"conditional_unsafe":0.5404040404040404,"unnecessary_review_rate":0.0},"effective":{"n":198,"acceptable":72,"unsafe":5,"review":107,"automated_count":91,"automated_acceptable":63,"automated_unsafe":5,"unnecessary_reviews":98,"accuracy":0.36363636363636365,"unsafe_rate":0.025252525252525252,"review_rate":0.5404040404040404,"automated_coverage":0.4595959595959596,"conditional_correctness":0.6923076923076923,"conditional_unsafe":0.054945054945054944,"unnecessary_review_rate":0.494949494949495}}},{"id":"impacte--laya-medical-escalation-fp32","platform":"Local","model_id":"impacte/laya-medical-escalation","revision":"91c869598a4c29ec4e68cfea40ac3761f609ca54","loading":"float32 / none","scoring_kind":"Native decision readout","settings":{"dtype":"float32","quantization":"none","max_tokens":512,"head_max_tokens":192,"temperature":1.0458,"autocast":false,"truncation":false,"cpu_offload":false,"guard_threshold":0.45,"seed":0,"tf32":false},"scoring_metadata":{"backend":"laya_native_local_builtin","readout":"native_mask_slot_scorer_numpy_float32_softmax","probability_source":"native_choice_unrounded_published_temperature_not_task_calibrated","latency_scope":"synchronized serial complete encoder+head+act forward,input transfer and native readout; tokenization excluded","native_disclosures":["Stock installed torch attention in place of ONNX dynamic equivalent; no custom kernels or repo imports.","FP32/no-autocast explicit precision: convert every saved F16/BF16/F32 tensor toFP32; no quantization.","Unrounded native probability; guard uses winner p, not entropy-concentration confidence.","Act head fully loaded and evaluated; not used to alter six-action answer.","Native choice:6-10 bucket else choice scalar0, T=1.0458. No task fitting, priors correction, prompt change or finetuning here."],"equal_mass_convention":"rank bins with floor boundaries at floor(i*n/k), k=min(10,n//2); minimum two rows"},"load_ms":null,"peak_allocated_bytes":1748760576,"peak_reserved_bytes":1870659584,"panels":{"diagnostic43-no_samples":{"n":43,"raw":{"n":43,"acceptable":13,"unsafe":29,"review":0,"automated_count":43,"automated_acceptable":13,"automated_unsafe":29,"unnecessary_reviews":0,"accuracy":0.3023255813953488,"unsafe_rate":0.6744186046511628,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.3023255813953488,"conditional_unsafe":0.6744186046511628,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.1860200013584818,"upper":0.45105364177729607,"confidence_level":0.95}},"effective":{"n":43,"acceptable":5,"unsafe":0,"review":40,"automated_count":3,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":35,"accuracy":0.11627906976744186,"unsafe_rate":0.0,"review_rate":0.9302325581395349,"automated_coverage":0.06976744186046512,"conditional_correctness":0.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.813953488372093,"accuracy_wilson_95":{"lower":0.05070427739804166,"upper":0.24479161810082128,"confidence_level":0.95}},"confusion":{"raw":{"keep":{"keep":12,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0},"drop":{"keep":15,"drop":1,"mask":0,"pseudonymize":0,"generalize":0,"review":0},"mask":{"keep":1,"drop":1,"mask":0,"pseudonymize":0,"generalize":0,"review":0},"pseudonymize":{"keep":2,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0},"generalize":{"keep":6,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0},"review":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0}},"effective":{"keep":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":12},"drop":{"keep":0,"drop":0,"mask":3,"pseudonymize":0,"generalize":0,"review":13},"mask":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":2},"pseudonymize":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":2},"generalize":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":6},"review":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0}}},"probability_metrics":{"singleton_n":38,"multiacceptable_n":5,"multiacceptable_coverage":0.11627906976744186,"nll":1.6046512043260683,"nll_zero_mass_n":0,"brier":0.7671331619630207,"acceptable_set_n":43,"acceptable_set_nll":1.5016274704282446,"acceptable_set_zero_mass_n":0},"calibration_all_labels":{"equal_width":{"n":43,"bin_count":10,"ece":0.03690882269726242,"signed_mean_gap":-0.03690882269726242,"absolute_mean_gap":0.03690882269726242,"bins":[{"index":0,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.0,"upper":0.1,"upper_inclusive":false},{"index":1,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.1,"upper":0.2,"upper_inclusive":false},{"index":2,"count":36,"confidence":0.25434500227371853,"empirical_correctness":0.2777777777777778,"signed_gap":-0.023432775504059256,"absolute_gap":0.023432775504059256,"lower":0.2,"upper":0.3,"upper_inclusive":false},{"index":3,"count":7,"confidence":0.3223572203091213,"empirical_correctness":0.42857142857142855,"signed_gap":-0.10621420826230726,"absolute_gap":0.10621420826230726,"lower":0.3,"upper":0.4,"upper_inclusive":false},{"index":4,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.4,"upper":0.5,"upper_inclusive":false},{"index":5,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.5,"upper":0.6,"upper_inclusive":false},{"index":6,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.6,"upper":0.7,"upper_inclusive":false},{"index":7,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.7,"upper":0.8,"upper_inclusive":false},{"index":8,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.8,"upper":0.9,"upper_inclusive":false},{"index":9,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.9,"upper":1.0,"upper_inclusive":true}]},"equal_mass":{"n":43,"bin_count":10,"ece":0.15353930343029112,"signed_mean_gap":-0.03690882269726244,"absolute_mean_gap":0.03690882269726244,"bins":[{"index":0,"count":4,"confidence":0.21728567779064178,"empirical_correctness":0.5,"signed_gap":-0.2827143222093582,"absolute_gap":0.2827143222093582,"min_confidence":0.2081620842218399,"max_confidence":0.22153252363204956},{"index":1,"count":4,"confidence":0.22576336190104485,"empirical_correctness":0.25,"signed_gap":-0.024236638098955154,"absolute_gap":0.024236638098955154,"min_confidence":0.22407355904579163,"max_confidence":0.22952412068843842},{"index":2,"count":4,"confidence":0.23740756511688232,"empirical_correctness":0.0,"signed_gap":0.23740756511688232,"absolute_gap":0.23740756511688232,"min_confidence":0.23083250224590302,"max_confidence":0.2414809912443161},{"index":3,"count":5,"confidence":0.24619472324848174,"empirical_correctness":0.4,"signed_gap":-0.15380527675151828,"absolute_gap":0.15380527675151828,"min_confidence":0.24549975991249084,"max_confidence":0.24705307185649872},{"index":4,"count":4,"confidence":0.25496210157871246,"empirical_correctness":0.25,"signed_gap":0.004962101578712463,"absolute_gap":0.004962101578712463,"min_confidence":0.25041401386260986,"max_confidence":0.26060011982917786},{"index":5,"count":4,"confidence":0.26217540353536606,"empirical_correctness":0.5,"signed_gap":-0.23782459646463394,"absolute_gap":0.23782459646463394,"min_confidence":0.26122716069221497,"max_confidence":0.26315993070602417},{"index":6,"count":5,"confidence":0.2749508857727051,"empirical_correctness":0.2,"signed_gap":0.07495088577270509,"absolute_gap":0.07495088577270509,"min_confidence":0.2676112651824951,"max_confidence":0.27920469641685486},{"index":7,"count":4,"confidence":0.290830560028553,"empirical_correctness":0.0,"signed_gap":0.290830560028553,"absolute_gap":0.290830560028553,"min_confidence":0.284807950258255,"max_confidence":0.2935350835323334},{"index":8,"count":4,"confidence":0.30097295343875885,"empirical_correctness":0.5,"signed_gap":-0.19902704656124115,"absolute_gap":0.19902704656124115,"min_confidence":0.29726237058639526,"max_confidence":0.3045034408569336},{"index":9,"count":5,"confidence":0.32992041707038877,"empirical_correctness":0.4,"signed_gap":-0.07007958292961125,"absolute_gap":0.07007958292961125,"min_confidence":0.30636247992515564,"max_confidence":0.35832446813583374}]}},"calibration_singleton_labels":{"equal_width":{"n":38,"bin_count":10,"ece":0.07941999168772446,"signed_mean_gap":-0.07941999168772446,"absolute_mean_gap":0.07941999168772446,"bins":[{"index":0,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.0,"upper":0.1,"upper_inclusive":false},{"index":1,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.1,"upper":0.2,"upper_inclusive":false},{"index":2,"count":33,"confidence":0.25346834099654,"empirical_correctness":0.30303030303030304,"signed_gap":-0.04956196203376306,"absolute_gap":0.04956196203376306,"lower":0.2,"upper":0.3,"upper_inclusive":false},{"index":3,"count":5,"confidence":0.32351701259613036,"empirical_correctness":0.6,"signed_gap":-0.2764829874038696,"absolute_gap":0.2764829874038696,"lower":0.3,"upper":0.4,"upper_inclusive":false},{"index":4,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.4,"upper":0.5,"upper_inclusive":false},{"index":5,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.5,"upper":0.6,"upper_inclusive":false},{"index":6,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.6,"upper":0.7,"upper_inclusive":false},{"index":7,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.7,"upper":0.8,"upper_inclusive":false},{"index":8,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.8,"upper":0.9,"upper_inclusive":false},{"index":9,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.9,"upper":1.0,"upper_inclusive":true}]},"equal_mass":{"n":38,"bin_count":10,"ece":0.14046948834469444,"signed_mean_gap":-0.07941999168772447,"absolute_mean_gap":0.07941999168772447,"bins":[{"index":0,"count":3,"confidence":0.21587006251017252,"empirical_correctness":0.6666666666666666,"signed_gap":-0.45079660415649414,"absolute_gap":0.45079660415649414,"min_confidence":0.2081620842218399,"max_confidence":0.2199031561613083},{"index":1,"count":4,"confidence":0.22483355924487114,"empirical_correctness":0.25,"signed_gap":-0.02516644075512886,"absolute_gap":0.02516644075512886,"min_confidence":0.22153252363204956,"max_confidence":0.22952412068843842},{"index":2,"count":4,"confidence":0.23740756511688232,"empirical_correctness":0.0,"signed_gap":0.23740756511688232,"absolute_gap":0.23740756511688232,"min_confidence":0.23083250224590302,"max_confidence":0.2414809912443161},{"index":3,"count":4,"confidence":0.2459801360964775,"empirical_correctness":0.5,"signed_gap":-0.2540198639035225,"absolute_gap":0.2540198639035225,"min_confidence":0.24549975991249084,"max_confidence":0.24672925472259521},{"index":4,"count":4,"confidence":0.2515753395855427,"empirical_correctness":0.25,"signed_gap":0.0015753395855426788,"absolute_gap":0.0015753395855426788,"min_confidence":0.24705307185649872,"max_confidence":0.2558419704437256},{"index":5,"count":3,"confidence":0.26104046901067096,"empirical_correctness":0.3333333333333333,"signed_gap":-0.07229286432266235,"absolute_gap":0.07229286432266235,"min_confidence":0.26060011982917786,"max_confidence":0.2612941265106201},{"index":6,"count":4,"confidence":0.26735761016607285,"empirical_correctness":0.25,"signed_gap":0.017357610166072845,"absolute_gap":0.017357610166072845,"min_confidence":0.263020396232605,"max_confidence":0.2756388485431671},{"index":7,"count":4,"confidence":0.2836445942521095,"empirical_correctness":0.25,"signed_gap":0.03364459425210953,"absolute_gap":0.03364459425210953,"min_confidence":0.27721503376960754,"max_confidence":0.2933506965637207},{"index":8,"count":4,"confidence":0.2982308641076088,"empirical_correctness":0.5,"signed_gap":-0.2017691358923912,"absolute_gap":0.2017691358923912,"min_confidence":0.2935350835323334,"max_confidence":0.3023950159549713},{"index":9,"count":4,"confidence":0.32879751175642014,"empirical_correctness":0.5,"signed_gap":-0.17120248824357986,"absolute_gap":0.17120248824357986,"min_confidence":0.3045034408569336,"max_confidence":0.35832446813583374}]}},"latency_ms":{"n":43,"median":18.181543971877545,"p95":18.711714103119448},"per_action":{"keep":{"n":12,"raw":{"n":12,"acceptable":12,"unsafe":0,"review":0,"automated_count":12,"automated_acceptable":12,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":1.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.7575059933447592,"upper":1.0,"confidence_level":0.95}},"effective":{"n":12,"acceptable":0,"unsafe":0,"review":12,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":12,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":2.7755575615628914e-17,"upper":0.24249400665524085,"confidence_level":0.95}}},"drop":{"n":16,"raw":{"n":16,"acceptable":1,"unsafe":15,"review":0,"automated_count":16,"automated_acceptable":1,"automated_unsafe":15,"unnecessary_reviews":0,"accuracy":0.0625,"unsafe_rate":0.9375,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0625,"conditional_unsafe":0.9375,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.01111934476464252,"upper":0.28328737570298945,"confidence_level":0.95}},"effective":{"n":16,"acceptable":0,"unsafe":0,"review":13,"automated_count":3,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":13,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":0.8125,"automated_coverage":0.1875,"conditional_correctness":0.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.8125,"accuracy_wilson_95":{"lower":0.0,"upper":0.1936076805344365,"confidence_level":0.95}}},"mask":{"n":2,"raw":{"n":2,"acceptable":0,"unsafe":1,"review":0,"automated_count":2,"automated_acceptable":0,"automated_unsafe":1,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":0.5,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":0.5,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.6576197724933469,"confidence_level":0.95}},"effective":{"n":2,"acceptable":0,"unsafe":0,"review":2,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":2,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.6576197724933469,"confidence_level":0.95}}},"pseudonymize":{"n":2,"raw":{"n":2,"acceptable":0,"unsafe":2,"review":0,"automated_count":2,"automated_acceptable":0,"automated_unsafe":2,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":1.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":1.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.6576197724933469,"confidence_level":0.95}},"effective":{"n":2,"acceptable":0,"unsafe":0,"review":2,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":2,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.6576197724933469,"confidence_level":0.95}}},"generalize":{"n":6,"raw":{"n":6,"acceptable":0,"unsafe":6,"review":0,"automated_count":6,"automated_acceptable":0,"automated_unsafe":6,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":1.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":1.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":2.7755575615628914e-17,"upper":0.3903342879021653,"confidence_level":0.95}},"effective":{"n":6,"acceptable":0,"unsafe":0,"review":6,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":6,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":2.7755575615628914e-17,"upper":0.3903342879021653,"confidence_level":0.95}}},"review":{"n":0,"raw":{"n":0,"acceptable":0,"unsafe":0,"review":0,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":null,"unsafe_rate":null,"review_rate":null,"automated_coverage":null,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":null},"effective":{"n":0,"acceptable":0,"unsafe":0,"review":0,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":null,"unsafe_rate":null,"review_rate":null,"automated_coverage":null,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":null}}}},"diagnostic43-compact_no_samples":{"n":43,"raw":{"n":43,"acceptable":15,"unsafe":22,"review":0,"automated_count":43,"automated_acceptable":15,"automated_unsafe":22,"unnecessary_reviews":0,"accuracy":0.3488372093023256,"unsafe_rate":0.5116279069767442,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.3488372093023256,"conditional_unsafe":0.5116279069767442,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.22418589997352706,"upper":0.49828218007147956,"confidence_level":0.95}},"effective":{"n":43,"acceptable":5,"unsafe":0,"review":43,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":38,"accuracy":0.11627906976744186,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":0.8837209302325582,"accuracy_wilson_95":{"lower":0.05070427739804166,"upper":0.24479161810082128,"confidence_level":0.95}},"confusion":{"raw":{"keep":{"keep":9,"drop":3,"mask":0,"pseudonymize":0,"generalize":0,"review":0},"drop":{"keep":11,"drop":4,"mask":0,"pseudonymize":1,"generalize":0,"review":0},"mask":{"keep":1,"drop":1,"mask":0,"pseudonymize":0,"generalize":0,"review":0},"pseudonymize":{"keep":2,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0},"generalize":{"keep":4,"drop":2,"mask":0,"pseudonymize":0,"generalize":0,"review":0},"review":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0}},"effective":{"keep":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":12},"drop":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":16},"mask":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":2},"pseudonymize":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":2},"generalize":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":6},"review":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0}}},"probability_metrics":{"singleton_n":38,"multiacceptable_n":5,"multiacceptable_coverage":0.11627906976744186,"nll":1.6017828255213327,"nll_zero_mass_n":0,"brier":0.7712232930510665,"acceptable_set_n":43,"acceptable_set_nll":1.499132748720719,"acceptable_set_zero_mass_n":0},"calibration_all_labels":{"equal_width":{"n":43,"bin_count":10,"ece":0.13921012504156247,"signed_mean_gap":-0.10688431803570238,"absolute_mean_gap":0.10688431803570238,"bins":[{"index":0,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.0,"upper":0.1,"upper_inclusive":false},{"index":1,"count":2,"confidence":0.19692955911159515,"empirical_correctness":0.0,"signed_gap":0.19692955911159515,"absolute_gap":0.19692955911159515,"lower":0.1,"upper":0.2,"upper_inclusive":false},{"index":2,"count":40,"confidence":0.24272423684597016,"empirical_correctness":0.375,"signed_gap":-0.13227576315402984,"absolute_gap":0.13227576315402984,"lower":0.2,"upper":0.3,"upper_inclusive":false},{"index":3,"count":1,"confidence":0.3011457324028015,"empirical_correctness":0.0,"signed_gap":0.3011457324028015,"absolute_gap":0.3011457324028015,"lower":0.3,"upper":0.4,"upper_inclusive":false},{"index":4,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.4,"upper":0.5,"upper_inclusive":false},{"index":5,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.5,"upper":0.6,"upper_inclusive":false},{"index":6,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.6,"upper":0.7,"upper_inclusive":false},{"index":7,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.7,"upper":0.8,"upper_inclusive":false},{"index":8,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.8,"upper":0.9,"upper_inclusive":false},{"index":9,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.9,"upper":1.0,"upper_inclusive":true}]},"equal_mass":{"n":43,"bin_count":10,"ece":0.16733616659807604,"signed_mean_gap":-0.10688431803570238,"absolute_mean_gap":0.10688431803570238,"bins":[{"index":0,"count":4,"confidence":0.1990361213684082,"empirical_correctness":0.25,"signed_gap":-0.0509638786315918,"absolute_gap":0.0509638786315918,"min_confidence":0.19471320509910583,"max_confidence":0.2020755112171173},{"index":1,"count":4,"confidence":0.20565564185380936,"empirical_correctness":0.25,"signed_gap":-0.04434435814619064,"absolute_gap":0.04434435814619064,"min_confidence":0.20232455432415009,"max_confidence":0.20999589562416077},{"index":2,"count":4,"confidence":0.22957255691289902,"empirical_correctness":0.25,"signed_gap":-0.020427443087100983,"absolute_gap":0.020427443087100983,"min_confidence":0.22881552577018738,"max_confidence":0.23020324110984802},{"index":3,"count":5,"confidence":0.23656120598316194,"empirical_correctness":0.6,"signed_gap":-0.36343879401683804,"absolute_gap":0.36343879401683804,"min_confidence":0.23087376356124878,"max_confidence":0.24163782596588135},{"index":4,"count":4,"confidence":0.2428310364484787,"empirical_correctness":0.5,"signed_gap":-0.2571689635515213,"absolute_gap":0.2571689635515213,"min_confidence":0.2424514889717102,"max_confidence":0.24344448745250702},{"index":5,"count":4,"confidence":0.24557023867964745,"empirical_correctness":0.5,"signed_gap":-0.25442976132035255,"absolute_gap":0.25442976132035255,"min_confidence":0.24462805688381195,"max_confidence":0.2465275228023529},{"index":6,"count":5,"confidence":0.248014634847641,"empirical_correctness":0.2,"signed_gap":0.048014634847640986,"absolute_gap":0.048014634847640986,"min_confidence":0.24685880541801453,"max_confidence":0.24944598972797394},{"index":7,"count":4,"confidence":0.25179116427898407,"empirical_correctness":0.5,"signed_gap":-0.24820883572101593,"absolute_gap":0.24820883572101593,"min_confidence":0.24977463483810425,"max_confidence":0.2540486752986908},{"index":8,"count":4,"confidence":0.26491039246320724,"empirical_correctness":0.0,"signed_gap":0.26491039246320724,"absolute_gap":0.26491039246320724,"min_confidence":0.258959025144577,"max_confidence":0.27066829800605774},{"index":9,"count":5,"confidence":0.28472530245780947,"empirical_correctness":0.4,"signed_gap":-0.11527469754219055,"absolute_gap":0.11527469754219055,"min_confidence":0.27593880891799927,"max_confidence":0.3011457324028015}]}},"calibration_singleton_labels":{"equal_width":{"n":38,"bin_count":10,"ece":0.13645128709705254,"signed_mean_gap":-0.09987208443252664,"absolute_mean_gap":0.09987208443252664,"bins":[{"index":0,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.0,"upper":0.1,"upper_inclusive":false},{"index":1,"count":2,"confidence":0.19692955911159515,"empirical_correctness":0.0,"signed_gap":0.19692955911159515,"absolute_gap":0.19692955911159515,"lower":0.1,"upper":0.2,"upper_inclusive":false},{"index":2,"count":35,"confidence":0.24313874116965703,"empirical_correctness":0.37142857142857144,"signed_gap":-0.1282898302589144,"absolute_gap":0.1282898302589144,"lower":0.2,"upper":0.3,"upper_inclusive":false},{"index":3,"count":1,"confidence":0.3011457324028015,"empirical_correctness":0.0,"signed_gap":0.3011457324028015,"absolute_gap":0.3011457324028015,"lower":0.3,"upper":0.4,"upper_inclusive":false},{"index":4,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.4,"upper":0.5,"upper_inclusive":false},{"index":5,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.5,"upper":0.6,"upper_inclusive":false},{"index":6,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.6,"upper":0.7,"upper_inclusive":false},{"index":7,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.7,"upper":0.8,"upper_inclusive":false},{"index":8,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.8,"upper":0.9,"upper_inclusive":false},{"index":9,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.9,"upper":1.0,"upper_inclusive":true}]},"equal_mass":{"n":38,"bin_count":10,"ece":0.14304280477134804,"signed_mean_gap":-0.09987208443252664,"absolute_mean_gap":0.09987208443252664,"bins":[{"index":0,"count":3,"confidence":0.19864487648010254,"empirical_correctness":0.0,"signed_gap":0.19864487648010254,"absolute_gap":0.19864487648010254,"min_confidence":0.19471320509910583,"max_confidence":0.2020755112171173},{"index":1,"count":4,"confidence":0.20565564185380936,"empirical_correctness":0.25,"signed_gap":-0.04434435814619064,"absolute_gap":0.04434435814619064,"min_confidence":0.20232455432415009,"max_confidence":0.20999589562416077},{"index":2,"count":4,"confidence":0.22957255691289902,"empirical_correctness":0.25,"signed_gap":-0.020427443087100983,"absolute_gap":0.020427443087100983,"min_confidence":0.22881552577018738,"max_confidence":0.23020324110984802},{"index":3,"count":4,"confidence":0.23529205098748207,"empirical_correctness":0.5,"signed_gap":-0.26470794901251793,"absolute_gap":0.26470794901251793,"min_confidence":0.23087376356124878,"max_confidence":0.2385055422782898},{"index":4,"count":4,"confidence":0.24292351305484772,"empirical_correctness":0.5,"signed_gap":-0.2570764869451523,"absolute_gap":0.2570764869451523,"min_confidence":0.24163782596588135,"max_confidence":0.24462805688381195},{"index":5,"count":3,"confidence":0.24588429927825928,"empirical_correctness":0.6666666666666666,"signed_gap":-0.42078236738840735,"absolute_gap":0.42078236738840735,"min_confidence":0.2454376071691513,"max_confidence":0.2465275228023529},{"index":6,"count":4,"confidence":0.24765679612755775,"empirical_correctness":0.25,"signed_gap":-0.0023432038724422455,"absolute_gap":0.0023432038724422455,"min_confidence":0.24685880541801453,"max_confidence":0.24857568740844727},{"index":7,"count":4,"confidence":0.25064049288630486,"empirical_correctness":0.5,"signed_gap":-0.24935950711369514,"absolute_gap":0.24935950711369514,"min_confidence":0.24944598972797394,"max_confidence":0.2532183825969696},{"index":8,"count":4,"confidence":0.2691553384065628,"empirical_correctness":0.25,"signed_gap":0.019155338406562805,"absolute_gap":0.019155338406562805,"min_confidence":0.2625395357608795,"max_confidence":0.27593880891799927},{"index":9,"count":4,"confidence":0.286921925842762,"empirical_correctness":0.25,"signed_gap":0.03692192584276199,"absolute_gap":0.03692192584276199,"min_confidence":0.28064608573913574,"max_confidence":0.3011457324028015}]}},"latency_ms":{"n":43,"median":20.76755801681429,"p95":21.026882994920015},"per_action":{"keep":{"n":12,"raw":{"n":12,"acceptable":9,"unsafe":0,"review":0,"automated_count":12,"automated_acceptable":9,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":0.75,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.75,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.46769466506643426,"upper":0.9110583316059453,"confidence_level":0.95}},"effective":{"n":12,"acceptable":0,"unsafe":0,"review":12,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":12,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":2.7755575615628914e-17,"upper":0.24249400665524085,"confidence_level":0.95}}},"drop":{"n":16,"raw":{"n":16,"acceptable":4,"unsafe":12,"review":0,"automated_count":16,"automated_acceptable":4,"automated_unsafe":12,"unnecessary_reviews":0,"accuracy":0.25,"unsafe_rate":0.75,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.25,"conditional_unsafe":0.75,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.10182067491213048,"upper":0.49498316535508774,"confidence_level":0.95}},"effective":{"n":16,"acceptable":0,"unsafe":0,"review":16,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":16,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.1936076805344365,"confidence_level":0.95}}},"mask":{"n":2,"raw":{"n":2,"acceptable":0,"unsafe":1,"review":0,"automated_count":2,"automated_acceptable":0,"automated_unsafe":1,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":0.5,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":0.5,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.6576197724933469,"confidence_level":0.95}},"effective":{"n":2,"acceptable":0,"unsafe":0,"review":2,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":2,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.6576197724933469,"confidence_level":0.95}}},"pseudonymize":{"n":2,"raw":{"n":2,"acceptable":0,"unsafe":2,"review":0,"automated_count":2,"automated_acceptable":0,"automated_unsafe":2,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":1.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":1.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.6576197724933469,"confidence_level":0.95}},"effective":{"n":2,"acceptable":0,"unsafe":0,"review":2,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":2,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.6576197724933469,"confidence_level":0.95}}},"generalize":{"n":6,"raw":{"n":6,"acceptable":0,"unsafe":4,"review":0,"automated_count":6,"automated_acceptable":0,"automated_unsafe":4,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":0.6666666666666666,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":0.6666666666666666,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":2.7755575615628914e-17,"upper":0.3903342879021653,"confidence_level":0.95}},"effective":{"n":6,"acceptable":0,"unsafe":0,"review":6,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":6,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":2.7755575615628914e-17,"upper":0.3903342879021653,"confidence_level":0.95}}},"review":{"n":0,"raw":{"n":0,"acceptable":0,"unsafe":0,"review":0,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":null,"unsafe_rate":null,"review_rate":null,"automated_coverage":null,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":null},"effective":{"n":0,"acceptable":0,"unsafe":0,"review":0,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":null,"unsafe_rate":null,"review_rate":null,"automated_coverage":null,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":null}}}},"challenge56-no_samples":{"n":56,"raw":{"n":56,"acceptable":29,"unsafe":27,"review":0,"automated_count":56,"automated_acceptable":29,"automated_unsafe":27,"unnecessary_reviews":0,"accuracy":0.5178571428571429,"unsafe_rate":0.48214285714285715,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.5178571428571429,"conditional_unsafe":0.48214285714285715,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.39010383924176983,"upper":0.6433178058754173,"confidence_level":0.95}},"effective":{"n":56,"acceptable":2,"unsafe":0,"review":56,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":54,"accuracy":0.03571428571428571,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":0.9642857142857143,"accuracy_wilson_95":{"lower":0.00984942514823639,"upper":0.12118780180490107,"confidence_level":0.95}},"confusion":{"raw":{"keep":{"keep":8,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0},"drop":{"keep":5,"drop":9,"mask":0,"pseudonymize":8,"generalize":0,"review":0},"mask":{"keep":2,"drop":0,"mask":6,"pseudonymize":0,"generalize":0,"review":0},"pseudonymize":{"keep":2,"drop":0,"mask":0,"pseudonymize":6,"generalize":0,"review":0},"generalize":{"keep":8,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0},"review":{"keep":2,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0}},"effective":{"keep":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":8},"drop":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":22},"mask":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":8},"pseudonymize":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":8},"generalize":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":8},"review":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":2}}},"probability_metrics":{"singleton_n":56,"multiacceptable_n":0,"multiacceptable_coverage":0.0,"nll":1.6253242132205437,"nll_zero_mass_n":0,"brier":0.7746616910153966,"acceptable_set_n":56,"acceptable_set_nll":1.6253242132205437,"acceptable_set_zero_mass_n":0},"calibration_all_labels":{"equal_width":{"n":56,"bin_count":10,"ece":0.2992267914648567,"signed_mean_gap":-0.2992267914648567,"absolute_mean_gap":0.2992267914648567,"bins":[{"index":0,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.0,"upper":0.1,"upper_inclusive":false},{"index":1,"count":9,"confidence":0.1903971251514223,"empirical_correctness":0.6666666666666666,"signed_gap":-0.47626954151524437,"absolute_gap":0.47626954151524437,"lower":0.1,"upper":0.2,"upper_inclusive":false},{"index":2,"count":47,"confidence":0.22403671386394095,"empirical_correctness":0.48936170212765956,"signed_gap":-0.26532498826371864,"absolute_gap":0.26532498826371864,"lower":0.2,"upper":0.3,"upper_inclusive":false},{"index":3,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.3,"upper":0.4,"upper_inclusive":false},{"index":4,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.4,"upper":0.5,"upper_inclusive":false},{"index":5,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.5,"upper":0.6,"upper_inclusive":false},{"index":6,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.6,"upper":0.7,"upper_inclusive":false},{"index":7,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.7,"upper":0.8,"upper_inclusive":false},{"index":8,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.8,"upper":0.9,"upper_inclusive":false},{"index":9,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.9,"upper":1.0,"upper_inclusive":true}]},"equal_mass":{"n":56,"bin_count":10,"ece":0.30031945795885157,"signed_mean_gap":-0.2992267914648567,"absolute_mean_gap":0.2992267914648567,"bins":[{"index":0,"count":5,"confidence":0.18551811277866365,"empirical_correctness":0.6,"signed_gap":-0.41448188722133633,"absolute_gap":0.41448188722133633,"min_confidence":0.18364417552947998,"max_confidence":0.18761847913265228},{"index":1,"count":6,"confidence":0.19820550084114075,"empirical_correctness":0.5,"signed_gap":-0.30179449915885925,"absolute_gap":0.30179449915885925,"min_confidence":0.19352537393569946,"max_confidence":0.2017897069454193},{"index":2,"count":5,"confidence":0.20611893236637116,"empirical_correctness":0.2,"signed_gap":0.006118932366371149,"absolute_gap":0.006118932366371149,"min_confidence":0.20479711890220642,"max_confidence":0.20712602138519287},{"index":3,"count":6,"confidence":0.2084420919418335,"empirical_correctness":0.3333333333333333,"signed_gap":-0.12489124139149982,"absolute_gap":0.12489124139149982,"min_confidence":0.20753902196884155,"max_confidence":0.21010540425777435},{"index":4,"count":6,"confidence":0.2139977067708969,"empirical_correctness":0.6666666666666666,"signed_gap":-0.4526689598957697,"absolute_gap":0.4526689598957697,"min_confidence":0.21136823296546936,"max_confidence":0.21529819071292877},{"index":5,"count":5,"confidence":0.21707605421543122,"empirical_correctness":0.8,"signed_gap":-0.5829239457845689,"absolute_gap":0.5829239457845689,"min_confidence":0.21554239094257355,"max_confidence":0.21855804324150085},{"index":6,"count":6,"confidence":0.2233751912911733,"empirical_correctness":0.3333333333333333,"signed_gap":-0.10995814204216,"absolute_gap":0.10995814204216,"min_confidence":0.22101300954818726,"max_confidence":0.22523310780525208},{"index":7,"count":5,"confidence":0.2311131775379181,"empirical_correctness":0.6,"signed_gap":-0.3688868224620819,"absolute_gap":0.3688868224620819,"min_confidence":0.2276143878698349,"max_confidence":0.23364382982254028},{"index":8,"count":6,"confidence":0.23720892518758774,"empirical_correctness":0.5,"signed_gap":-0.26279107481241226,"absolute_gap":0.26279107481241226,"min_confidence":0.23383557796478271,"max_confidence":0.24247576296329498},{"index":9,"count":6,"confidence":0.2594652995467186,"empirical_correctness":0.6666666666666666,"signed_gap":-0.40720136711994803,"absolute_gap":0.40720136711994803,"min_confidence":0.24373666942119598,"max_confidence":0.28285160660743713}]}},"calibration_singleton_labels":{"equal_width":{"n":56,"bin_count":10,"ece":0.2992267914648567,"signed_mean_gap":-0.2992267914648567,"absolute_mean_gap":0.2992267914648567,"bins":[{"index":0,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.0,"upper":0.1,"upper_inclusive":false},{"index":1,"count":9,"confidence":0.1903971251514223,"empirical_correctness":0.6666666666666666,"signed_gap":-0.47626954151524437,"absolute_gap":0.47626954151524437,"lower":0.1,"upper":0.2,"upper_inclusive":false},{"index":2,"count":47,"confidence":0.22403671386394095,"empirical_correctness":0.48936170212765956,"signed_gap":-0.26532498826371864,"absolute_gap":0.26532498826371864,"lower":0.2,"upper":0.3,"upper_inclusive":false},{"index":3,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.3,"upper":0.4,"upper_inclusive":false},{"index":4,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.4,"upper":0.5,"upper_inclusive":false},{"index":5,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.5,"upper":0.6,"upper_inclusive":false},{"index":6,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.6,"upper":0.7,"upper_inclusive":false},{"index":7,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.7,"upper":0.8,"upper_inclusive":false},{"index":8,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.8,"upper":0.9,"upper_inclusive":false},{"index":9,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.9,"upper":1.0,"upper_inclusive":true}]},"equal_mass":{"n":56,"bin_count":10,"ece":0.30031945795885157,"signed_mean_gap":-0.2992267914648567,"absolute_mean_gap":0.2992267914648567,"bins":[{"index":0,"count":5,"confidence":0.18551811277866365,"empirical_correctness":0.6,"signed_gap":-0.41448188722133633,"absolute_gap":0.41448188722133633,"min_confidence":0.18364417552947998,"max_confidence":0.18761847913265228},{"index":1,"count":6,"confidence":0.19820550084114075,"empirical_correctness":0.5,"signed_gap":-0.30179449915885925,"absolute_gap":0.30179449915885925,"min_confidence":0.19352537393569946,"max_confidence":0.2017897069454193},{"index":2,"count":5,"confidence":0.20611893236637116,"empirical_correctness":0.2,"signed_gap":0.006118932366371149,"absolute_gap":0.006118932366371149,"min_confidence":0.20479711890220642,"max_confidence":0.20712602138519287},{"index":3,"count":6,"confidence":0.2084420919418335,"empirical_correctness":0.3333333333333333,"signed_gap":-0.12489124139149982,"absolute_gap":0.12489124139149982,"min_confidence":0.20753902196884155,"max_confidence":0.21010540425777435},{"index":4,"count":6,"confidence":0.2139977067708969,"empirical_correctness":0.6666666666666666,"signed_gap":-0.4526689598957697,"absolute_gap":0.4526689598957697,"min_confidence":0.21136823296546936,"max_confidence":0.21529819071292877},{"index":5,"count":5,"confidence":0.21707605421543122,"empirical_correctness":0.8,"signed_gap":-0.5829239457845689,"absolute_gap":0.5829239457845689,"min_confidence":0.21554239094257355,"max_confidence":0.21855804324150085},{"index":6,"count":6,"confidence":0.2233751912911733,"empirical_correctness":0.3333333333333333,"signed_gap":-0.10995814204216,"absolute_gap":0.10995814204216,"min_confidence":0.22101300954818726,"max_confidence":0.22523310780525208},{"index":7,"count":5,"confidence":0.2311131775379181,"empirical_correctness":0.6,"signed_gap":-0.3688868224620819,"absolute_gap":0.3688868224620819,"min_confidence":0.2276143878698349,"max_confidence":0.23364382982254028},{"index":8,"count":6,"confidence":0.23720892518758774,"empirical_correctness":0.5,"signed_gap":-0.26279107481241226,"absolute_gap":0.26279107481241226,"min_confidence":0.23383557796478271,"max_confidence":0.24247576296329498},{"index":9,"count":6,"confidence":0.2594652995467186,"empirical_correctness":0.6666666666666666,"signed_gap":-0.40720136711994803,"absolute_gap":0.40720136711994803,"min_confidence":0.24373666942119598,"max_confidence":0.28285160660743713}]}},"latency_ms":{"n":56,"median":15.519527980359271,"p95":15.930881738313474},"per_action":{"keep":{"n":8,"raw":{"n":8,"acceptable":8,"unsafe":0,"review":0,"automated_count":8,"automated_acceptable":8,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":1.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.6755924351161198,"upper":1.0,"confidence_level":0.95}},"effective":{"n":8,"acceptable":0,"unsafe":0,"review":8,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":8,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.32440756488388023,"confidence_level":0.95}}},"drop":{"n":22,"raw":{"n":22,"acceptable":9,"unsafe":13,"review":0,"automated_count":22,"automated_acceptable":9,"automated_unsafe":13,"unnecessary_reviews":0,"accuracy":0.4090909090909091,"unsafe_rate":0.5909090909090909,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.4090909090909091,"conditional_unsafe":0.5909090909090909,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.2325581619647187,"upper":0.6126518152368488,"confidence_level":0.95}},"effective":{"n":22,"acceptable":0,"unsafe":0,"review":22,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":22,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.14865487460862087,"confidence_level":0.95}}},"mask":{"n":8,"raw":{"n":8,"acceptable":6,"unsafe":2,"review":0,"automated_count":8,"automated_acceptable":6,"automated_unsafe":2,"unnecessary_reviews":0,"accuracy":0.75,"unsafe_rate":0.25,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.75,"conditional_unsafe":0.25,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.40927543031016883,"upper":0.9285207872478909,"confidence_level":0.95}},"effective":{"n":8,"acceptable":0,"unsafe":0,"review":8,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":8,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.32440756488388023,"confidence_level":0.95}}},"pseudonymize":{"n":8,"raw":{"n":8,"acceptable":6,"unsafe":2,"review":0,"automated_count":8,"automated_acceptable":6,"automated_unsafe":2,"unnecessary_reviews":0,"accuracy":0.75,"unsafe_rate":0.25,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.75,"conditional_unsafe":0.25,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.40927543031016883,"upper":0.9285207872478909,"confidence_level":0.95}},"effective":{"n":8,"acceptable":0,"unsafe":0,"review":8,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":8,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.32440756488388023,"confidence_level":0.95}}},"generalize":{"n":8,"raw":{"n":8,"acceptable":0,"unsafe":8,"review":0,"automated_count":8,"automated_acceptable":0,"automated_unsafe":8,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":1.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":1.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.32440756488388023,"confidence_level":0.95}},"effective":{"n":8,"acceptable":0,"unsafe":0,"review":8,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":8,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.32440756488388023,"confidence_level":0.95}}},"review":{"n":2,"raw":{"n":2,"acceptable":0,"unsafe":2,"review":0,"automated_count":2,"automated_acceptable":0,"automated_unsafe":2,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":1.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":1.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.6576197724933469,"confidence_level":0.95}},"effective":{"n":2,"acceptable":2,"unsafe":0,"review":2,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":1.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.34238022750665303,"upper":1.0,"confidence_level":0.95}}}}},"challenge56-compact_no_samples":{"n":56,"raw":{"n":56,"acceptable":16,"unsafe":39,"review":0,"automated_count":56,"automated_acceptable":16,"automated_unsafe":39,"unnecessary_reviews":0,"accuracy":0.2857142857142857,"unsafe_rate":0.6964285714285714,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.2857142857142857,"conditional_unsafe":0.6964285714285714,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.18418775143858906,"upper":0.4147525071551667,"confidence_level":0.95}},"effective":{"n":56,"acceptable":2,"unsafe":0,"review":56,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":54,"accuracy":0.03571428571428571,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":0.9642857142857143,"accuracy_wilson_95":{"lower":0.00984942514823639,"upper":0.12118780180490107,"confidence_level":0.95}},"confusion":{"raw":{"keep":{"keep":8,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0},"drop":{"keep":12,"drop":8,"mask":0,"pseudonymize":2,"generalize":0,"review":0},"mask":{"keep":7,"drop":0,"mask":0,"pseudonymize":1,"generalize":0,"review":0},"pseudonymize":{"keep":8,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0},"generalize":{"keep":7,"drop":0,"mask":0,"pseudonymize":1,"generalize":0,"review":0},"review":{"keep":2,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0}},"effective":{"keep":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":8},"drop":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":22},"mask":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":8},"pseudonymize":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":8},"generalize":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":8},"review":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":2}}},"probability_metrics":{"singleton_n":56,"multiacceptable_n":0,"multiacceptable_coverage":0.0,"nll":1.6457436058456791,"nll_zero_mass_n":0,"brier":0.7818289812015305,"acceptable_set_n":56,"acceptable_set_nll":1.6457436058456791,"acceptable_set_zero_mass_n":0},"calibration_all_labels":{"equal_width":{"n":56,"bin_count":10,"ece":0.10515293053218297,"signed_mean_gap":-0.06396252023322242,"absolute_mean_gap":0.06396252023322242,"bins":[{"index":0,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.0,"upper":0.1,"upper_inclusive":false},{"index":1,"count":6,"confidence":0.19222191472848257,"empirical_correctness":0.0,"signed_gap":0.19222191472848257,"absolute_gap":0.19222191472848257,"lower":0.1,"upper":0.2,"upper_inclusive":false},{"index":2,"count":50,"confidence":0.22529534757137298,"empirical_correctness":0.32,"signed_gap":-0.09470465242862702,"absolute_gap":0.09470465242862702,"lower":0.2,"upper":0.3,"upper_inclusive":false},{"index":3,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.3,"upper":0.4,"upper_inclusive":false},{"index":4,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.4,"upper":0.5,"upper_inclusive":false},{"index":5,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.5,"upper":0.6,"upper_inclusive":false},{"index":6,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.6,"upper":0.7,"upper_inclusive":false},{"index":7,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.7,"upper":0.8,"upper_inclusive":false},{"index":8,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.8,"upper":0.9,"upper_inclusive":false},{"index":9,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.9,"upper":1.0,"upper_inclusive":true}]},"equal_mass":{"n":56,"bin_count":10,"ece":0.2496103873210294,"signed_mean_gap":-0.06396252023322242,"absolute_mean_gap":0.06396252023322242,"bins":[{"index":0,"count":5,"confidence":0.1912054032087326,"empirical_correctness":0.0,"signed_gap":0.1912054032087326,"absolute_gap":0.1912054032087326,"min_confidence":0.1860254555940628,"max_confidence":0.19484871625900269},{"index":1,"count":6,"confidence":0.2020264739791552,"empirical_correctness":0.16666666666666666,"signed_gap":0.035359807312488556,"absolute_gap":0.035359807312488556,"min_confidence":0.19730447232723236,"max_confidence":0.20551185309886932},{"index":2,"count":5,"confidence":0.20684250295162201,"empirical_correctness":0.0,"signed_gap":0.20684250295162201,"absolute_gap":0.20684250295162201,"min_confidence":0.2058405578136444,"max_confidence":0.2080131471157074},{"index":3,"count":6,"confidence":0.20909264187018076,"empirical_correctness":0.0,"signed_gap":0.20909264187018076,"absolute_gap":0.20909264187018076,"min_confidence":0.20814694464206696,"max_confidence":0.21133966743946075},{"index":4,"count":6,"confidence":0.2129250094294548,"empirical_correctness":0.0,"signed_gap":0.2129250094294548,"absolute_gap":0.2129250094294548,"min_confidence":0.21147115528583527,"max_confidence":0.21648521721363068},{"index":5,"count":5,"confidence":0.21998005211353303,"empirical_correctness":0.2,"signed_gap":0.019980052113533014,"absolute_gap":0.019980052113533014,"min_confidence":0.2186390906572342,"max_confidence":0.2224094122648239},{"index":6,"count":6,"confidence":0.22728928923606873,"empirical_correctness":0.16666666666666666,"signed_gap":0.06062262256940207,"absolute_gap":0.06062262256940207,"min_confidence":0.22354236245155334,"max_confidence":0.23173083364963531},{"index":7,"count":5,"confidence":0.23436197638511658,"empirical_correctness":0.4,"signed_gap":-0.16563802361488345,"absolute_gap":0.16563802361488345,"min_confidence":0.23335395753383636,"max_confidence":0.23626211285591125},{"index":8,"count":6,"confidence":0.24426117291053137,"empirical_correctness":0.8333333333333334,"signed_gap":-0.589072160422802,"absolute_gap":0.589072160422802,"min_confidence":0.23823468387126923,"max_confidence":0.25450047850608826},{"index":9,"count":6,"confidence":0.26376361151536304,"empirical_correctness":1.0,"signed_gap":-0.736236388484637,"absolute_gap":0.736236388484637,"min_confidence":0.2572636008262634,"max_confidence":0.2679406702518463}]}},"calibration_singleton_labels":{"equal_width":{"n":56,"bin_count":10,"ece":0.10515293053218297,"signed_mean_gap":-0.06396252023322242,"absolute_mean_gap":0.06396252023322242,"bins":[{"index":0,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.0,"upper":0.1,"upper_inclusive":false},{"index":1,"count":6,"confidence":0.19222191472848257,"empirical_correctness":0.0,"signed_gap":0.19222191472848257,"absolute_gap":0.19222191472848257,"lower":0.1,"upper":0.2,"upper_inclusive":false},{"index":2,"count":50,"confidence":0.22529534757137298,"empirical_correctness":0.32,"signed_gap":-0.09470465242862702,"absolute_gap":0.09470465242862702,"lower":0.2,"upper":0.3,"upper_inclusive":false},{"index":3,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.3,"upper":0.4,"upper_inclusive":false},{"index":4,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.4,"upper":0.5,"upper_inclusive":false},{"index":5,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.5,"upper":0.6,"upper_inclusive":false},{"index":6,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.6,"upper":0.7,"upper_inclusive":false},{"index":7,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.7,"upper":0.8,"upper_inclusive":false},{"index":8,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.8,"upper":0.9,"upper_inclusive":false},{"index":9,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.9,"upper":1.0,"upper_inclusive":true}]},"equal_mass":{"n":56,"bin_count":10,"ece":0.2496103873210294,"signed_mean_gap":-0.06396252023322242,"absolute_mean_gap":0.06396252023322242,"bins":[{"index":0,"count":5,"confidence":0.1912054032087326,"empirical_correctness":0.0,"signed_gap":0.1912054032087326,"absolute_gap":0.1912054032087326,"min_confidence":0.1860254555940628,"max_confidence":0.19484871625900269},{"index":1,"count":6,"confidence":0.2020264739791552,"empirical_correctness":0.16666666666666666,"signed_gap":0.035359807312488556,"absolute_gap":0.035359807312488556,"min_confidence":0.19730447232723236,"max_confidence":0.20551185309886932},{"index":2,"count":5,"confidence":0.20684250295162201,"empirical_correctness":0.0,"signed_gap":0.20684250295162201,"absolute_gap":0.20684250295162201,"min_confidence":0.2058405578136444,"max_confidence":0.2080131471157074},{"index":3,"count":6,"confidence":0.20909264187018076,"empirical_correctness":0.0,"signed_gap":0.20909264187018076,"absolute_gap":0.20909264187018076,"min_confidence":0.20814694464206696,"max_confidence":0.21133966743946075},{"index":4,"count":6,"confidence":0.2129250094294548,"empirical_correctness":0.0,"signed_gap":0.2129250094294548,"absolute_gap":0.2129250094294548,"min_confidence":0.21147115528583527,"max_confidence":0.21648521721363068},{"index":5,"count":5,"confidence":0.21998005211353303,"empirical_correctness":0.2,"signed_gap":0.019980052113533014,"absolute_gap":0.019980052113533014,"min_confidence":0.2186390906572342,"max_confidence":0.2224094122648239},{"index":6,"count":6,"confidence":0.22728928923606873,"empirical_correctness":0.16666666666666666,"signed_gap":0.06062262256940207,"absolute_gap":0.06062262256940207,"min_confidence":0.22354236245155334,"max_confidence":0.23173083364963531},{"index":7,"count":5,"confidence":0.23436197638511658,"empirical_correctness":0.4,"signed_gap":-0.16563802361488345,"absolute_gap":0.16563802361488345,"min_confidence":0.23335395753383636,"max_confidence":0.23626211285591125},{"index":8,"count":6,"confidence":0.24426117291053137,"empirical_correctness":0.8333333333333334,"signed_gap":-0.589072160422802,"absolute_gap":0.589072160422802,"min_confidence":0.23823468387126923,"max_confidence":0.25450047850608826},{"index":9,"count":6,"confidence":0.26376361151536304,"empirical_correctness":1.0,"signed_gap":-0.736236388484637,"absolute_gap":0.736236388484637,"min_confidence":0.2572636008262634,"max_confidence":0.2679406702518463}]}},"latency_ms":{"n":56,"median":18.06340299663134,"p95":18.437158985761926},"per_action":{"keep":{"n":8,"raw":{"n":8,"acceptable":8,"unsafe":0,"review":0,"automated_count":8,"automated_acceptable":8,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":1.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.6755924351161198,"upper":1.0,"confidence_level":0.95}},"effective":{"n":8,"acceptable":0,"unsafe":0,"review":8,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":8,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.32440756488388023,"confidence_level":0.95}}},"drop":{"n":22,"raw":{"n":22,"acceptable":8,"unsafe":14,"review":0,"automated_count":22,"automated_acceptable":8,"automated_unsafe":14,"unnecessary_reviews":0,"accuracy":0.36363636363636365,"unsafe_rate":0.6363636363636364,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.36363636363636365,"conditional_unsafe":0.6363636363636364,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.19733204670366125,"upper":0.5704829190986899,"confidence_level":0.95}},"effective":{"n":22,"acceptable":0,"unsafe":0,"review":22,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":22,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.14865487460862087,"confidence_level":0.95}}},"mask":{"n":8,"raw":{"n":8,"acceptable":0,"unsafe":8,"review":0,"automated_count":8,"automated_acceptable":0,"automated_unsafe":8,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":1.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":1.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.32440756488388023,"confidence_level":0.95}},"effective":{"n":8,"acceptable":0,"unsafe":0,"review":8,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":8,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.32440756488388023,"confidence_level":0.95}}},"pseudonymize":{"n":8,"raw":{"n":8,"acceptable":0,"unsafe":8,"review":0,"automated_count":8,"automated_acceptable":0,"automated_unsafe":8,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":1.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":1.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.32440756488388023,"confidence_level":0.95}},"effective":{"n":8,"acceptable":0,"unsafe":0,"review":8,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":8,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.32440756488388023,"confidence_level":0.95}}},"generalize":{"n":8,"raw":{"n":8,"acceptable":0,"unsafe":7,"review":0,"automated_count":8,"automated_acceptable":0,"automated_unsafe":7,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":0.875,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":0.875,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.32440756488388023,"confidence_level":0.95}},"effective":{"n":8,"acceptable":0,"unsafe":0,"review":8,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":8,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.32440756488388023,"confidence_level":0.95}}},"review":{"n":2,"raw":{"n":2,"acceptable":0,"unsafe":2,"review":0,"automated_count":2,"automated_acceptable":0,"automated_unsafe":2,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":1.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":1.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.6576197724933469,"confidence_level":0.95}},"effective":{"n":2,"acceptable":2,"unsafe":0,"review":2,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":1.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.34238022750665303,"upper":1.0,"confidence_level":0.95}}}}}},"composite":{"n":198,"scope":"descriptive sum across four correlated panels","raw":{"n":198,"acceptable":73,"unsafe":117,"review":0,"automated_count":198,"automated_acceptable":73,"automated_unsafe":117,"unnecessary_reviews":0,"accuracy":0.3686868686868687,"unsafe_rate":0.5909090909090909,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.3686868686868687,"conditional_unsafe":0.5909090909090909,"unnecessary_review_rate":0.0},"effective":{"n":198,"acceptable":14,"unsafe":0,"review":195,"automated_count":3,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":181,"accuracy":0.0707070707070707,"unsafe_rate":0.0,"review_rate":0.9848484848484849,"automated_coverage":0.015151515151515152,"conditional_correctness":0.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.9141414141414141}}},{"id":"11wanqi--laya-command-safety-ft-fp32","platform":"Local","model_id":"11wanqi/laya-command-safety-ft","revision":"a9f42a0b877fd33b849ab1ed32839e7322fefe03","loading":"float32 / none","scoring_kind":"Native decision readout","settings":{"dtype":"float32","quantization":"none","max_tokens":1024,"head_max_tokens":256,"temperature":1.2575325965881348,"autocast":false,"truncation":false,"cpu_offload":false,"guard_threshold":0.45,"seed":0,"tf32":false},"scoring_metadata":{"backend":"laya_native_local_builtin","readout":"native_mask_slot_scorer_numpy_float32_softmax","probability_source":"native_choice_unrounded_published_temperature_not_task_calibrated","latency_scope":"synchronized serial complete encoder+head+act forward,input transfer and native readout; tokenization excluded","native_disclosures":["Stock installed torch attention in place of ONNX dynamic equivalent; no custom kernels or repo imports.","FP32/no-autocast explicit precision: convert every saved F16/BF16/F32 tensor toFP32; no quantization.","Unrounded native probability; guard uses winner p, not entropy-concentration confidence.","Act head fully loaded and evaluated; not used to alter six-action answer.","Native choice:6-10 bucket else choice scalar0, T=1.2575325965881348. No task fitting, priors correction, prompt change or finetuning here."],"equal_mass_convention":"rank bins with floor boundaries at floor(i*n/k), k=min(10,n//2); minimum two rows"},"load_ms":null,"peak_allocated_bytes":1363195392,"peak_reserved_bytes":1396703232,"panels":{"diagnostic43-no_samples":{"n":43,"raw":{"n":43,"acceptable":22,"unsafe":0,"review":0,"automated_count":43,"automated_acceptable":22,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":0.5116279069767442,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.5116279069767442,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.3675230846944931,"upper":0.6538255245328142,"confidence_level":0.95}},"effective":{"n":43,"acceptable":17,"unsafe":0,"review":20,"automated_count":23,"automated_acceptable":15,"automated_unsafe":0,"unnecessary_reviews":18,"accuracy":0.3953488372093023,"unsafe_rate":0.0,"review_rate":0.46511627906976744,"automated_coverage":0.5348837209302325,"conditional_correctness":0.6521739130434783,"conditional_unsafe":0.0,"unnecessary_review_rate":0.4186046511627907,"accuracy_wilson_95":{"lower":0.2636531475829744,"upper":0.5442093693712609,"confidence_level":0.95}},"confusion":{"raw":{"keep":{"keep":1,"drop":11,"mask":0,"pseudonymize":0,"generalize":0,"review":0},"drop":{"keep":0,"drop":16,"mask":0,"pseudonymize":0,"generalize":0,"review":0},"mask":{"keep":0,"drop":2,"mask":0,"pseudonymize":0,"generalize":0,"review":0},"pseudonymize":{"keep":0,"drop":2,"mask":0,"pseudonymize":0,"generalize":0,"review":0},"generalize":{"keep":0,"drop":6,"mask":0,"pseudonymize":0,"generalize":0,"review":0},"review":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0}},"effective":{"keep":{"keep":0,"drop":3,"mask":0,"pseudonymize":0,"generalize":0,"review":9},"drop":{"keep":0,"drop":12,"mask":0,"pseudonymize":0,"generalize":0,"review":4},"mask":{"keep":0,"drop":1,"mask":0,"pseudonymize":0,"generalize":0,"review":1},"pseudonymize":{"keep":0,"drop":1,"mask":0,"pseudonymize":0,"generalize":0,"review":1},"generalize":{"keep":0,"drop":3,"mask":0,"pseudonymize":0,"generalize":0,"review":3},"review":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0}}},"probability_metrics":{"singleton_n":38,"multiacceptable_n":5,"multiacceptable_coverage":0.11627906976744186,"nll":1.498115407568878,"nll_zero_mass_n":0,"brier":0.7122764199052461,"acceptable_set_n":43,"acceptable_set_nll":1.3715167513737565,"acceptable_set_zero_mass_n":0},"calibration_all_labels":{"equal_width":{"n":43,"bin_count":10,"ece":0.14169192799302036,"signed_mean_gap":-0.014986644650614538,"absolute_mean_gap":0.014986644650614538,"bins":[{"index":0,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.0,"upper":0.1,"upper_inclusive":false},{"index":1,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.1,"upper":0.2,"upper_inclusive":false},{"index":2,"count":4,"confidence":0.26087380200624466,"empirical_correctness":0.5,"signed_gap":-0.23912619799375534,"absolute_gap":0.23912619799375534,"lower":0.2,"upper":0.3,"upper_inclusive":false},{"index":3,"count":6,"confidence":0.34570741653442383,"empirical_correctness":0.5,"signed_gap":-0.15429258346557617,"absolute_gap":0.15429258346557617,"lower":0.3,"upper":0.4,"upper_inclusive":false},{"index":4,"count":16,"confidence":0.44388655945658684,"empirical_correctness":0.375,"signed_gap":0.06888655945658684,"absolute_gap":0.06888655945658684,"lower":0.4,"upper":0.5,"upper_inclusive":false},{"index":5,"count":8,"confidence":0.5425516590476036,"empirical_correctness":0.625,"signed_gap":-0.08244834095239639,"absolute_gap":0.08244834095239639,"lower":0.5,"upper":0.6,"upper_inclusive":false},{"index":6,"count":2,"confidence":0.6815258264541626,"empirical_correctness":1.0,"signed_gap":-0.3184741735458374,"absolute_gap":0.3184741735458374,"lower":0.6,"upper":0.7,"upper_inclusive":false},{"index":7,"count":6,"confidence":0.7703297734260559,"empirical_correctness":0.5,"signed_gap":0.2703297734260559,"absolute_gap":0.2703297734260559,"lower":0.7,"upper":0.8,"upper_inclusive":false},{"index":8,"count":1,"confidence":0.8102060556411743,"empirical_correctness":1.0,"signed_gap":-0.18979394435882568,"absolute_gap":0.18979394435882568,"lower":0.8,"upper":0.9,"upper_inclusive":false},{"index":9,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.9,"upper":1.0,"upper_inclusive":true}]},"equal_mass":{"n":43,"bin_count":10,"ece":0.211913934280706,"signed_mean_gap":-0.014986644650614538,"absolute_mean_gap":0.014986644650614538,"bins":[{"index":0,"count":4,"confidence":0.26087380200624466,"empirical_correctness":0.5,"signed_gap":-0.23912619799375534,"absolute_gap":0.23912619799375534,"min_confidence":0.24898016452789307,"max_confidence":0.26863178610801697},{"index":1,"count":4,"confidence":0.32644664496183395,"empirical_correctness":0.5,"signed_gap":-0.17355335503816605,"absolute_gap":0.17355335503816605,"min_confidence":0.3050265312194824,"max_confidence":0.3537122905254364},{"index":2,"count":4,"confidence":0.3947531059384346,"empirical_correctness":0.25,"signed_gap":0.1447531059384346,"absolute_gap":0.1447531059384346,"min_confidence":0.36934900283813477,"max_confidence":0.40651562809944153},{"index":3,"count":5,"confidence":0.42698013186454775,"empirical_correctness":0.0,"signed_gap":0.42698013186454775,"absolute_gap":0.42698013186454775,"min_confidence":0.41652438044548035,"max_confidence":0.4423515200614929},{"index":4,"count":4,"confidence":0.45053859800100327,"empirical_correctness":0.75,"signed_gap":-0.29946140199899673,"absolute_gap":0.29946140199899673,"min_confidence":0.4454891085624695,"max_confidence":0.461220383644104},{"index":5,"count":4,"confidence":0.4653189927339554,"empirical_correctness":0.75,"signed_gap":-0.2846810072660446,"absolute_gap":0.2846810072660446,"min_confidence":0.46180641651153564,"max_confidence":0.46992358565330505},{"index":6,"count":5,"confidence":0.5145289540290833,"empirical_correctness":0.4,"signed_gap":0.11452895402908325,"absolute_gap":0.11452895402908325,"min_confidence":0.4932994246482849,"max_confidence":0.5457789897918701},{"index":7,"count":4,"confidence":0.5652669817209244,"empirical_correctness":0.75,"signed_gap":-0.18473301827907562,"absolute_gap":0.18473301827907562,"min_confidence":0.5523136258125305,"max_confidence":0.5775802731513977},{"index":8,"count":4,"confidence":0.7119643688201904,"empirical_correctness":0.75,"signed_gap":-0.03803563117980957,"absolute_gap":0.03803563117980957,"min_confidence":0.6644905805587769,"max_confidence":0.7464307546615601},{"index":9,"count":5,"confidence":0.7894757747650146,"empirical_correctness":0.6,"signed_gap":0.18947577476501465,"absolute_gap":0.18947577476501465,"min_confidence":0.774325430393219,"max_confidence":0.8102060556411743}]}},"calibration_singleton_labels":{"equal_width":{"n":38,"bin_count":10,"ece":0.10487421876505802,"signed_mean_gap":0.049845563737969646,"absolute_mean_gap":0.049845563737969646,"bins":[{"index":0,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.0,"upper":0.1,"upper_inclusive":false},{"index":1,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.1,"upper":0.2,"upper_inclusive":false},{"index":2,"count":3,"confidence":0.25828780730565387,"empirical_correctness":0.3333333333333333,"signed_gap":-0.07504552602767944,"absolute_gap":0.07504552602767944,"lower":0.2,"upper":0.3,"upper_inclusive":false},{"index":3,"count":5,"confidence":0.34097909927368164,"empirical_correctness":0.4,"signed_gap":-0.05902090072631838,"absolute_gap":0.05902090072631838,"lower":0.3,"upper":0.4,"upper_inclusive":false},{"index":4,"count":16,"confidence":0.44388655945658684,"empirical_correctness":0.375,"signed_gap":0.06888655945658684,"absolute_gap":0.06888655945658684,"lower":0.4,"upper":0.5,"upper_inclusive":false},{"index":5,"count":6,"confidence":0.5359187126159668,"empirical_correctness":0.5,"signed_gap":0.0359187126159668,"absolute_gap":0.0359187126159668,"lower":0.5,"upper":0.6,"upper_inclusive":false},{"index":6,"count":1,"confidence":0.6644905805587769,"empirical_correctness":1.0,"signed_gap":-0.33550941944122314,"absolute_gap":0.33550941944122314,"lower":0.6,"upper":0.7,"upper_inclusive":false},{"index":7,"count":6,"confidence":0.7703297734260559,"empirical_correctness":0.5,"signed_gap":0.2703297734260559,"absolute_gap":0.2703297734260559,"lower":0.7,"upper":0.8,"upper_inclusive":false},{"index":8,"count":1,"confidence":0.8102060556411743,"empirical_correctness":1.0,"signed_gap":-0.18979394435882568,"absolute_gap":0.18979394435882568,"lower":0.8,"upper":0.9,"upper_inclusive":false},{"index":9,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.9,"upper":1.0,"upper_inclusive":true}]},"equal_mass":{"n":38,"bin_count":10,"ece":0.24681762645119115,"signed_mean_gap":0.049845563737969646,"absolute_mean_gap":0.049845563737969646,"bins":[{"index":0,"count":3,"confidence":0.25828780730565387,"empirical_correctness":0.3333333333333333,"signed_gap":-0.07504552602767944,"absolute_gap":0.07504552602767944,"min_confidence":0.24898016452789307,"max_confidence":0.266541987657547},{"index":1,"count":4,"confidence":0.32644664496183395,"empirical_correctness":0.5,"signed_gap":-0.17355335503816605,"absolute_gap":0.17355335503816605,"min_confidence":0.3050265312194824,"max_confidence":0.3537122905254364},{"index":2,"count":4,"confidence":0.406546950340271,"empirical_correctness":0.0,"signed_gap":0.406546950340271,"absolute_gap":0.406546950340271,"min_confidence":0.3991089165210724,"max_confidence":0.41652438044548035},{"index":3,"count":4,"confidence":0.4295940697193146,"empirical_correctness":0.0,"signed_gap":0.4295940697193146,"absolute_gap":0.4295940697193146,"min_confidence":0.417971134185791,"max_confidence":0.4423515200614929},{"index":4,"count":4,"confidence":0.45053859800100327,"empirical_correctness":0.75,"signed_gap":-0.29946140199899673,"absolute_gap":0.29946140199899673,"min_confidence":0.4454891085624695,"max_confidence":0.461220383644104},{"index":5,"count":3,"confidence":0.4637841284275055,"empirical_correctness":1.0,"signed_gap":-0.5362158715724945,"absolute_gap":0.5362158715724945,"min_confidence":0.46180641651153564,"max_confidence":0.4657527208328247},{"index":6,"count":4,"confidence":0.4958435073494911,"empirical_correctness":0.5,"signed_gap":-0.004156492650508881,"absolute_gap":0.004156492650508881,"min_confidence":0.46992358565330505,"max_confidence":0.5120652914047241},{"index":7,"count":4,"confidence":0.5488403141498566,"empirical_correctness":0.25,"signed_gap":0.29884031414985657,"absolute_gap":0.29884031414985657,"min_confidence":0.5134153366088867,"max_confidence":0.5775802731513977},{"index":8,"count":4,"confidence":0.7309054583311081,"empirical_correctness":0.5,"signed_gap":0.2309054583311081,"absolute_gap":0.2309054583311081,"min_confidence":0.6644905805587769,"max_confidence":0.774325430393219},{"index":9,"count":4,"confidence":0.7932633608579636,"empirical_correctness":0.75,"signed_gap":0.04326336085796356,"absolute_gap":0.04326336085796356,"min_confidence":0.780809223651886,"max_confidence":0.8102060556411743}]}},"latency_ms":{"n":43,"median":29.960599960759282,"p95":31.424660165794194},"per_action":{"keep":{"n":12,"raw":{"n":12,"acceptable":1,"unsafe":0,"review":0,"automated_count":12,"automated_acceptable":1,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":0.08333333333333333,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.08333333333333333,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.014865094404917095,"upper":0.35387991114111694,"confidence_level":0.95}},"effective":{"n":12,"acceptable":0,"unsafe":0,"review":9,"automated_count":3,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":9,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":0.75,"automated_coverage":0.25,"conditional_correctness":0.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.75,"accuracy_wilson_95":{"lower":2.7755575615628914e-17,"upper":0.24249400665524085,"confidence_level":0.95}}},"drop":{"n":16,"raw":{"n":16,"acceptable":16,"unsafe":0,"review":0,"automated_count":16,"automated_acceptable":16,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":1.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.8063923194655637,"upper":1.0,"confidence_level":0.95}},"effective":{"n":16,"acceptable":12,"unsafe":0,"review":4,"automated_count":12,"automated_acceptable":12,"automated_unsafe":0,"unnecessary_reviews":4,"accuracy":0.75,"unsafe_rate":0.0,"review_rate":0.25,"automated_coverage":0.75,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.25,"accuracy_wilson_95":{"lower":0.5050168346449122,"upper":0.8981793250878696,"confidence_level":0.95}}},"mask":{"n":2,"raw":{"n":2,"acceptable":0,"unsafe":0,"review":0,"automated_count":2,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.6576197724933469,"confidence_level":0.95}},"effective":{"n":2,"acceptable":0,"unsafe":0,"review":1,"automated_count":1,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":1,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":0.5,"automated_coverage":0.5,"conditional_correctness":0.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.5,"accuracy_wilson_95":{"lower":0.0,"upper":0.6576197724933469,"confidence_level":0.95}}},"pseudonymize":{"n":2,"raw":{"n":2,"acceptable":0,"unsafe":0,"review":0,"automated_count":2,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.6576197724933469,"confidence_level":0.95}},"effective":{"n":2,"acceptable":0,"unsafe":0,"review":1,"automated_count":1,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":1,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":0.5,"automated_coverage":0.5,"conditional_correctness":0.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.5,"accuracy_wilson_95":{"lower":0.0,"upper":0.6576197724933469,"confidence_level":0.95}}},"generalize":{"n":6,"raw":{"n":6,"acceptable":0,"unsafe":0,"review":0,"automated_count":6,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":2.7755575615628914e-17,"upper":0.3903342879021653,"confidence_level":0.95}},"effective":{"n":6,"acceptable":0,"unsafe":0,"review":3,"automated_count":3,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":3,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":0.5,"automated_coverage":0.5,"conditional_correctness":0.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.5,"accuracy_wilson_95":{"lower":2.7755575615628914e-17,"upper":0.3903342879021653,"confidence_level":0.95}}},"review":{"n":0,"raw":{"n":0,"acceptable":0,"unsafe":0,"review":0,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":null,"unsafe_rate":null,"review_rate":null,"automated_coverage":null,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":null},"effective":{"n":0,"acceptable":0,"unsafe":0,"review":0,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":null,"unsafe_rate":null,"review_rate":null,"automated_coverage":null,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":null}}}},"diagnostic43-compact_no_samples":{"n":43,"raw":{"n":43,"acceptable":21,"unsafe":0,"review":0,"automated_count":43,"automated_acceptable":21,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":0.4883720930232558,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.4883720930232558,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.3461744754671859,"upper":0.632476915305507,"confidence_level":0.95}},"effective":{"n":43,"acceptable":19,"unsafe":0,"review":10,"automated_count":33,"automated_acceptable":17,"automated_unsafe":0,"unnecessary_reviews":8,"accuracy":0.4418604651162791,"unsafe_rate":0.0,"review_rate":0.23255813953488372,"automated_coverage":0.7674418604651163,"conditional_correctness":0.5151515151515151,"conditional_unsafe":0.0,"unnecessary_review_rate":0.18604651162790697,"accuracy_wilson_95":{"lower":0.30433309839170514,"upper":0.5889238554717591,"confidence_level":0.95}},"confusion":{"raw":{"keep":{"keep":0,"drop":12,"mask":0,"pseudonymize":0,"generalize":0,"review":0},"drop":{"keep":0,"drop":16,"mask":0,"pseudonymize":0,"generalize":0,"review":0},"mask":{"keep":0,"drop":2,"mask":0,"pseudonymize":0,"generalize":0,"review":0},"pseudonymize":{"keep":0,"drop":2,"mask":0,"pseudonymize":0,"generalize":0,"review":0},"generalize":{"keep":0,"drop":6,"mask":0,"pseudonymize":0,"generalize":0,"review":0},"review":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0}},"effective":{"keep":{"keep":0,"drop":10,"mask":0,"pseudonymize":0,"generalize":0,"review":2},"drop":{"keep":0,"drop":14,"mask":0,"pseudonymize":0,"generalize":0,"review":2},"mask":{"keep":0,"drop":2,"mask":0,"pseudonymize":0,"generalize":0,"review":0},"pseudonymize":{"keep":0,"drop":1,"mask":0,"pseudonymize":0,"generalize":0,"review":1},"generalize":{"keep":0,"drop":3,"mask":0,"pseudonymize":0,"generalize":0,"review":3},"review":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0}}},"probability_metrics":{"singleton_n":38,"multiacceptable_n":5,"multiacceptable_coverage":0.11627906976744186,"nll":1.5538122864501818,"nll_zero_mass_n":0,"brier":0.7598386830288612,"acceptable_set_n":43,"acceptable_set_nll":1.4145244572691986,"acceptable_set_zero_mass_n":0},"calibration_all_labels":{"equal_width":{"n":43,"bin_count":10,"ece":0.07612846895705822,"signed_mean_gap":0.036743925061336793,"absolute_mean_gap":0.036743925061336793,"bins":[{"index":0,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.0,"upper":0.1,"upper_inclusive":false},{"index":1,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.1,"upper":0.2,"upper_inclusive":false},{"index":2,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.2,"upper":0.3,"upper_inclusive":false},{"index":3,"count":3,"confidence":0.36294228831926983,"empirical_correctness":0.3333333333333333,"signed_gap":0.02960895498593652,"absolute_gap":0.02960895498593652,"lower":0.3,"upper":0.4,"upper_inclusive":false},{"index":4,"count":18,"confidence":0.4529573503467772,"empirical_correctness":0.5,"signed_gap":-0.0470426496532228,"absolute_gap":0.0470426496532228,"lower":0.4,"upper":0.5,"upper_inclusive":false},{"index":5,"count":13,"confidence":0.5402185229154733,"empirical_correctness":0.5384615384615384,"signed_gap":0.0017569844539349067,"absolute_gap":0.0017569844539349067,"lower":0.5,"upper":0.6,"upper_inclusive":false},{"index":6,"count":3,"confidence":0.6400423049926758,"empirical_correctness":0.3333333333333333,"signed_gap":0.30670897165934247,"absolute_gap":0.30670897165934247,"lower":0.6,"upper":0.7,"upper_inclusive":false},{"index":7,"count":6,"confidence":0.732493648926417,"empirical_correctness":0.5,"signed_gap":0.232493648926417,"absolute_gap":0.232493648926417,"lower":0.7,"upper":0.8,"upper_inclusive":false},{"index":8,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.8,"upper":0.9,"upper_inclusive":false},{"index":9,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.9,"upper":1.0,"upper_inclusive":true}]},"equal_mass":{"n":43,"bin_count":10,"ece":0.15530373467955477,"signed_mean_gap":0.03674392506133678,"absolute_mean_gap":0.03674392506133678,"bins":[{"index":0,"count":4,"confidence":0.37352342903614044,"empirical_correctness":0.25,"signed_gap":0.12352342903614044,"absolute_gap":0.12352342903614044,"min_confidence":0.3341394066810608,"max_confidence":0.4052668511867523},{"index":1,"count":4,"confidence":0.4218854382634163,"empirical_correctness":0.5,"signed_gap":-0.07811456173658371,"absolute_gap":0.07811456173658371,"min_confidence":0.4087352752685547,"max_confidence":0.42861178517341614},{"index":2,"count":4,"confidence":0.4508945867419243,"empirical_correctness":0.75,"signed_gap":-0.2991054132580757,"absolute_gap":0.2991054132580757,"min_confidence":0.4425728917121887,"max_confidence":0.45718610286712646},{"index":3,"count":5,"confidence":0.4671317577362061,"empirical_correctness":0.4,"signed_gap":0.06713175773620605,"absolute_gap":0.06713175773620605,"min_confidence":0.46050456166267395,"max_confidence":0.47333747148513794},{"index":4,"count":4,"confidence":0.48029664158821106,"empirical_correctness":0.5,"signed_gap":-0.01970335841178894,"absolute_gap":0.01970335841178894,"min_confidence":0.4744929373264313,"max_confidence":0.4883209764957428},{"index":5,"count":4,"confidence":0.5096643567085266,"empirical_correctness":0.75,"signed_gap":-0.2403356432914734,"absolute_gap":0.2403356432914734,"min_confidence":0.5004886388778687,"max_confidence":0.517778217792511},{"index":6,"count":5,"confidence":0.532148015499115,"empirical_correctness":0.4,"signed_gap":0.13214801549911492,"absolute_gap":0.13214801549911492,"min_confidence":0.5188180208206177,"max_confidence":0.5490885972976685},{"index":7,"count":4,"confidence":0.580860823392868,"empirical_correctness":0.5,"signed_gap":0.08086082339286804,"absolute_gap":0.08086082339286804,"min_confidence":0.5649277567863464,"max_confidence":0.5939330458641052},{"index":8,"count":4,"confidence":0.6574743092060089,"empirical_correctness":0.25,"signed_gap":0.4074743092060089,"absolute_gap":0.4074743092060089,"min_confidence":0.6061515808105469,"max_confidence":0.7097703218460083},{"index":9,"count":5,"confidence":0.7370383143424988,"empirical_correctness":0.6,"signed_gap":0.1370383143424988,"absolute_gap":0.1370383143424988,"min_confidence":0.7114461064338684,"max_confidence":0.7810513377189636}]}},"calibration_singleton_labels":{"equal_width":{"n":38,"bin_count":10,"ece":0.1043615294130225,"signed_mean_gap":0.1043615294130225,"absolute_mean_gap":0.1043615294130225,"bins":[{"index":0,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.0,"upper":0.1,"upper_inclusive":false},{"index":1,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.1,"upper":0.2,"upper_inclusive":false},{"index":2,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.2,"upper":0.3,"upper_inclusive":false},{"index":3,"count":3,"confidence":0.36294228831926983,"empirical_correctness":0.3333333333333333,"signed_gap":0.02960895498593652,"absolute_gap":0.02960895498593652,"lower":0.3,"upper":0.4,"upper_inclusive":false},{"index":4,"count":16,"confidence":0.45637025870382786,"empirical_correctness":0.4375,"signed_gap":0.018870258703827858,"absolute_gap":0.018870258703827858,"lower":0.4,"upper":0.5,"upper_inclusive":false},{"index":5,"count":11,"confidence":0.5357934019782327,"empirical_correctness":0.45454545454545453,"signed_gap":0.08124794743277813,"absolute_gap":0.08124794743277813,"lower":0.5,"upper":0.6,"upper_inclusive":false},{"index":6,"count":2,"confidence":0.6431488990783691,"empirical_correctness":0.0,"signed_gap":0.6431488990783691,"absolute_gap":0.6431488990783691,"lower":0.6,"upper":0.7,"upper_inclusive":false},{"index":7,"count":6,"confidence":0.732493648926417,"empirical_correctness":0.5,"signed_gap":0.232493648926417,"absolute_gap":0.232493648926417,"lower":0.7,"upper":0.8,"upper_inclusive":false},{"index":8,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.8,"upper":0.9,"upper_inclusive":false},{"index":9,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.9,"upper":1.0,"upper_inclusive":true}]},"equal_mass":{"n":38,"bin_count":10,"ece":0.24976567375032524,"signed_mean_gap":0.1043615294130225,"absolute_mean_gap":0.1043615294130225,"bins":[{"index":0,"count":3,"confidence":0.36294228831926983,"empirical_correctness":0.3333333333333333,"signed_gap":0.02960895498593652,"absolute_gap":0.02960895498593652,"min_confidence":0.3341394066810608,"max_confidence":0.3951885402202606},{"index":1,"count":4,"confidence":0.4210183322429657,"empirical_correctness":0.25,"signed_gap":0.1710183322429657,"absolute_gap":0.1710183322429657,"min_confidence":0.4052668511867523,"max_confidence":0.42861178517341614},{"index":2,"count":4,"confidence":0.4553775042295456,"empirical_correctness":0.75,"signed_gap":-0.2946224957704544,"absolute_gap":0.2946224957704544,"min_confidence":0.44856932759284973,"max_confidence":0.46050456166267395},{"index":3,"count":4,"confidence":0.4687885567545891,"empirical_correctness":0.25,"signed_gap":0.21878855675458908,"absolute_gap":0.21878855675458908,"min_confidence":0.46334344148635864,"max_confidence":0.47333747148513794},{"index":4,"count":4,"confidence":0.48029664158821106,"empirical_correctness":0.5,"signed_gap":-0.01970335841178894,"absolute_gap":0.01970335841178894,"min_confidence":0.4744929373264313,"max_confidence":0.4883209764957428},{"index":5,"count":3,"confidence":0.5069597363471985,"empirical_correctness":1.0,"signed_gap":-0.4930402636528015,"absolute_gap":0.4930402636528015,"min_confidence":0.5004886388778687,"max_confidence":0.5141335129737854},{"index":6,"count":4,"confidence":0.5235623419284821,"empirical_correctness":0.25,"signed_gap":0.27356234192848206,"absolute_gap":0.27356234192848206,"min_confidence":0.517778217792511,"max_confidence":0.5364951491355896},{"index":7,"count":4,"confidence":0.5696497112512589,"empirical_correctness":0.25,"signed_gap":0.31964971125125885,"absolute_gap":0.31964971125125885,"min_confidence":0.5490885972976685,"max_confidence":0.5842247605323792},{"index":8,"count":4,"confidence":0.6768785566091537,"empirical_correctness":0.0,"signed_gap":0.6768785566091537,"absolute_gap":0.6768785566091537,"min_confidence":0.6061515808105469,"max_confidence":0.7114461064338684},{"index":9,"count":4,"confidence":0.7434363663196564,"empirical_correctness":0.75,"signed_gap":-0.006563633680343628,"absolute_gap":0.006563633680343628,"min_confidence":0.72498619556427,"max_confidence":0.7810513377189636}]}},"latency_ms":{"n":43,"median":31.99220006354153,"p95":33.38641989976168},"per_action":{"keep":{"n":12,"raw":{"n":12,"acceptable":0,"unsafe":0,"review":0,"automated_count":12,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":2.7755575615628914e-17,"upper":0.24249400665524085,"confidence_level":0.95}},"effective":{"n":12,"acceptable":0,"unsafe":0,"review":2,"automated_count":10,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":2,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":0.16666666666666666,"automated_coverage":0.8333333333333334,"conditional_correctness":0.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.16666666666666666,"accuracy_wilson_95":{"lower":2.7755575615628914e-17,"upper":0.24249400665524085,"confidence_level":0.95}}},"drop":{"n":16,"raw":{"n":16,"acceptable":16,"unsafe":0,"review":0,"automated_count":16,"automated_acceptable":16,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":1.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.8063923194655637,"upper":1.0,"confidence_level":0.95}},"effective":{"n":16,"acceptable":14,"unsafe":0,"review":2,"automated_count":14,"automated_acceptable":14,"automated_unsafe":0,"unnecessary_reviews":2,"accuracy":0.875,"unsafe_rate":0.0,"review_rate":0.125,"automated_coverage":0.875,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.125,"accuracy_wilson_95":{"lower":0.639771727342413,"upper":0.9650225122567595,"confidence_level":0.95}}},"mask":{"n":2,"raw":{"n":2,"acceptable":0,"unsafe":0,"review":0,"automated_count":2,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.6576197724933469,"confidence_level":0.95}},"effective":{"n":2,"acceptable":0,"unsafe":0,"review":0,"automated_count":2,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.6576197724933469,"confidence_level":0.95}}},"pseudonymize":{"n":2,"raw":{"n":2,"acceptable":0,"unsafe":0,"review":0,"automated_count":2,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.6576197724933469,"confidence_level":0.95}},"effective":{"n":2,"acceptable":0,"unsafe":0,"review":1,"automated_count":1,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":1,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":0.5,"automated_coverage":0.5,"conditional_correctness":0.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.5,"accuracy_wilson_95":{"lower":0.0,"upper":0.6576197724933469,"confidence_level":0.95}}},"generalize":{"n":6,"raw":{"n":6,"acceptable":0,"unsafe":0,"review":0,"automated_count":6,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":2.7755575615628914e-17,"upper":0.3903342879021653,"confidence_level":0.95}},"effective":{"n":6,"acceptable":0,"unsafe":0,"review":3,"automated_count":3,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":3,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":0.5,"automated_coverage":0.5,"conditional_correctness":0.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.5,"accuracy_wilson_95":{"lower":2.7755575615628914e-17,"upper":0.3903342879021653,"confidence_level":0.95}}},"review":{"n":0,"raw":{"n":0,"acceptable":0,"unsafe":0,"review":0,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":null,"unsafe_rate":null,"review_rate":null,"automated_coverage":null,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":null},"effective":{"n":0,"acceptable":0,"unsafe":0,"review":0,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":null,"unsafe_rate":null,"review_rate":null,"automated_coverage":null,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":null}}}},"challenge56-no_samples":{"n":56,"raw":{"n":56,"acceptable":19,"unsafe":25,"review":0,"automated_count":56,"automated_acceptable":19,"automated_unsafe":25,"unnecessary_reviews":0,"accuracy":0.3392857142857143,"unsafe_rate":0.44642857142857145,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.3392857142857143,"conditional_unsafe":0.44642857142857145,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.2291997240159191,"upper":0.4700054699293978,"confidence_level":0.95}},"effective":{"n":56,"acceptable":10,"unsafe":0,"review":46,"automated_count":10,"automated_acceptable":8,"automated_unsafe":0,"unnecessary_reviews":44,"accuracy":0.17857142857142858,"unsafe_rate":0.0,"review_rate":0.8214285714285714,"automated_coverage":0.17857142857142858,"conditional_correctness":0.8,"conditional_unsafe":0.0,"unnecessary_review_rate":0.7857142857142857,"accuracy_wilson_95":{"lower":0.09999856752412305,"upper":0.2984118203665106,"confidence_level":0.95}},"confusion":{"raw":{"keep":{"keep":0,"drop":5,"mask":3,"pseudonymize":0,"generalize":0,"review":0},"drop":{"keep":6,"drop":16,"mask":0,"pseudonymize":0,"generalize":0,"review":0},"mask":{"keep":8,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0},"pseudonymize":{"keep":5,"drop":1,"mask":2,"pseudonymize":0,"generalize":0,"review":0},"generalize":{"keep":5,"drop":0,"mask":0,"pseudonymize":0,"generalize":3,"review":0},"review":{"keep":1,"drop":1,"mask":0,"pseudonymize":0,"generalize":0,"review":0}},"effective":{"keep":{"keep":0,"drop":1,"mask":1,"pseudonymize":0,"generalize":0,"review":6},"drop":{"keep":0,"drop":7,"mask":0,"pseudonymize":0,"generalize":0,"review":15},"mask":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":8},"pseudonymize":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":8},"generalize":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":1,"review":7},"review":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":2}}},"probability_metrics":{"singleton_n":56,"multiacceptable_n":0,"multiacceptable_coverage":0.0,"nll":1.8199138260523224,"nll_zero_mass_n":0,"brier":0.8181411494526601,"acceptable_set_n":56,"acceptable_set_nll":1.8199138260523224,"acceptable_set_zero_mass_n":0},"calibration_all_labels":{"equal_width":{"n":56,"bin_count":10,"ece":0.0823979281953403,"signed_mean_gap":0.06241147007261005,"absolute_mean_gap":0.06241147007261005,"bins":[{"index":0,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.0,"upper":0.1,"upper_inclusive":false},{"index":1,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.1,"upper":0.2,"upper_inclusive":false},{"index":2,"count":8,"confidence":0.2558494880795479,"empirical_correctness":0.125,"signed_gap":0.13084948807954788,"absolute_gap":0.13084948807954788,"lower":0.2,"upper":0.3,"upper_inclusive":false},{"index":3,"count":21,"confidence":0.3470167333171481,"empirical_correctness":0.2857142857142857,"signed_gap":0.06130244760286241,"absolute_gap":0.06130244760286241,"lower":0.3,"upper":0.4,"upper_inclusive":false},{"index":4,"count":15,"confidence":0.4293586115042369,"empirical_correctness":0.4666666666666667,"signed_gap":-0.0373080551624298,"absolute_gap":0.0373080551624298,"lower":0.4,"upper":0.5,"upper_inclusive":false},{"index":5,"count":10,"confidence":0.5418941855430603,"empirical_correctness":0.4,"signed_gap":0.14189418554306032,"absolute_gap":0.14189418554306032,"lower":0.5,"upper":0.6,"upper_inclusive":false},{"index":6,"count":2,"confidence":0.6507869958877563,"empirical_correctness":0.5,"signed_gap":0.15078699588775635,"absolute_gap":0.15078699588775635,"lower":0.6,"upper":0.7,"upper_inclusive":false},{"index":7,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.7,"upper":0.8,"upper_inclusive":false},{"index":8,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.8,"upper":0.9,"upper_inclusive":false},{"index":9,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.9,"upper":1.0,"upper_inclusive":true}]},"equal_mass":{"n":56,"bin_count":10,"ece":0.15171762981585094,"signed_mean_gap":0.06241147007261003,"absolute_mean_gap":0.06241147007261003,"bins":[{"index":0,"count":5,"confidence":0.24152833223342896,"empirical_correctness":0.0,"signed_gap":0.24152833223342896,"absolute_gap":0.24152833223342896,"min_confidence":0.2063942700624466,"max_confidence":0.2675246298313141},{"index":1,"count":6,"confidence":0.2953960696856181,"empirical_correctness":0.16666666666666666,"signed_gap":0.12872940301895144,"absolute_gap":0.12872940301895144,"min_confidence":0.27507349848747253,"max_confidence":0.3149698078632355},{"index":2,"count":5,"confidence":0.3327968239784241,"empirical_correctness":0.0,"signed_gap":0.3327968239784241,"absolute_gap":0.3327968239784241,"min_confidence":0.32022422552108765,"max_confidence":0.3411448001861572},{"index":3,"count":6,"confidence":0.35176792244116467,"empirical_correctness":0.3333333333333333,"signed_gap":0.018434589107831356,"absolute_gap":0.018434589107831356,"min_confidence":0.34557661414146423,"max_confidence":0.35571083426475525},{"index":4,"count":6,"confidence":0.36520833273728687,"empirical_correctness":0.5,"signed_gap":-0.13479166726271313,"absolute_gap":0.13479166726271313,"min_confidence":0.35675328969955444,"max_confidence":0.3880380392074585},{"index":5,"count":5,"confidence":0.40269010066986083,"empirical_correctness":0.4,"signed_gap":0.0026901006698608065,"absolute_gap":0.0026901006698608065,"min_confidence":0.38828757405281067,"max_confidence":0.4104720950126648},{"index":6,"count":6,"confidence":0.41783978541692096,"empirical_correctness":0.3333333333333333,"signed_gap":0.08450645208358765,"absolute_gap":0.08450645208358765,"min_confidence":0.41086429357528687,"max_confidence":0.4268872141838074},{"index":7,"count":5,"confidence":0.46163550615310667,"empirical_correctness":0.8,"signed_gap":-0.3383644938468934,"absolute_gap":0.3383644938468934,"min_confidence":0.4401300251483917,"max_confidence":0.4861544072628021},{"index":8,"count":6,"confidence":0.5351603130499522,"empirical_correctness":0.5,"signed_gap":0.03516031304995215,"absolute_gap":0.03516031304995215,"min_confidence":0.5167147517204285,"max_confidence":0.5458181500434875},{"index":9,"count":6,"confidence":0.5849256614844004,"empirical_correctness":0.3333333333333333,"signed_gap":0.2515923281510671,"absolute_gap":0.2515923281510671,"min_confidence":0.546680748462677,"max_confidence":0.6593695282936096}]}},"calibration_singleton_labels":{"equal_width":{"n":56,"bin_count":10,"ece":0.0823979281953403,"signed_mean_gap":0.06241147007261005,"absolute_mean_gap":0.06241147007261005,"bins":[{"index":0,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.0,"upper":0.1,"upper_inclusive":false},{"index":1,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.1,"upper":0.2,"upper_inclusive":false},{"index":2,"count":8,"confidence":0.2558494880795479,"empirical_correctness":0.125,"signed_gap":0.13084948807954788,"absolute_gap":0.13084948807954788,"lower":0.2,"upper":0.3,"upper_inclusive":false},{"index":3,"count":21,"confidence":0.3470167333171481,"empirical_correctness":0.2857142857142857,"signed_gap":0.06130244760286241,"absolute_gap":0.06130244760286241,"lower":0.3,"upper":0.4,"upper_inclusive":false},{"index":4,"count":15,"confidence":0.4293586115042369,"empirical_correctness":0.4666666666666667,"signed_gap":-0.0373080551624298,"absolute_gap":0.0373080551624298,"lower":0.4,"upper":0.5,"upper_inclusive":false},{"index":5,"count":10,"confidence":0.5418941855430603,"empirical_correctness":0.4,"signed_gap":0.14189418554306032,"absolute_gap":0.14189418554306032,"lower":0.5,"upper":0.6,"upper_inclusive":false},{"index":6,"count":2,"confidence":0.6507869958877563,"empirical_correctness":0.5,"signed_gap":0.15078699588775635,"absolute_gap":0.15078699588775635,"lower":0.6,"upper":0.7,"upper_inclusive":false},{"index":7,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.7,"upper":0.8,"upper_inclusive":false},{"index":8,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.8,"upper":0.9,"upper_inclusive":false},{"index":9,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.9,"upper":1.0,"upper_inclusive":true}]},"equal_mass":{"n":56,"bin_count":10,"ece":0.15171762981585094,"signed_mean_gap":0.06241147007261003,"absolute_mean_gap":0.06241147007261003,"bins":[{"index":0,"count":5,"confidence":0.24152833223342896,"empirical_correctness":0.0,"signed_gap":0.24152833223342896,"absolute_gap":0.24152833223342896,"min_confidence":0.2063942700624466,"max_confidence":0.2675246298313141},{"index":1,"count":6,"confidence":0.2953960696856181,"empirical_correctness":0.16666666666666666,"signed_gap":0.12872940301895144,"absolute_gap":0.12872940301895144,"min_confidence":0.27507349848747253,"max_confidence":0.3149698078632355},{"index":2,"count":5,"confidence":0.3327968239784241,"empirical_correctness":0.0,"signed_gap":0.3327968239784241,"absolute_gap":0.3327968239784241,"min_confidence":0.32022422552108765,"max_confidence":0.3411448001861572},{"index":3,"count":6,"confidence":0.35176792244116467,"empirical_correctness":0.3333333333333333,"signed_gap":0.018434589107831356,"absolute_gap":0.018434589107831356,"min_confidence":0.34557661414146423,"max_confidence":0.35571083426475525},{"index":4,"count":6,"confidence":0.36520833273728687,"empirical_correctness":0.5,"signed_gap":-0.13479166726271313,"absolute_gap":0.13479166726271313,"min_confidence":0.35675328969955444,"max_confidence":0.3880380392074585},{"index":5,"count":5,"confidence":0.40269010066986083,"empirical_correctness":0.4,"signed_gap":0.0026901006698608065,"absolute_gap":0.0026901006698608065,"min_confidence":0.38828757405281067,"max_confidence":0.4104720950126648},{"index":6,"count":6,"confidence":0.41783978541692096,"empirical_correctness":0.3333333333333333,"signed_gap":0.08450645208358765,"absolute_gap":0.08450645208358765,"min_confidence":0.41086429357528687,"max_confidence":0.4268872141838074},{"index":7,"count":5,"confidence":0.46163550615310667,"empirical_correctness":0.8,"signed_gap":-0.3383644938468934,"absolute_gap":0.3383644938468934,"min_confidence":0.4401300251483917,"max_confidence":0.4861544072628021},{"index":8,"count":6,"confidence":0.5351603130499522,"empirical_correctness":0.5,"signed_gap":0.03516031304995215,"absolute_gap":0.03516031304995215,"min_confidence":0.5167147517204285,"max_confidence":0.5458181500434875},{"index":9,"count":6,"confidence":0.5849256614844004,"empirical_correctness":0.3333333333333333,"signed_gap":0.2515923281510671,"absolute_gap":0.2515923281510671,"min_confidence":0.546680748462677,"max_confidence":0.6593695282936096}]}},"latency_ms":{"n":56,"median":25.50504996906966,"p95":26.74784994451329},"per_action":{"keep":{"n":8,"raw":{"n":8,"acceptable":0,"unsafe":0,"review":0,"automated_count":8,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.32440756488388023,"confidence_level":0.95}},"effective":{"n":8,"acceptable":0,"unsafe":0,"review":6,"automated_count":2,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":6,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":0.75,"automated_coverage":0.25,"conditional_correctness":0.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.75,"accuracy_wilson_95":{"lower":0.0,"upper":0.32440756488388023,"confidence_level":0.95}}},"drop":{"n":22,"raw":{"n":22,"acceptable":16,"unsafe":6,"review":0,"automated_count":22,"automated_acceptable":16,"automated_unsafe":6,"unnecessary_reviews":0,"accuracy":0.7272727272727273,"unsafe_rate":0.2727272727272727,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.7272727272727273,"conditional_unsafe":0.2727272727272727,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.5184826598759762,"upper":0.8684923971201053,"confidence_level":0.95}},"effective":{"n":22,"acceptable":7,"unsafe":0,"review":15,"automated_count":7,"automated_acceptable":7,"automated_unsafe":0,"unnecessary_reviews":15,"accuracy":0.3181818181818182,"unsafe_rate":0.0,"review_rate":0.6818181818181818,"automated_coverage":0.3181818181818182,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.6818181818181818,"accuracy_wilson_95":{"lower":0.1636059318135624,"upper":0.5268140225895723,"confidence_level":0.95}}},"mask":{"n":8,"raw":{"n":8,"acceptable":0,"unsafe":8,"review":0,"automated_count":8,"automated_acceptable":0,"automated_unsafe":8,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":1.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":1.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.32440756488388023,"confidence_level":0.95}},"effective":{"n":8,"acceptable":0,"unsafe":0,"review":8,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":8,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.32440756488388023,"confidence_level":0.95}}},"pseudonymize":{"n":8,"raw":{"n":8,"acceptable":0,"unsafe":5,"review":0,"automated_count":8,"automated_acceptable":0,"automated_unsafe":5,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":0.625,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":0.625,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.32440756488388023,"confidence_level":0.95}},"effective":{"n":8,"acceptable":0,"unsafe":0,"review":8,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":8,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.32440756488388023,"confidence_level":0.95}}},"generalize":{"n":8,"raw":{"n":8,"acceptable":3,"unsafe":5,"review":0,"automated_count":8,"automated_acceptable":3,"automated_unsafe":5,"unnecessary_reviews":0,"accuracy":0.375,"unsafe_rate":0.625,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.375,"conditional_unsafe":0.625,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.13684428582359742,"upper":0.6942576053973727,"confidence_level":0.95}},"effective":{"n":8,"acceptable":1,"unsafe":0,"review":7,"automated_count":1,"automated_acceptable":1,"automated_unsafe":0,"unnecessary_reviews":7,"accuracy":0.125,"unsafe_rate":0.0,"review_rate":0.875,"automated_coverage":0.125,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.875,"accuracy_wilson_95":{"lower":0.02241749145005667,"upper":0.4708881822128535,"confidence_level":0.95}}},"review":{"n":2,"raw":{"n":2,"acceptable":0,"unsafe":1,"review":0,"automated_count":2,"automated_acceptable":0,"automated_unsafe":1,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":0.5,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":0.5,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.6576197724933469,"confidence_level":0.95}},"effective":{"n":2,"acceptable":2,"unsafe":0,"review":2,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":1.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.34238022750665303,"upper":1.0,"confidence_level":0.95}}}}},"challenge56-compact_no_samples":{"n":56,"raw":{"n":56,"acceptable":16,"unsafe":30,"review":0,"automated_count":56,"automated_acceptable":16,"automated_unsafe":30,"unnecessary_reviews":0,"accuracy":0.2857142857142857,"unsafe_rate":0.5357142857142857,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.2857142857142857,"conditional_unsafe":0.5357142857142857,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.18418775143858906,"upper":0.4147525071551667,"confidence_level":0.95}},"effective":{"n":56,"acceptable":7,"unsafe":0,"review":45,"automated_count":11,"automated_acceptable":6,"automated_unsafe":0,"unnecessary_reviews":44,"accuracy":0.125,"unsafe_rate":0.0,"review_rate":0.8035714285714286,"automated_coverage":0.19642857142857142,"conditional_correctness":0.5454545454545454,"conditional_unsafe":0.0,"unnecessary_review_rate":0.7857142857142857,"accuracy_wilson_95":{"lower":0.06189068695717122,"upper":0.2362547655819014,"confidence_level":0.95}},"confusion":{"raw":{"keep":{"keep":3,"drop":5,"mask":0,"pseudonymize":0,"generalize":0,"review":0},"drop":{"keep":9,"drop":13,"mask":0,"pseudonymize":0,"generalize":0,"review":0},"mask":{"keep":8,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0},"pseudonymize":{"keep":8,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0},"generalize":{"keep":5,"drop":3,"mask":0,"pseudonymize":0,"generalize":0,"review":0},"review":{"keep":0,"drop":2,"mask":0,"pseudonymize":0,"generalize":0,"review":0}},"effective":{"keep":{"keep":0,"drop":2,"mask":0,"pseudonymize":0,"generalize":0,"review":6},"drop":{"keep":0,"drop":6,"mask":0,"pseudonymize":0,"generalize":0,"review":16},"mask":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":8},"pseudonymize":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":8},"generalize":{"keep":0,"drop":2,"mask":0,"pseudonymize":0,"generalize":0,"review":6},"review":{"keep":0,"drop":1,"mask":0,"pseudonymize":0,"generalize":0,"review":1}}},"probability_metrics":{"singleton_n":56,"multiacceptable_n":0,"multiacceptable_coverage":0.0,"nll":1.9629181978740768,"nll_zero_mass_n":0,"brier":0.8996891516394905,"acceptable_set_n":56,"acceptable_set_nll":1.9629181978740768,"acceptable_set_zero_mass_n":0},"calibration_all_labels":{"equal_width":{"n":56,"bin_count":10,"ece":0.2288393814648901,"signed_mean_gap":0.2032894332494055,"absolute_mean_gap":0.2032894332494055,"bins":[{"index":0,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.0,"upper":0.1,"upper_inclusive":false},{"index":1,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.1,"upper":0.2,"upper_inclusive":false},{"index":2,"count":1,"confidence":0.28460144996643066,"empirical_correctness":1.0,"signed_gap":-0.7153985500335693,"absolute_gap":0.7153985500335693,"lower":0.2,"upper":0.3,"upper_inclusive":false},{"index":3,"count":5,"confidence":0.3658793270587921,"empirical_correctness":0.2,"signed_gap":0.1658793270587921,"absolute_gap":0.1658793270587921,"lower":0.3,"upper":0.4,"upper_inclusive":false},{"index":4,"count":27,"confidence":0.44210197859340244,"empirical_correctness":0.4074074074074074,"signed_gap":0.03469457118599506,"absolute_gap":0.03469457118599506,"lower":0.4,"upper":0.5,"upper_inclusive":false},{"index":5,"count":15,"confidence":0.5463688015937805,"empirical_correctness":0.13333333333333333,"signed_gap":0.41303546826044724,"absolute_gap":0.41303546826044724,"lower":0.5,"upper":0.6,"upper_inclusive":false},{"index":6,"count":8,"confidence":0.6422405913472176,"empirical_correctness":0.125,"signed_gap":0.5172405913472176,"absolute_gap":0.5172405913472176,"lower":0.6,"upper":0.7,"upper_inclusive":false},{"index":7,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.7,"upper":0.8,"upper_inclusive":false},{"index":8,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.8,"upper":0.9,"upper_inclusive":false},{"index":9,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.9,"upper":1.0,"upper_inclusive":true}]},"equal_mass":{"n":56,"bin_count":10,"ece":0.26343345642089844,"signed_mean_gap":0.20328943324940546,"absolute_mean_gap":0.20328943324940546,"bins":[{"index":0,"count":5,"confidence":0.34449188113212587,"empirical_correctness":0.4,"signed_gap":-0.05550811886787416,"absolute_gap":0.05550811886787416,"min_confidence":0.28460144996643066,"max_confidence":0.3782387673854828},{"index":1,"count":6,"confidence":0.4023165206114451,"empirical_correctness":0.3333333333333333,"signed_gap":0.06898318727811181,"absolute_gap":0.06898318727811181,"min_confidence":0.39153867959976196,"max_confidence":0.40634313225746155},{"index":2,"count":5,"confidence":0.4116859078407288,"empirical_correctness":0.4,"signed_gap":0.01168590784072876,"absolute_gap":0.01168590784072876,"min_confidence":0.41014280915260315,"max_confidence":0.41515353322029114},{"index":3,"count":6,"confidence":0.43225132425626117,"empirical_correctness":0.6666666666666666,"signed_gap":-0.23441534241040546,"absolute_gap":0.23441534241040546,"min_confidence":0.42075300216674805,"max_confidence":0.4403689205646515},{"index":4,"count":6,"confidence":0.4650016874074936,"empirical_correctness":0.3333333333333333,"signed_gap":0.13166835407416028,"absolute_gap":0.13166835407416028,"min_confidence":0.44140198826789856,"max_confidence":0.4778588116168976},{"index":5,"count":5,"confidence":0.49448907375335693,"empirical_correctness":0.2,"signed_gap":0.2944890737533569,"absolute_gap":0.2944890737533569,"min_confidence":0.4891493320465088,"max_confidence":0.49924251437187195},{"index":6,"count":6,"confidence":0.520641028881073,"empirical_correctness":0.3333333333333333,"signed_gap":0.18730769554773968,"absolute_gap":0.18730769554773968,"min_confidence":0.5055351853370667,"max_confidence":0.5329491496086121},{"index":7,"count":5,"confidence":0.550051212310791,"empirical_correctness":0.0,"signed_gap":0.550051212310791,"absolute_gap":0.550051212310791,"min_confidence":0.5407163500785828,"max_confidence":0.5650370717048645},{"index":8,"count":6,"confidence":0.5932685534159342,"empirical_correctness":0.0,"signed_gap":0.5932685534159342,"absolute_gap":0.5932685534159342,"min_confidence":0.5682176351547241,"max_confidence":0.6198132038116455},{"index":9,"count":6,"confidence":0.6499571998914083,"empirical_correctness":0.16666666666666666,"signed_gap":0.4832905332247417,"absolute_gap":0.4832905332247417,"min_confidence":0.6281993389129639,"max_confidence":0.6850895881652832}]}},"calibration_singleton_labels":{"equal_width":{"n":56,"bin_count":10,"ece":0.2288393814648901,"signed_mean_gap":0.2032894332494055,"absolute_mean_gap":0.2032894332494055,"bins":[{"index":0,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.0,"upper":0.1,"upper_inclusive":false},{"index":1,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.1,"upper":0.2,"upper_inclusive":false},{"index":2,"count":1,"confidence":0.28460144996643066,"empirical_correctness":1.0,"signed_gap":-0.7153985500335693,"absolute_gap":0.7153985500335693,"lower":0.2,"upper":0.3,"upper_inclusive":false},{"index":3,"count":5,"confidence":0.3658793270587921,"empirical_correctness":0.2,"signed_gap":0.1658793270587921,"absolute_gap":0.1658793270587921,"lower":0.3,"upper":0.4,"upper_inclusive":false},{"index":4,"count":27,"confidence":0.44210197859340244,"empirical_correctness":0.4074074074074074,"signed_gap":0.03469457118599506,"absolute_gap":0.03469457118599506,"lower":0.4,"upper":0.5,"upper_inclusive":false},{"index":5,"count":15,"confidence":0.5463688015937805,"empirical_correctness":0.13333333333333333,"signed_gap":0.41303546826044724,"absolute_gap":0.41303546826044724,"lower":0.5,"upper":0.6,"upper_inclusive":false},{"index":6,"count":8,"confidence":0.6422405913472176,"empirical_correctness":0.125,"signed_gap":0.5172405913472176,"absolute_gap":0.5172405913472176,"lower":0.6,"upper":0.7,"upper_inclusive":false},{"index":7,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.7,"upper":0.8,"upper_inclusive":false},{"index":8,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.8,"upper":0.9,"upper_inclusive":false},{"index":9,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.9,"upper":1.0,"upper_inclusive":true}]},"equal_mass":{"n":56,"bin_count":10,"ece":0.26343345642089844,"signed_mean_gap":0.20328943324940546,"absolute_mean_gap":0.20328943324940546,"bins":[{"index":0,"count":5,"confidence":0.34449188113212587,"empirical_correctness":0.4,"signed_gap":-0.05550811886787416,"absolute_gap":0.05550811886787416,"min_confidence":0.28460144996643066,"max_confidence":0.3782387673854828},{"index":1,"count":6,"confidence":0.4023165206114451,"empirical_correctness":0.3333333333333333,"signed_gap":0.06898318727811181,"absolute_gap":0.06898318727811181,"min_confidence":0.39153867959976196,"max_confidence":0.40634313225746155},{"index":2,"count":5,"confidence":0.4116859078407288,"empirical_correctness":0.4,"signed_gap":0.01168590784072876,"absolute_gap":0.01168590784072876,"min_confidence":0.41014280915260315,"max_confidence":0.41515353322029114},{"index":3,"count":6,"confidence":0.43225132425626117,"empirical_correctness":0.6666666666666666,"signed_gap":-0.23441534241040546,"absolute_gap":0.23441534241040546,"min_confidence":0.42075300216674805,"max_confidence":0.4403689205646515},{"index":4,"count":6,"confidence":0.4650016874074936,"empirical_correctness":0.3333333333333333,"signed_gap":0.13166835407416028,"absolute_gap":0.13166835407416028,"min_confidence":0.44140198826789856,"max_confidence":0.4778588116168976},{"index":5,"count":5,"confidence":0.49448907375335693,"empirical_correctness":0.2,"signed_gap":0.2944890737533569,"absolute_gap":0.2944890737533569,"min_confidence":0.4891493320465088,"max_confidence":0.49924251437187195},{"index":6,"count":6,"confidence":0.520641028881073,"empirical_correctness":0.3333333333333333,"signed_gap":0.18730769554773968,"absolute_gap":0.18730769554773968,"min_confidence":0.5055351853370667,"max_confidence":0.5329491496086121},{"index":7,"count":5,"confidence":0.550051212310791,"empirical_correctness":0.0,"signed_gap":0.550051212310791,"absolute_gap":0.550051212310791,"min_confidence":0.5407163500785828,"max_confidence":0.5650370717048645},{"index":8,"count":6,"confidence":0.5932685534159342,"empirical_correctness":0.0,"signed_gap":0.5932685534159342,"absolute_gap":0.5932685534159342,"min_confidence":0.5682176351547241,"max_confidence":0.6198132038116455},{"index":9,"count":6,"confidence":0.6499571998914083,"empirical_correctness":0.16666666666666666,"signed_gap":0.4832905332247417,"absolute_gap":0.4832905332247417,"min_confidence":0.6281993389129639,"max_confidence":0.6850895881652832}]}},"latency_ms":{"n":56,"median":31.002350035123527,"p95":32.40317484596744},"per_action":{"keep":{"n":8,"raw":{"n":8,"acceptable":3,"unsafe":0,"review":0,"automated_count":8,"automated_acceptable":3,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":0.375,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.375,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.13684428582359742,"upper":0.6942576053973727,"confidence_level":0.95}},"effective":{"n":8,"acceptable":0,"unsafe":0,"review":6,"automated_count":2,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":6,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":0.75,"automated_coverage":0.25,"conditional_correctness":0.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.75,"accuracy_wilson_95":{"lower":0.0,"upper":0.32440756488388023,"confidence_level":0.95}}},"drop":{"n":22,"raw":{"n":22,"acceptable":13,"unsafe":9,"review":0,"automated_count":22,"automated_acceptable":13,"automated_unsafe":9,"unnecessary_reviews":0,"accuracy":0.5909090909090909,"unsafe_rate":0.4090909090909091,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.5909090909090909,"conditional_unsafe":0.4090909090909091,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.3873481847631513,"upper":0.7674418380352814,"confidence_level":0.95}},"effective":{"n":22,"acceptable":6,"unsafe":0,"review":16,"automated_count":6,"automated_acceptable":6,"automated_unsafe":0,"unnecessary_reviews":16,"accuracy":0.2727272727272727,"unsafe_rate":0.0,"review_rate":0.7272727272727273,"automated_coverage":0.2727272727272727,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.7272727272727273,"accuracy_wilson_95":{"lower":0.13150760287989469,"upper":0.4815173401240238,"confidence_level":0.95}}},"mask":{"n":8,"raw":{"n":8,"acceptable":0,"unsafe":8,"review":0,"automated_count":8,"automated_acceptable":0,"automated_unsafe":8,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":1.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":1.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.32440756488388023,"confidence_level":0.95}},"effective":{"n":8,"acceptable":0,"unsafe":0,"review":8,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":8,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.32440756488388023,"confidence_level":0.95}}},"pseudonymize":{"n":8,"raw":{"n":8,"acceptable":0,"unsafe":8,"review":0,"automated_count":8,"automated_acceptable":0,"automated_unsafe":8,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":1.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":1.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.32440756488388023,"confidence_level":0.95}},"effective":{"n":8,"acceptable":0,"unsafe":0,"review":8,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":8,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.32440756488388023,"confidence_level":0.95}}},"generalize":{"n":8,"raw":{"n":8,"acceptable":0,"unsafe":5,"review":0,"automated_count":8,"automated_acceptable":0,"automated_unsafe":5,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":0.625,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":0.625,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.32440756488388023,"confidence_level":0.95}},"effective":{"n":8,"acceptable":0,"unsafe":0,"review":6,"automated_count":2,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":6,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":0.75,"automated_coverage":0.25,"conditional_correctness":0.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.75,"accuracy_wilson_95":{"lower":0.0,"upper":0.32440756488388023,"confidence_level":0.95}}},"review":{"n":2,"raw":{"n":2,"acceptable":0,"unsafe":0,"review":0,"automated_count":2,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.6576197724933469,"confidence_level":0.95}},"effective":{"n":2,"acceptable":1,"unsafe":0,"review":1,"automated_count":1,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":0.5,"unsafe_rate":0.0,"review_rate":0.5,"automated_coverage":0.5,"conditional_correctness":0.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.09453120573423074,"upper":0.9054687942657693,"confidence_level":0.95}}}}}},"composite":{"n":198,"scope":"descriptive sum across four correlated panels","raw":{"n":198,"acceptable":78,"unsafe":55,"review":0,"automated_count":198,"automated_acceptable":78,"automated_unsafe":55,"unnecessary_reviews":0,"accuracy":0.3939393939393939,"unsafe_rate":0.2777777777777778,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.3939393939393939,"conditional_unsafe":0.2777777777777778,"unnecessary_review_rate":0.0},"effective":{"n":198,"acceptable":53,"unsafe":0,"review":121,"automated_count":77,"automated_acceptable":46,"automated_unsafe":0,"unnecessary_reviews":114,"accuracy":0.2676767676767677,"unsafe_rate":0.0,"review_rate":0.6111111111111112,"automated_coverage":0.3888888888888889,"conditional_correctness":0.5974025974025974,"conditional_unsafe":0.0,"unnecessary_review_rate":0.5757575757575758}}},{"id":"Suarify--laya-pdpa-my-v4-fp32","platform":"Local","model_id":"Suarify/laya-pdpa-my-v4","revision":"314b5a365f3fa641cf79322b8ec52c3c80e495e1","loading":"float32 / none","scoring_kind":"Native decision readout","settings":{"dtype":"float32","quantization":"none","max_tokens":1024,"head_max_tokens":256,"temperature":1.0323784351348877,"autocast":false,"truncation":false,"cpu_offload":false,"guard_threshold":0.45,"seed":0,"tf32":false},"scoring_metadata":{"backend":"laya_native_local_builtin","readout":"native_mask_slot_scorer_numpy_float32_softmax","probability_source":"native_choice_unrounded_published_temperature_not_task_calibrated","latency_scope":"synchronized serial complete encoder+head+act forward,input transfer and native readout; tokenization excluded","native_disclosures":["Stock installed torch attention in place of ONNX dynamic equivalent; no custom kernels or repo imports.","FP32/no-autocast explicit precision: convert every saved F16/BF16/F32 tensor toFP32; no quantization.","Unrounded native probability; guard uses winner p, not entropy-concentration confidence.","Act head fully loaded and evaluated; not used to alter six-action answer.","Native choice:6-10 bucket else choice scalar0, T=1.0323784351348877. No task fitting, priors correction, prompt change or finetuning here."],"equal_mass_convention":"rank bins with floor boundaries at floor(i*n/k), k=min(10,n//2); minimum two rows"},"load_ms":null,"peak_allocated_bytes":1363195392,"peak_reserved_bytes":1396703232,"panels":{"diagnostic43-no_samples":{"n":43,"raw":{"n":43,"acceptable":23,"unsafe":4,"review":1,"automated_count":42,"automated_acceptable":22,"automated_unsafe":4,"unnecessary_reviews":0,"accuracy":0.5348837209302325,"unsafe_rate":0.09302325581395349,"review_rate":0.023255813953488372,"automated_coverage":0.9767441860465116,"conditional_correctness":0.5238095238095238,"conditional_unsafe":0.09523809523809523,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.38915640472313967,"upper":0.6748894229587816,"confidence_level":0.95}},"effective":{"n":43,"acceptable":20,"unsafe":0,"review":14,"automated_count":29,"automated_acceptable":18,"automated_unsafe":0,"unnecessary_reviews":12,"accuracy":0.46511627906976744,"unsafe_rate":0.0,"review_rate":0.32558139534883723,"automated_coverage":0.6744186046511628,"conditional_correctness":0.6206896551724138,"conditional_unsafe":0.0,"unnecessary_review_rate":0.27906976744186046,"accuracy_wilson_95":{"lower":0.32511057704121815,"upper":0.6108435952768602,"confidence_level":0.95}},"confusion":{"raw":{"keep":{"keep":3,"drop":9,"mask":0,"pseudonymize":0,"generalize":0,"review":0},"drop":{"keep":2,"drop":14,"mask":0,"pseudonymize":0,"generalize":0,"review":0},"mask":{"keep":1,"drop":1,"mask":0,"pseudonymize":0,"generalize":0,"review":0},"pseudonymize":{"keep":0,"drop":2,"mask":0,"pseudonymize":0,"generalize":0,"review":0},"generalize":{"keep":0,"drop":3,"mask":0,"pseudonymize":1,"generalize":2,"review":0},"review":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0}},"effective":{"keep":{"keep":3,"drop":6,"mask":0,"pseudonymize":0,"generalize":0,"review":3},"drop":{"keep":0,"drop":12,"mask":0,"pseudonymize":0,"generalize":0,"review":4},"mask":{"keep":0,"drop":1,"mask":0,"pseudonymize":0,"generalize":0,"review":1},"pseudonymize":{"keep":0,"drop":2,"mask":0,"pseudonymize":0,"generalize":0,"review":0},"generalize":{"keep":0,"drop":2,"mask":0,"pseudonymize":0,"generalize":0,"review":4},"review":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0}}},"probability_metrics":{"singleton_n":38,"multiacceptable_n":5,"multiacceptable_coverage":0.11627906976744186,"nll":1.6523224686287912,"nll_zero_mass_n":0,"brier":0.76972835327329,"acceptable_set_n":43,"acceptable_set_nll":1.523559359302367,"acceptable_set_zero_mass_n":0},"calibration_all_labels":{"equal_width":{"n":43,"bin_count":10,"ece":0.2982506925283476,"signed_mean_gap":0.1423885163872741,"absolute_mean_gap":0.1423885163872741,"bins":[{"index":0,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.0,"upper":0.1,"upper_inclusive":false},{"index":1,"count":1,"confidence":0.19377794861793518,"empirical_correctness":0.0,"signed_gap":0.19377794861793518,"absolute_gap":0.19377794861793518,"lower":0.1,"upper":0.2,"upper_inclusive":false},{"index":2,"count":4,"confidence":0.22261185944080353,"empirical_correctness":0.5,"signed_gap":-0.27738814055919647,"absolute_gap":0.27738814055919647,"lower":0.2,"upper":0.3,"upper_inclusive":false},{"index":3,"count":5,"confidence":0.3673316717147827,"empirical_correctness":0.4,"signed_gap":-0.03266832828521732,"absolute_gap":0.03266832828521732,"lower":0.3,"upper":0.4,"upper_inclusive":false},{"index":4,"count":2,"confidence":0.4878920018672943,"empirical_correctness":0.0,"signed_gap":0.4878920018672943,"absolute_gap":0.4878920018672943,"lower":0.4,"upper":0.5,"upper_inclusive":false},{"index":5,"count":2,"confidence":0.5326344072818756,"empirical_correctness":1.0,"signed_gap":-0.4673655927181244,"absolute_gap":0.4673655927181244,"lower":0.5,"upper":0.6,"upper_inclusive":false},{"index":6,"count":3,"confidence":0.6188628673553467,"empirical_correctness":1.0,"signed_gap":-0.3811371326446533,"absolute_gap":0.3811371326446533,"lower":0.6,"upper":0.7,"upper_inclusive":false},{"index":7,"count":4,"confidence":0.7593721002340317,"empirical_correctness":0.25,"signed_gap":0.5093721002340317,"absolute_gap":0.5093721002340317,"lower":0.7,"upper":0.8,"upper_inclusive":false},{"index":8,"count":15,"confidence":0.8604497075080871,"empirical_correctness":0.5333333333333333,"signed_gap":0.3271163741747538,"absolute_gap":0.3271163741747538,"lower":0.8,"upper":0.9,"upper_inclusive":false},{"index":9,"count":7,"confidence":0.90856386082513,"empirical_correctness":0.7142857142857143,"signed_gap":0.19427814653941566,"absolute_gap":0.19427814653941566,"lower":0.9,"upper":1.0,"upper_inclusive":true}]},"equal_mass":{"n":43,"bin_count":10,"ece":0.2937822723111441,"signed_mean_gap":0.1423885163872741,"absolute_mean_gap":0.1423885163872741,"bins":[{"index":0,"count":4,"confidence":0.20579420775175095,"empirical_correctness":0.5,"signed_gap":-0.29420579224824905,"absolute_gap":0.29420579224824905,"min_confidence":0.19377794861793518,"max_confidence":0.2139846235513687},{"index":1,"count":4,"confidence":0.3300873637199402,"empirical_correctness":0.25,"signed_gap":0.08008736371994019,"absolute_gap":0.08008736371994019,"min_confidence":0.2610485553741455,"max_confidence":0.3768385350704193},{"index":2,"count":4,"confidence":0.43828536570072174,"empirical_correctness":0.25,"signed_gap":0.18828536570072174,"absolute_gap":0.18828536570072174,"min_confidence":0.38703638315200806,"max_confidence":0.49560457468032837},{"index":3,"count":5,"confidence":0.5843714833259582,"empirical_correctness":1.0,"signed_gap":-0.41562851667404177,"absolute_gap":0.41562851667404177,"min_confidence":0.5176056027412415,"max_confidence":0.6262155175209045},{"index":4,"count":4,"confidence":0.7593721002340317,"empirical_correctness":0.25,"signed_gap":0.5093721002340317,"absolute_gap":0.5093721002340317,"min_confidence":0.7277738451957703,"max_confidence":0.7868255376815796},{"index":5,"count":4,"confidence":0.8208900988101959,"empirical_correctness":0.5,"signed_gap":0.3208900988101959,"absolute_gap":0.3208900988101959,"min_confidence":0.8170597553253174,"max_confidence":0.8248232007026672},{"index":6,"count":5,"confidence":0.8572390913963318,"empirical_correctness":0.4,"signed_gap":0.4572390913963318,"absolute_gap":0.4572390913963318,"min_confidence":0.8468538522720337,"max_confidence":0.8653526306152344},{"index":7,"count":4,"confidence":0.886197105050087,"empirical_correctness":0.75,"signed_gap":0.13619710505008698,"absolute_gap":0.13619710505008698,"min_confidence":0.8787842392921448,"max_confidence":0.8938862681388855},{"index":8,"count":4,"confidence":0.8990834802389145,"empirical_correctness":0.5,"signed_gap":0.3990834802389145,"absolute_gap":0.3990834802389145,"min_confidence":0.8943731784820557,"max_confidence":0.9033029675483704},{"index":9,"count":5,"confidence":0.9111628890037536,"empirical_correctness":0.8,"signed_gap":0.1111628890037536,"absolute_gap":0.1111628890037536,"min_confidence":0.9063014984130859,"max_confidence":0.918705403804779}]}},"calibration_singleton_labels":{"equal_width":{"n":38,"bin_count":10,"ece":0.31522896650590393,"signed_mean_gap":0.1585314767925363,"absolute_mean_gap":0.1585314767925363,"bins":[{"index":0,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.0,"upper":0.1,"upper_inclusive":false},{"index":1,"count":1,"confidence":0.19377794861793518,"empirical_correctness":0.0,"signed_gap":0.19377794861793518,"absolute_gap":0.19377794861793518,"lower":0.1,"upper":0.2,"upper_inclusive":false},{"index":2,"count":4,"confidence":0.22261185944080353,"empirical_correctness":0.5,"signed_gap":-0.27738814055919647,"absolute_gap":0.27738814055919647,"lower":0.2,"upper":0.3,"upper_inclusive":false},{"index":3,"count":5,"confidence":0.3673316717147827,"empirical_correctness":0.4,"signed_gap":-0.03266832828521732,"absolute_gap":0.03266832828521732,"lower":0.3,"upper":0.4,"upper_inclusive":false},{"index":4,"count":2,"confidence":0.4878920018672943,"empirical_correctness":0.0,"signed_gap":0.4878920018672943,"absolute_gap":0.4878920018672943,"lower":0.4,"upper":0.5,"upper_inclusive":false},{"index":5,"count":2,"confidence":0.5326344072818756,"empirical_correctness":1.0,"signed_gap":-0.4673655927181244,"absolute_gap":0.4673655927181244,"lower":0.5,"upper":0.6,"upper_inclusive":false},{"index":6,"count":2,"confidence":0.6151865422725677,"empirical_correctness":1.0,"signed_gap":-0.38481345772743225,"absolute_gap":0.38481345772743225,"lower":0.6,"upper":0.7,"upper_inclusive":false},{"index":7,"count":4,"confidence":0.7593721002340317,"empirical_correctness":0.25,"signed_gap":0.5093721002340317,"absolute_gap":0.5093721002340317,"lower":0.7,"upper":0.8,"upper_inclusive":false},{"index":8,"count":12,"confidence":0.8620901703834534,"empirical_correctness":0.5,"signed_gap":0.36209017038345337,"absolute_gap":0.36209017038345337,"lower":0.8,"upper":0.9,"upper_inclusive":false},{"index":9,"count":6,"confidence":0.9082193374633789,"empirical_correctness":0.6666666666666666,"signed_gap":0.24155267079671228,"absolute_gap":0.24155267079671228,"lower":0.9,"upper":1.0,"upper_inclusive":true}]},"equal_mass":{"n":38,"bin_count":10,"ece":0.24186815086163974,"signed_mean_gap":0.1585314767925363,"absolute_mean_gap":0.1585314767925363,"bins":[{"index":0,"count":3,"confidence":0.20306406915187836,"empirical_correctness":0.3333333333333333,"signed_gap":-0.13026926418145496,"absolute_gap":0.13026926418145496,"min_confidence":0.19377794861793518,"max_confidence":0.21036724746227264},{"index":1,"count":4,"confidence":0.28937388584017754,"empirical_correctness":0.25,"signed_gap":0.039373885840177536,"absolute_gap":0.039373885840177536,"min_confidence":0.2139846235513687,"max_confidence":0.36233747005462646},{"index":2,"count":4,"confidence":0.4085938557982445,"empirical_correctness":0.5,"signed_gap":-0.09140614420175552,"absolute_gap":0.09140614420175552,"min_confidence":0.3768385350704193,"max_confidence":0.48017942905426025},{"index":3,"count":4,"confidence":0.5432588905096054,"empirical_correctness":0.75,"signed_gap":-0.2067411094903946,"absolute_gap":0.2067411094903946,"min_confidence":0.49560457468032837,"max_confidence":0.612162172794342},{"index":4,"count":4,"confidence":0.7172184437513351,"empirical_correctness":0.25,"signed_gap":0.46721844375133514,"absolute_gap":0.46721844375133514,"min_confidence":0.6182109117507935,"max_confidence":0.7778074145317078},{"index":5,"count":3,"confidence":0.8094211419423422,"empirical_correctness":0.6666666666666666,"signed_gap":0.14275447527567553,"absolute_gap":0.14275447527567553,"min_confidence":0.7868255376815796,"max_confidence":0.8243781328201294},{"index":6,"count":4,"confidence":0.8489187359809875,"empirical_correctness":0.5,"signed_gap":0.34891873598098755,"absolute_gap":0.34891873598098755,"min_confidence":0.8248232007026672,"max_confidence":0.8622633814811707},{"index":7,"count":4,"confidence":0.8790636956691742,"empirical_correctness":0.75,"signed_gap":0.1290636956691742,"absolute_gap":0.1290636956691742,"min_confidence":0.8653526306152344,"max_confidence":0.8872765302658081},{"index":8,"count":4,"confidence":0.898961752653122,"empirical_correctness":0.25,"signed_gap":0.648961752653122,"absolute_gap":0.648961752653122,"min_confidence":0.8938862681388855,"max_confidence":0.9033029675483704},{"index":9,"count":4,"confidence":0.9112958610057831,"empirical_correctness":0.75,"signed_gap":0.16129586100578308,"absolute_gap":0.16129586100578308,"min_confidence":0.9063014984130859,"max_confidence":0.918705403804779}]}},"latency_ms":{"n":43,"median":8.436764997895807,"p95":9.008267970057204},"per_action":{"keep":{"n":12,"raw":{"n":12,"acceptable":3,"unsafe":0,"review":0,"automated_count":12,"automated_acceptable":3,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":0.25,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.25,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.08894166839405471,"upper":0.5323053349335657,"confidence_level":0.95}},"effective":{"n":12,"acceptable":3,"unsafe":0,"review":3,"automated_count":9,"automated_acceptable":3,"automated_unsafe":0,"unnecessary_reviews":3,"accuracy":0.25,"unsafe_rate":0.0,"review_rate":0.25,"automated_coverage":0.75,"conditional_correctness":0.3333333333333333,"conditional_unsafe":0.0,"unnecessary_review_rate":0.25,"accuracy_wilson_95":{"lower":0.08894166839405471,"upper":0.5323053349335657,"confidence_level":0.95}}},"drop":{"n":16,"raw":{"n":16,"acceptable":14,"unsafe":2,"review":0,"automated_count":16,"automated_acceptable":14,"automated_unsafe":2,"unnecessary_reviews":0,"accuracy":0.875,"unsafe_rate":0.125,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.875,"conditional_unsafe":0.125,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.639771727342413,"upper":0.9650225122567595,"confidence_level":0.95}},"effective":{"n":16,"acceptable":12,"unsafe":0,"review":4,"automated_count":12,"automated_acceptable":12,"automated_unsafe":0,"unnecessary_reviews":4,"accuracy":0.75,"unsafe_rate":0.0,"review_rate":0.25,"automated_coverage":0.75,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.25,"accuracy_wilson_95":{"lower":0.5050168346449122,"upper":0.8981793250878696,"confidence_level":0.95}}},"mask":{"n":2,"raw":{"n":2,"acceptable":0,"unsafe":1,"review":0,"automated_count":2,"automated_acceptable":0,"automated_unsafe":1,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":0.5,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":0.5,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.6576197724933469,"confidence_level":0.95}},"effective":{"n":2,"acceptable":0,"unsafe":0,"review":1,"automated_count":1,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":1,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":0.5,"automated_coverage":0.5,"conditional_correctness":0.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.5,"accuracy_wilson_95":{"lower":0.0,"upper":0.6576197724933469,"confidence_level":0.95}}},"pseudonymize":{"n":2,"raw":{"n":2,"acceptable":0,"unsafe":0,"review":0,"automated_count":2,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.6576197724933469,"confidence_level":0.95}},"effective":{"n":2,"acceptable":0,"unsafe":0,"review":0,"automated_count":2,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.6576197724933469,"confidence_level":0.95}}},"generalize":{"n":6,"raw":{"n":6,"acceptable":2,"unsafe":0,"review":0,"automated_count":6,"automated_acceptable":2,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":0.3333333333333333,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.3333333333333333,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.09677141110578041,"upper":0.700006684861608,"confidence_level":0.95}},"effective":{"n":6,"acceptable":0,"unsafe":0,"review":4,"automated_count":2,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":4,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":0.6666666666666666,"automated_coverage":0.3333333333333333,"conditional_correctness":0.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.6666666666666666,"accuracy_wilson_95":{"lower":2.7755575615628914e-17,"upper":0.3903342879021653,"confidence_level":0.95}}},"review":{"n":0,"raw":{"n":0,"acceptable":0,"unsafe":0,"review":0,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":null,"unsafe_rate":null,"review_rate":null,"automated_coverage":null,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":null},"effective":{"n":0,"acceptable":0,"unsafe":0,"review":0,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":null,"unsafe_rate":null,"review_rate":null,"automated_coverage":null,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":null}}}},"diagnostic43-compact_no_samples":{"n":43,"raw":{"n":43,"acceptable":17,"unsafe":2,"review":7,"automated_count":36,"automated_acceptable":16,"automated_unsafe":2,"unnecessary_reviews":6,"accuracy":0.3953488372093023,"unsafe_rate":0.046511627906976744,"review_rate":0.16279069767441862,"automated_coverage":0.8372093023255814,"conditional_correctness":0.4444444444444444,"conditional_unsafe":0.05555555555555555,"unnecessary_review_rate":0.13953488372093023,"accuracy_wilson_95":{"lower":0.2636531475829744,"upper":0.5442093693712609,"confidence_level":0.95}},"effective":{"n":43,"acceptable":14,"unsafe":0,"review":15,"automated_count":28,"automated_acceptable":12,"automated_unsafe":0,"unnecessary_reviews":13,"accuracy":0.32558139534883723,"unsafe_rate":0.0,"review_rate":0.3488372093023256,"automated_coverage":0.6511627906976745,"conditional_correctness":0.42857142857142855,"conditional_unsafe":0.0,"unnecessary_review_rate":0.3023255813953488,"accuracy_wilson_95":{"lower":0.20493297126568708,"upper":0.47483789032470525,"confidence_level":0.95}},"confusion":{"raw":{"keep":{"keep":0,"drop":10,"mask":0,"pseudonymize":0,"generalize":0,"review":2},"drop":{"keep":0,"drop":12,"mask":0,"pseudonymize":0,"generalize":0,"review":4},"mask":{"keep":0,"drop":2,"mask":0,"pseudonymize":0,"generalize":0,"review":0},"pseudonymize":{"keep":0,"drop":2,"mask":0,"pseudonymize":0,"generalize":0,"review":0},"generalize":{"keep":2,"drop":4,"mask":0,"pseudonymize":0,"generalize":0,"review":0},"review":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0}},"effective":{"keep":{"keep":0,"drop":9,"mask":0,"pseudonymize":0,"generalize":0,"review":3},"drop":{"keep":0,"drop":9,"mask":0,"pseudonymize":0,"generalize":0,"review":7},"mask":{"keep":0,"drop":2,"mask":0,"pseudonymize":0,"generalize":0,"review":0},"pseudonymize":{"keep":0,"drop":2,"mask":0,"pseudonymize":0,"generalize":0,"review":0},"generalize":{"keep":0,"drop":3,"mask":0,"pseudonymize":0,"generalize":0,"review":3},"review":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0}}},"probability_metrics":{"singleton_n":38,"multiacceptable_n":5,"multiacceptable_coverage":0.11627906976744186,"nll":2.3320258876340705,"nll_zero_mass_n":0,"brier":1.103328389069112,"acceptable_set_n":43,"acceptable_set_nll":2.0923813069558297,"acceptable_set_zero_mass_n":0},"calibration_all_labels":{"equal_width":{"n":43,"bin_count":10,"ece":0.47256361779778505,"signed_mean_gap":0.34280698347923366,"absolute_mean_gap":0.34280698347923366,"bins":[{"index":0,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.0,"upper":0.1,"upper_inclusive":false},{"index":1,"count":3,"confidence":0.19604026277860007,"empirical_correctness":0.3333333333333333,"signed_gap":-0.13729307055473325,"absolute_gap":0.13729307055473325,"lower":0.1,"upper":0.2,"upper_inclusive":false},{"index":2,"count":4,"confidence":0.2531845085322857,"empirical_correctness":0.75,"signed_gap":-0.4968154914677143,"absolute_gap":0.4968154914677143,"lower":0.2,"upper":0.3,"upper_inclusive":false},{"index":3,"count":2,"confidence":0.3046867698431015,"empirical_correctness":0.5,"signed_gap":-0.1953132301568985,"absolute_gap":0.1953132301568985,"lower":0.3,"upper":0.4,"upper_inclusive":false},{"index":4,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.4,"upper":0.5,"upper_inclusive":false},{"index":5,"count":1,"confidence":0.5564959049224854,"empirical_correctness":0.0,"signed_gap":0.5564959049224854,"absolute_gap":0.5564959049224854,"lower":0.5,"upper":0.6,"upper_inclusive":false},{"index":6,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.6,"upper":0.7,"upper_inclusive":false},{"index":7,"count":4,"confidence":0.7700469493865967,"empirical_correctness":0.0,"signed_gap":0.7700469493865967,"absolute_gap":0.7700469493865967,"lower":0.7,"upper":0.8,"upper_inclusive":false},{"index":8,"count":13,"confidence":0.858934957247514,"empirical_correctness":0.3076923076923077,"signed_gap":0.5512426495552063,"absolute_gap":0.5512426495552063,"lower":0.8,"upper":0.9,"upper_inclusive":false},{"index":9,"count":16,"confidence":0.9204768612980843,"empirical_correctness":0.5,"signed_gap":0.42047686129808426,"absolute_gap":0.42047686129808426,"lower":0.9,"upper":1.0,"upper_inclusive":true}]},"equal_mass":{"n":43,"bin_count":10,"ece":0.44023123626099075,"signed_mean_gap":0.34280698347923366,"absolute_mean_gap":0.34280698347923366,"bins":[{"index":0,"count":4,"confidence":0.1975288726389408,"empirical_correctness":0.5,"signed_gap":-0.3024711273610592,"absolute_gap":0.3024711273610592,"min_confidence":0.1930011808872223,"max_confidence":0.20199470221996307},{"index":1,"count":4,"confidence":0.27881576865911484,"empirical_correctness":0.5,"signed_gap":-0.22118423134088516,"absolute_gap":0.22118423134088516,"min_confidence":0.25199541449546814,"max_confidence":0.30451974272727966},{"index":2,"count":4,"confidence":0.590239942073822,"empirical_correctness":0.25,"signed_gap":0.340239942073822,"absolute_gap":0.340239942073822,"min_confidence":0.30485379695892334,"max_confidence":0.7841169834136963},{"index":3,"count":5,"confidence":0.8100226640701294,"empirical_correctness":0.2,"signed_gap":0.6100226640701294,"absolute_gap":0.6100226640701294,"min_confidence":0.7885826826095581,"max_confidence":0.8306629657745361},{"index":4,"count":4,"confidence":0.8573464751243591,"empirical_correctness":0.25,"signed_gap":0.6073464751243591,"absolute_gap":0.6073464751243591,"min_confidence":0.8538807034492493,"max_confidence":0.8634387850761414},{"index":5,"count":4,"confidence":0.873103529214859,"empirical_correctness":0.25,"signed_gap":0.623103529214859,"absolute_gap":0.623103529214859,"min_confidence":0.8649097084999084,"max_confidence":0.8824483752250671},{"index":6,"count":5,"confidence":0.8966139793395996,"empirical_correctness":0.6,"signed_gap":0.29661397933959965,"absolute_gap":0.29661397933959965,"min_confidence":0.8843344449996948,"max_confidence":0.9055969715118408},{"index":7,"count":4,"confidence":0.9091062545776367,"empirical_correctness":0.25,"signed_gap":0.6591062545776367,"absolute_gap":0.6591062545776367,"min_confidence":0.9061963558197021,"max_confidence":0.9120896458625793},{"index":8,"count":4,"confidence":0.928561732172966,"empirical_correctness":0.5,"signed_gap":0.428561732172966,"absolute_gap":0.428561732172966,"min_confidence":0.9236929416656494,"max_confidence":0.9317876100540161},{"index":9,"count":5,"confidence":0.9337413549423218,"empirical_correctness":0.6,"signed_gap":0.3337413549423218,"absolute_gap":0.3337413549423218,"min_confidence":0.9328271746635437,"max_confidence":0.9364843368530273}]}},"calibration_singleton_labels":{"equal_width":{"n":38,"bin_count":10,"ece":0.5031170703862843,"signed_mean_gap":0.4356539453330793,"absolute_mean_gap":0.4356539453330793,"bins":[{"index":0,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.0,"upper":0.1,"upper_inclusive":false},{"index":1,"count":3,"confidence":0.19604026277860007,"empirical_correctness":0.3333333333333333,"signed_gap":-0.13729307055473325,"absolute_gap":0.13729307055473325,"lower":0.1,"upper":0.2,"upper_inclusive":false},{"index":2,"count":2,"confidence":0.260353147983551,"empirical_correctness":0.5,"signed_gap":-0.23964685201644897,"absolute_gap":0.23964685201644897,"lower":0.2,"upper":0.3,"upper_inclusive":false},{"index":3,"count":2,"confidence":0.3046867698431015,"empirical_correctness":0.5,"signed_gap":-0.1953132301568985,"absolute_gap":0.1953132301568985,"lower":0.3,"upper":0.4,"upper_inclusive":false},{"index":4,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.4,"upper":0.5,"upper_inclusive":false},{"index":5,"count":1,"confidence":0.5564959049224854,"empirical_correctness":0.0,"signed_gap":0.5564959049224854,"absolute_gap":0.5564959049224854,"lower":0.5,"upper":0.6,"upper_inclusive":false},{"index":6,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.6,"upper":0.7,"upper_inclusive":false},{"index":7,"count":4,"confidence":0.7700469493865967,"empirical_correctness":0.0,"signed_gap":0.7700469493865967,"absolute_gap":0.7700469493865967,"lower":0.7,"upper":0.8,"upper_inclusive":false},{"index":8,"count":12,"confidence":0.8592299968004227,"empirical_correctness":0.25,"signed_gap":0.6092299968004227,"absolute_gap":0.6092299968004227,"lower":0.8,"upper":0.9,"upper_inclusive":false},{"index":9,"count":14,"confidence":0.9206575453281403,"empirical_correctness":0.42857142857142855,"signed_gap":0.4920861167567117,"absolute_gap":0.4920861167567117,"lower":0.9,"upper":1.0,"upper_inclusive":true}]},"equal_mass":{"n":38,"bin_count":10,"ece":0.5031170703862843,"signed_mean_gap":0.4356539453330793,"absolute_mean_gap":0.4356539453330793,"bins":[{"index":0,"count":3,"confidence":0.19604026277860007,"empirical_correctness":0.3333333333333333,"signed_gap":-0.13729307055473325,"absolute_gap":0.13729307055473325,"min_confidence":0.1930011808872223,"max_confidence":0.19991609454154968},{"index":1,"count":4,"confidence":0.28251995891332626,"empirical_correctness":0.5,"signed_gap":-0.21748004108667374,"absolute_gap":0.21748004108667374,"min_confidence":0.25199541449546814,"max_confidence":0.30485379695892334},{"index":2,"count":4,"confidence":0.7111721634864807,"empirical_correctness":0.0,"signed_gap":0.7111721634864807,"absolute_gap":0.7111721634864807,"min_confidence":0.5564959049224854,"max_confidence":0.7885826826095581},{"index":3,"count":4,"confidence":0.8153826594352722,"empirical_correctness":0.25,"signed_gap":0.5653826594352722,"absolute_gap":0.5653826594352722,"min_confidence":0.7919950485229492,"max_confidence":0.8306629657745361},{"index":4,"count":4,"confidence":0.8597252815961838,"empirical_correctness":0.0,"signed_gap":0.8597252815961838,"absolute_gap":0.8597252815961838,"min_confidence":0.8538807034492493,"max_confidence":0.8649097084999084},{"index":5,"count":3,"confidence":0.8758348027865092,"empirical_correctness":0.3333333333333333,"signed_gap":0.5425014694531758,"absolute_gap":0.5425014694531758,"min_confidence":0.8672317862510681,"max_confidence":0.8824483752250671},{"index":6,"count":4,"confidence":0.8943682312965393,"empirical_correctness":0.5,"signed_gap":0.3943682312965393,"absolute_gap":0.3943682312965393,"min_confidence":0.8843344449996948,"max_confidence":0.901957094669342},{"index":7,"count":4,"confidence":0.9091062545776367,"empirical_correctness":0.25,"signed_gap":0.6591062545776367,"absolute_gap":0.6591062545776367,"min_confidence":0.9061963558197021,"max_confidence":0.9120896458625793},{"index":8,"count":4,"confidence":0.928561732172966,"empirical_correctness":0.5,"signed_gap":0.428561732172966,"absolute_gap":0.428561732172966,"min_confidence":0.9236929416656494,"max_confidence":0.9317876100540161},{"index":9,"count":4,"confidence":0.9339699000120163,"empirical_correctness":0.5,"signed_gap":0.4339699000120163,"absolute_gap":0.4339699000120163,"min_confidence":0.9330558776855469,"max_confidence":0.9364843368530273}]}},"latency_ms":{"n":43,"median":8.66218499140814,"p95":9.259952959837392},"per_action":{"keep":{"n":12,"raw":{"n":12,"acceptable":0,"unsafe":0,"review":2,"automated_count":10,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":2,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":0.16666666666666666,"automated_coverage":0.8333333333333334,"conditional_correctness":0.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.16666666666666666,"accuracy_wilson_95":{"lower":2.7755575615628914e-17,"upper":0.24249400665524085,"confidence_level":0.95}},"effective":{"n":12,"acceptable":0,"unsafe":0,"review":3,"automated_count":9,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":3,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":0.25,"automated_coverage":0.75,"conditional_correctness":0.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.25,"accuracy_wilson_95":{"lower":2.7755575615628914e-17,"upper":0.24249400665524085,"confidence_level":0.95}}},"drop":{"n":16,"raw":{"n":16,"acceptable":12,"unsafe":0,"review":4,"automated_count":12,"automated_acceptable":12,"automated_unsafe":0,"unnecessary_reviews":4,"accuracy":0.75,"unsafe_rate":0.0,"review_rate":0.25,"automated_coverage":0.75,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.25,"accuracy_wilson_95":{"lower":0.5050168346449122,"upper":0.8981793250878696,"confidence_level":0.95}},"effective":{"n":16,"acceptable":9,"unsafe":0,"review":7,"automated_count":9,"automated_acceptable":9,"automated_unsafe":0,"unnecessary_reviews":7,"accuracy":0.5625,"unsafe_rate":0.0,"review_rate":0.4375,"automated_coverage":0.5625,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.4375,"accuracy_wilson_95":{"lower":0.331785563988119,"upper":0.7690134759450765,"confidence_level":0.95}}},"mask":{"n":2,"raw":{"n":2,"acceptable":0,"unsafe":0,"review":0,"automated_count":2,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.6576197724933469,"confidence_level":0.95}},"effective":{"n":2,"acceptable":0,"unsafe":0,"review":0,"automated_count":2,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.6576197724933469,"confidence_level":0.95}}},"pseudonymize":{"n":2,"raw":{"n":2,"acceptable":0,"unsafe":0,"review":0,"automated_count":2,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.6576197724933469,"confidence_level":0.95}},"effective":{"n":2,"acceptable":0,"unsafe":0,"review":0,"automated_count":2,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.6576197724933469,"confidence_level":0.95}}},"generalize":{"n":6,"raw":{"n":6,"acceptable":0,"unsafe":2,"review":0,"automated_count":6,"automated_acceptable":0,"automated_unsafe":2,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":0.3333333333333333,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":0.3333333333333333,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":2.7755575615628914e-17,"upper":0.3903342879021653,"confidence_level":0.95}},"effective":{"n":6,"acceptable":0,"unsafe":0,"review":3,"automated_count":3,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":3,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":0.5,"automated_coverage":0.5,"conditional_correctness":0.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.5,"accuracy_wilson_95":{"lower":2.7755575615628914e-17,"upper":0.3903342879021653,"confidence_level":0.95}}},"review":{"n":0,"raw":{"n":0,"acceptable":0,"unsafe":0,"review":0,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":null,"unsafe_rate":null,"review_rate":null,"automated_coverage":null,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":null},"effective":{"n":0,"acceptable":0,"unsafe":0,"review":0,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":null,"unsafe_rate":null,"review_rate":null,"automated_coverage":null,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":null}}}},"challenge56-no_samples":{"n":56,"raw":{"n":56,"acceptable":22,"unsafe":3,"review":1,"automated_count":55,"automated_acceptable":22,"automated_unsafe":3,"unnecessary_reviews":1,"accuracy":0.39285714285714285,"unsafe_rate":0.05357142857142857,"review_rate":0.017857142857142856,"automated_coverage":0.9821428571428571,"conditional_correctness":0.4,"conditional_unsafe":0.05454545454545454,"unnecessary_review_rate":0.017857142857142856,"accuracy_wilson_95":{"lower":0.2758041573011051,"upper":0.5236659719957729,"confidence_level":0.95}},"effective":{"n":56,"acceptable":20,"unsafe":0,"review":13,"automated_count":43,"automated_acceptable":20,"automated_unsafe":0,"unnecessary_reviews":13,"accuracy":0.35714285714285715,"unsafe_rate":0.0,"review_rate":0.23214285714285715,"automated_coverage":0.7678571428571429,"conditional_correctness":0.46511627906976744,"conditional_unsafe":0.0,"unnecessary_review_rate":0.23214285714285715,"accuracy_wilson_95":{"lower":0.24456571707649905,"upper":0.4880611219860049,"confidence_level":0.95}},"confusion":{"raw":{"keep":{"keep":0,"drop":7,"mask":0,"pseudonymize":1,"generalize":0,"review":0},"drop":{"keep":2,"drop":19,"mask":0,"pseudonymize":0,"generalize":0,"review":1},"mask":{"keep":1,"drop":7,"mask":0,"pseudonymize":0,"generalize":0,"review":0},"pseudonymize":{"keep":0,"drop":6,"mask":0,"pseudonymize":0,"generalize":2,"review":0},"generalize":{"keep":0,"drop":5,"mask":0,"pseudonymize":0,"generalize":3,"review":0},"review":{"keep":0,"drop":2,"mask":0,"pseudonymize":0,"generalize":0,"review":0}},"effective":{"keep":{"keep":0,"drop":6,"mask":0,"pseudonymize":0,"generalize":0,"review":2},"drop":{"keep":0,"drop":17,"mask":0,"pseudonymize":0,"generalize":0,"review":5},"mask":{"keep":0,"drop":6,"mask":0,"pseudonymize":0,"generalize":0,"review":2},"pseudonymize":{"keep":0,"drop":6,"mask":0,"pseudonymize":0,"generalize":0,"review":2},"generalize":{"keep":0,"drop":3,"mask":0,"pseudonymize":0,"generalize":3,"review":2},"review":{"keep":0,"drop":2,"mask":0,"pseudonymize":0,"generalize":0,"review":0}}},"probability_metrics":{"singleton_n":56,"multiacceptable_n":0,"multiacceptable_coverage":0.0,"nll":1.777908840123113,"nll_zero_mass_n":0,"brier":0.8011803614102914,"acceptable_set_n":56,"acceptable_set_nll":1.777908840123113,"acceptable_set_zero_mass_n":0},"calibration_all_labels":{"equal_width":{"n":56,"bin_count":10,"ece":0.3076226634106466,"signed_mean_gap":0.2856681950922523,"absolute_mean_gap":0.2856681950922523,"bins":[{"index":0,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.0,"upper":0.1,"upper_inclusive":false},{"index":1,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.1,"upper":0.2,"upper_inclusive":false},{"index":2,"count":5,"confidence":0.25053163468837736,"empirical_correctness":0.0,"signed_gap":0.25053163468837736,"absolute_gap":0.25053163468837736,"lower":0.2,"upper":0.3,"upper_inclusive":false},{"index":3,"count":4,"confidence":0.3572756350040436,"empirical_correctness":0.5,"signed_gap":-0.14272436499595642,"absolute_gap":0.14272436499595642,"lower":0.3,"upper":0.4,"upper_inclusive":false},{"index":4,"count":6,"confidence":0.4588451286156972,"empirical_correctness":0.16666666666666666,"signed_gap":0.2921784619490305,"absolute_gap":0.2921784619490305,"lower":0.4,"upper":0.5,"upper_inclusive":false},{"index":5,"count":6,"confidence":0.5651054183642069,"empirical_correctness":0.0,"signed_gap":0.5651054183642069,"absolute_gap":0.5651054183642069,"lower":0.5,"upper":0.6,"upper_inclusive":false},{"index":6,"count":6,"confidence":0.6485948264598846,"empirical_correctness":0.0,"signed_gap":0.6485948264598846,"absolute_gap":0.6485948264598846,"lower":0.6,"upper":0.7,"upper_inclusive":false},{"index":7,"count":4,"confidence":0.7401220798492432,"empirical_correctness":0.0,"signed_gap":0.7401220798492432,"absolute_gap":0.7401220798492432,"lower":0.7,"upper":0.8,"upper_inclusive":false},{"index":8,"count":12,"confidence":0.8636437753836314,"empirical_correctness":0.5833333333333334,"signed_gap":0.280310442050298,"absolute_gap":0.280310442050298,"lower":0.8,"upper":0.9,"upper_inclusive":false},{"index":9,"count":13,"confidence":0.9197055651591375,"empirical_correctness":0.9230769230769231,"signed_gap":-0.0033713579177856445,"absolute_gap":0.0033713579177856445,"lower":0.9,"upper":1.0,"upper_inclusive":true}]},"equal_mass":{"n":56,"bin_count":10,"ece":0.30113170695091995,"signed_mean_gap":0.28566819509225233,"absolute_mean_gap":0.28566819509225233,"bins":[{"index":0,"count":5,"confidence":0.25053163468837736,"empirical_correctness":0.0,"signed_gap":0.25053163468837736,"absolute_gap":0.25053163468837736,"min_confidence":0.2199295163154602,"max_confidence":0.29850542545318604},{"index":1,"count":6,"confidence":0.37491479019323987,"empirical_correctness":0.3333333333333333,"signed_gap":0.04158145685990655,"absolute_gap":0.04158145685990655,"min_confidence":0.31461790204048157,"max_confidence":0.4128859043121338},{"index":2,"count":5,"confidence":0.4924896895885468,"empirical_correctness":0.2,"signed_gap":0.29248968958854676,"absolute_gap":0.29248968958854676,"min_confidence":0.45177996158599854,"max_confidence":0.5297638773918152},{"index":3,"count":6,"confidence":0.5779039661089579,"empirical_correctness":0.0,"signed_gap":0.5779039661089579,"absolute_gap":0.5779039661089579,"min_confidence":0.5495904684066772,"max_confidence":0.606555163860321},{"index":4,"count":6,"confidence":0.6660536130269369,"empirical_correctness":0.0,"signed_gap":0.6660536130269369,"absolute_gap":0.6660536130269369,"min_confidence":0.6304386854171753,"max_confidence":0.7113078832626343},{"index":5,"count":5,"confidence":0.7736122608184814,"empirical_correctness":0.4,"signed_gap":0.3736122608184814,"absolute_gap":0.3736122608184814,"min_confidence":0.7355962991714478,"max_confidence":0.8163962960243225},{"index":6,"count":6,"confidence":0.8644458552201589,"empirical_correctness":0.5,"signed_gap":0.36444585522015893,"absolute_gap":0.36444585522015893,"min_confidence":0.8369182348251343,"max_confidence":0.8830894231796265},{"index":7,"count":5,"confidence":0.8920187830924988,"empirical_correctness":0.6,"signed_gap":0.29201878309249885,"absolute_gap":0.29201878309249885,"min_confidence":0.8860782384872437,"max_confidence":0.9019246101379395},{"index":8,"count":6,"confidence":0.9145376781622568,"empirical_correctness":0.8333333333333334,"signed_gap":0.08120434482892347,"absolute_gap":0.08120434482892347,"min_confidence":0.9059269428253174,"max_confidence":0.921344518661499},{"index":9,"count":6,"confidence":0.927836944659551,"empirical_correctness":1.0,"signed_gap":-0.07216305534044898,"absolute_gap":0.07216305534044898,"min_confidence":0.9228033423423767,"max_confidence":0.9334819912910461}]}},"calibration_singleton_labels":{"equal_width":{"n":56,"bin_count":10,"ece":0.3076226634106466,"signed_mean_gap":0.2856681950922523,"absolute_mean_gap":0.2856681950922523,"bins":[{"index":0,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.0,"upper":0.1,"upper_inclusive":false},{"index":1,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.1,"upper":0.2,"upper_inclusive":false},{"index":2,"count":5,"confidence":0.25053163468837736,"empirical_correctness":0.0,"signed_gap":0.25053163468837736,"absolute_gap":0.25053163468837736,"lower":0.2,"upper":0.3,"upper_inclusive":false},{"index":3,"count":4,"confidence":0.3572756350040436,"empirical_correctness":0.5,"signed_gap":-0.14272436499595642,"absolute_gap":0.14272436499595642,"lower":0.3,"upper":0.4,"upper_inclusive":false},{"index":4,"count":6,"confidence":0.4588451286156972,"empirical_correctness":0.16666666666666666,"signed_gap":0.2921784619490305,"absolute_gap":0.2921784619490305,"lower":0.4,"upper":0.5,"upper_inclusive":false},{"index":5,"count":6,"confidence":0.5651054183642069,"empirical_correctness":0.0,"signed_gap":0.5651054183642069,"absolute_gap":0.5651054183642069,"lower":0.5,"upper":0.6,"upper_inclusive":false},{"index":6,"count":6,"confidence":0.6485948264598846,"empirical_correctness":0.0,"signed_gap":0.6485948264598846,"absolute_gap":0.6485948264598846,"lower":0.6,"upper":0.7,"upper_inclusive":false},{"index":7,"count":4,"confidence":0.7401220798492432,"empirical_correctness":0.0,"signed_gap":0.7401220798492432,"absolute_gap":0.7401220798492432,"lower":0.7,"upper":0.8,"upper_inclusive":false},{"index":8,"count":12,"confidence":0.8636437753836314,"empirical_correctness":0.5833333333333334,"signed_gap":0.280310442050298,"absolute_gap":0.280310442050298,"lower":0.8,"upper":0.9,"upper_inclusive":false},{"index":9,"count":13,"confidence":0.9197055651591375,"empirical_correctness":0.9230769230769231,"signed_gap":-0.0033713579177856445,"absolute_gap":0.0033713579177856445,"lower":0.9,"upper":1.0,"upper_inclusive":true}]},"equal_mass":{"n":56,"bin_count":10,"ece":0.30113170695091995,"signed_mean_gap":0.28566819509225233,"absolute_mean_gap":0.28566819509225233,"bins":[{"index":0,"count":5,"confidence":0.25053163468837736,"empirical_correctness":0.0,"signed_gap":0.25053163468837736,"absolute_gap":0.25053163468837736,"min_confidence":0.2199295163154602,"max_confidence":0.29850542545318604},{"index":1,"count":6,"confidence":0.37491479019323987,"empirical_correctness":0.3333333333333333,"signed_gap":0.04158145685990655,"absolute_gap":0.04158145685990655,"min_confidence":0.31461790204048157,"max_confidence":0.4128859043121338},{"index":2,"count":5,"confidence":0.4924896895885468,"empirical_correctness":0.2,"signed_gap":0.29248968958854676,"absolute_gap":0.29248968958854676,"min_confidence":0.45177996158599854,"max_confidence":0.5297638773918152},{"index":3,"count":6,"confidence":0.5779039661089579,"empirical_correctness":0.0,"signed_gap":0.5779039661089579,"absolute_gap":0.5779039661089579,"min_confidence":0.5495904684066772,"max_confidence":0.606555163860321},{"index":4,"count":6,"confidence":0.6660536130269369,"empirical_correctness":0.0,"signed_gap":0.6660536130269369,"absolute_gap":0.6660536130269369,"min_confidence":0.6304386854171753,"max_confidence":0.7113078832626343},{"index":5,"count":5,"confidence":0.7736122608184814,"empirical_correctness":0.4,"signed_gap":0.3736122608184814,"absolute_gap":0.3736122608184814,"min_confidence":0.7355962991714478,"max_confidence":0.8163962960243225},{"index":6,"count":6,"confidence":0.8644458552201589,"empirical_correctness":0.5,"signed_gap":0.36444585522015893,"absolute_gap":0.36444585522015893,"min_confidence":0.8369182348251343,"max_confidence":0.8830894231796265},{"index":7,"count":5,"confidence":0.8920187830924988,"empirical_correctness":0.6,"signed_gap":0.29201878309249885,"absolute_gap":0.29201878309249885,"min_confidence":0.8860782384872437,"max_confidence":0.9019246101379395},{"index":8,"count":6,"confidence":0.9145376781622568,"empirical_correctness":0.8333333333333334,"signed_gap":0.08120434482892347,"absolute_gap":0.08120434482892347,"min_confidence":0.9059269428253174,"max_confidence":0.921344518661499},{"index":9,"count":6,"confidence":0.927836944659551,"empirical_correctness":1.0,"signed_gap":-0.07216305534044898,"absolute_gap":0.07216305534044898,"min_confidence":0.9228033423423767,"max_confidence":0.9334819912910461}]}},"latency_ms":{"n":56,"median":6.887052033562213,"p95":7.0265620306599885},"per_action":{"keep":{"n":8,"raw":{"n":8,"acceptable":0,"unsafe":0,"review":0,"automated_count":8,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.32440756488388023,"confidence_level":0.95}},"effective":{"n":8,"acceptable":0,"unsafe":0,"review":2,"automated_count":6,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":2,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":0.25,"automated_coverage":0.75,"conditional_correctness":0.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.25,"accuracy_wilson_95":{"lower":0.0,"upper":0.32440756488388023,"confidence_level":0.95}}},"drop":{"n":22,"raw":{"n":22,"acceptable":19,"unsafe":2,"review":1,"automated_count":21,"automated_acceptable":19,"automated_unsafe":2,"unnecessary_reviews":1,"accuracy":0.8636363636363636,"unsafe_rate":0.09090909090909091,"review_rate":0.045454545454545456,"automated_coverage":0.9545454545454546,"conditional_correctness":0.9047619047619048,"conditional_unsafe":0.09523809523809523,"unnecessary_review_rate":0.045454545454545456,"accuracy_wilson_95":{"lower":0.6666501283342828,"upper":0.9525099628594474,"confidence_level":0.95}},"effective":{"n":22,"acceptable":17,"unsafe":0,"review":5,"automated_count":17,"automated_acceptable":17,"automated_unsafe":0,"unnecessary_reviews":5,"accuracy":0.7727272727272727,"unsafe_rate":0.0,"review_rate":0.22727272727272727,"automated_coverage":0.7727272727272727,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.22727272727272727,"accuracy_wilson_95":{"lower":0.5656004682477829,"upper":0.8987696001475147,"confidence_level":0.95}}},"mask":{"n":8,"raw":{"n":8,"acceptable":0,"unsafe":1,"review":0,"automated_count":8,"automated_acceptable":0,"automated_unsafe":1,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":0.125,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":0.125,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.32440756488388023,"confidence_level":0.95}},"effective":{"n":8,"acceptable":0,"unsafe":0,"review":2,"automated_count":6,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":2,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":0.25,"automated_coverage":0.75,"conditional_correctness":0.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.25,"accuracy_wilson_95":{"lower":0.0,"upper":0.32440756488388023,"confidence_level":0.95}}},"pseudonymize":{"n":8,"raw":{"n":8,"acceptable":0,"unsafe":0,"review":0,"automated_count":8,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.32440756488388023,"confidence_level":0.95}},"effective":{"n":8,"acceptable":0,"unsafe":0,"review":2,"automated_count":6,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":2,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":0.25,"automated_coverage":0.75,"conditional_correctness":0.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.25,"accuracy_wilson_95":{"lower":0.0,"upper":0.32440756488388023,"confidence_level":0.95}}},"generalize":{"n":8,"raw":{"n":8,"acceptable":3,"unsafe":0,"review":0,"automated_count":8,"automated_acceptable":3,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":0.375,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.375,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.13684428582359742,"upper":0.6942576053973727,"confidence_level":0.95}},"effective":{"n":8,"acceptable":3,"unsafe":0,"review":2,"automated_count":6,"automated_acceptable":3,"automated_unsafe":0,"unnecessary_reviews":2,"accuracy":0.375,"unsafe_rate":0.0,"review_rate":0.25,"automated_coverage":0.75,"conditional_correctness":0.5,"conditional_unsafe":0.0,"unnecessary_review_rate":0.25,"accuracy_wilson_95":{"lower":0.13684428582359742,"upper":0.6942576053973727,"confidence_level":0.95}}},"review":{"n":2,"raw":{"n":2,"acceptable":0,"unsafe":0,"review":0,"automated_count":2,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.6576197724933469,"confidence_level":0.95}},"effective":{"n":2,"acceptable":0,"unsafe":0,"review":0,"automated_count":2,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.6576197724933469,"confidence_level":0.95}}}}},"challenge56-compact_no_samples":{"n":56,"raw":{"n":56,"acceptable":20,"unsafe":9,"review":1,"automated_count":55,"automated_acceptable":20,"automated_unsafe":9,"unnecessary_reviews":1,"accuracy":0.35714285714285715,"unsafe_rate":0.16071428571428573,"review_rate":0.017857142857142856,"automated_coverage":0.9821428571428571,"conditional_correctness":0.36363636363636365,"conditional_unsafe":0.16363636363636364,"unnecessary_review_rate":0.017857142857142856,"accuracy_wilson_95":{"lower":0.24456571707649905,"upper":0.4880611219860049,"confidence_level":0.95}},"effective":{"n":56,"acceptable":18,"unsafe":0,"review":20,"automated_count":36,"automated_acceptable":18,"automated_unsafe":0,"unnecessary_reviews":20,"accuracy":0.32142857142857145,"unsafe_rate":0.0,"review_rate":0.35714285714285715,"automated_coverage":0.6428571428571429,"conditional_correctness":0.5,"conditional_unsafe":0.0,"unnecessary_review_rate":0.35714285714285715,"accuracy_wilson_95":{"lower":0.21400994792551523,"upper":0.45177360090261476,"confidence_level":0.95}},"confusion":{"raw":{"keep":{"keep":0,"drop":8,"mask":0,"pseudonymize":0,"generalize":0,"review":0},"drop":{"keep":2,"drop":19,"mask":0,"pseudonymize":0,"generalize":0,"review":1},"mask":{"keep":5,"drop":3,"mask":0,"pseudonymize":0,"generalize":0,"review":0},"pseudonymize":{"keep":1,"drop":6,"mask":1,"pseudonymize":0,"generalize":0,"review":0},"generalize":{"keep":1,"drop":6,"mask":0,"pseudonymize":0,"generalize":1,"review":0},"review":{"keep":0,"drop":2,"mask":0,"pseudonymize":0,"generalize":0,"review":0}},"effective":{"keep":{"keep":0,"drop":6,"mask":0,"pseudonymize":0,"generalize":0,"review":2},"drop":{"keep":0,"drop":17,"mask":0,"pseudonymize":0,"generalize":0,"review":5},"mask":{"keep":0,"drop":3,"mask":0,"pseudonymize":0,"generalize":0,"review":5},"pseudonymize":{"keep":0,"drop":3,"mask":0,"pseudonymize":0,"generalize":0,"review":5},"generalize":{"keep":0,"drop":4,"mask":0,"pseudonymize":0,"generalize":1,"review":3},"review":{"keep":0,"drop":2,"mask":0,"pseudonymize":0,"generalize":0,"review":0}}},"probability_metrics":{"singleton_n":56,"multiacceptable_n":0,"multiacceptable_coverage":0.0,"nll":1.690364439221537,"nll_zero_mass_n":0,"brier":0.7891415529613284,"acceptable_set_n":56,"acceptable_set_nll":1.690364439221537,"acceptable_set_zero_mass_n":0},"calibration_all_labels":{"equal_width":{"n":56,"bin_count":10,"ece":0.2689618154295853,"signed_mean_gap":0.2689618154295853,"absolute_mean_gap":0.2689618154295853,"bins":[{"index":0,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.0,"upper":0.1,"upper_inclusive":false},{"index":1,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.1,"upper":0.2,"upper_inclusive":false},{"index":2,"count":10,"confidence":0.275234517455101,"empirical_correctness":0.2,"signed_gap":0.07523451745510101,"absolute_gap":0.07523451745510101,"lower":0.2,"upper":0.3,"upper_inclusive":false},{"index":3,"count":9,"confidence":0.33160651061269975,"empirical_correctness":0.0,"signed_gap":0.33160651061269975,"absolute_gap":0.33160651061269975,"lower":0.3,"upper":0.4,"upper_inclusive":false},{"index":4,"count":3,"confidence":0.45179946223894757,"empirical_correctness":0.0,"signed_gap":0.45179946223894757,"absolute_gap":0.45179946223894757,"lower":0.4,"upper":0.5,"upper_inclusive":false},{"index":5,"count":3,"confidence":0.5559185942014059,"empirical_correctness":0.0,"signed_gap":0.5559185942014059,"absolute_gap":0.5559185942014059,"lower":0.5,"upper":0.6,"upper_inclusive":false},{"index":6,"count":2,"confidence":0.6767387390136719,"empirical_correctness":0.5,"signed_gap":0.17673873901367188,"absolute_gap":0.17673873901367188,"lower":0.6,"upper":0.7,"upper_inclusive":false},{"index":7,"count":5,"confidence":0.7584225296974182,"empirical_correctness":0.4,"signed_gap":0.35842252969741817,"absolute_gap":0.35842252969741817,"lower":0.7,"upper":0.8,"upper_inclusive":false},{"index":8,"count":14,"confidence":0.8552950492927006,"empirical_correctness":0.7142857142857143,"signed_gap":0.1410093350069863,"absolute_gap":0.1410093350069863,"lower":0.8,"upper":0.9,"upper_inclusive":false},{"index":9,"count":10,"confidence":0.9182182908058166,"empirical_correctness":0.5,"signed_gap":0.41821829080581663,"absolute_gap":0.41821829080581663,"lower":0.9,"upper":1.0,"upper_inclusive":true}]},"equal_mass":{"n":56,"bin_count":10,"ece":0.2763793920832021,"signed_mean_gap":0.26896181542958536,"absolute_mean_gap":0.26896181542958536,"bins":[{"index":0,"count":5,"confidence":0.2537809908390045,"empirical_correctness":0.0,"signed_gap":0.2537809908390045,"absolute_gap":0.2537809908390045,"min_confidence":0.21950747072696686,"max_confidence":0.29090914130210876},{"index":1,"count":6,"confidence":0.2987179756164551,"empirical_correctness":0.3333333333333333,"signed_gap":-0.03461535771687824,"absolute_gap":0.03461535771687824,"min_confidence":0.29377642273902893,"max_confidence":0.3088676333427429},{"index":2,"count":5,"confidence":0.3258474290370941,"empirical_correctness":0.0,"signed_gap":0.3258474290370941,"absolute_gap":0.3258474290370941,"min_confidence":0.3125802278518677,"max_confidence":0.33409881591796875},{"index":3,"count":6,"confidence":0.4002920339504878,"empirical_correctness":0.0,"signed_gap":0.4002920339504878,"absolute_gap":0.4002920339504878,"min_confidence":0.34032735228538513,"max_confidence":0.4942655563354492},{"index":4,"count":6,"confidence":0.6237975160280863,"empirical_correctness":0.3333333333333333,"signed_gap":0.290464182694753,"absolute_gap":0.290464182694753,"min_confidence":0.5105740427970886,"max_confidence":0.7215518355369568},{"index":5,"count":5,"confidence":0.7748021960258484,"empirical_correctness":0.4,"signed_gap":0.3748021960258484,"absolute_gap":0.3748021960258484,"min_confidence":0.7425537705421448,"max_confidence":0.8034501671791077},{"index":6,"count":6,"confidence":0.8389711578687032,"empirical_correctness":0.6666666666666666,"signed_gap":0.1723044912020366,"absolute_gap":0.1723044912020366,"min_confidence":0.8270289301872253,"max_confidence":0.8544089198112488},{"index":7,"count":5,"confidence":0.8717630982398987,"empirical_correctness":0.8,"signed_gap":0.07176309823989868,"absolute_gap":0.07176309823989868,"min_confidence":0.8586470484733582,"max_confidence":0.8857632875442505},{"index":8,"count":6,"confidence":0.9001896778742472,"empirical_correctness":0.6666666666666666,"signed_gap":0.23352301120758057,"absolute_gap":0.23352301120758057,"min_confidence":0.8865945339202881,"max_confidence":0.9113597273826599},{"index":9,"count":6,"confidence":0.9265138208866119,"empirical_correctness":0.3333333333333333,"signed_gap":0.5931804875532787,"absolute_gap":0.5931804875532787,"min_confidence":0.9221100807189941,"max_confidence":0.9315935969352722}]}},"calibration_singleton_labels":{"equal_width":{"n":56,"bin_count":10,"ece":0.2689618154295853,"signed_mean_gap":0.2689618154295853,"absolute_mean_gap":0.2689618154295853,"bins":[{"index":0,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.0,"upper":0.1,"upper_inclusive":false},{"index":1,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.1,"upper":0.2,"upper_inclusive":false},{"index":2,"count":10,"confidence":0.275234517455101,"empirical_correctness":0.2,"signed_gap":0.07523451745510101,"absolute_gap":0.07523451745510101,"lower":0.2,"upper":0.3,"upper_inclusive":false},{"index":3,"count":9,"confidence":0.33160651061269975,"empirical_correctness":0.0,"signed_gap":0.33160651061269975,"absolute_gap":0.33160651061269975,"lower":0.3,"upper":0.4,"upper_inclusive":false},{"index":4,"count":3,"confidence":0.45179946223894757,"empirical_correctness":0.0,"signed_gap":0.45179946223894757,"absolute_gap":0.45179946223894757,"lower":0.4,"upper":0.5,"upper_inclusive":false},{"index":5,"count":3,"confidence":0.5559185942014059,"empirical_correctness":0.0,"signed_gap":0.5559185942014059,"absolute_gap":0.5559185942014059,"lower":0.5,"upper":0.6,"upper_inclusive":false},{"index":6,"count":2,"confidence":0.6767387390136719,"empirical_correctness":0.5,"signed_gap":0.17673873901367188,"absolute_gap":0.17673873901367188,"lower":0.6,"upper":0.7,"upper_inclusive":false},{"index":7,"count":5,"confidence":0.7584225296974182,"empirical_correctness":0.4,"signed_gap":0.35842252969741817,"absolute_gap":0.35842252969741817,"lower":0.7,"upper":0.8,"upper_inclusive":false},{"index":8,"count":14,"confidence":0.8552950492927006,"empirical_correctness":0.7142857142857143,"signed_gap":0.1410093350069863,"absolute_gap":0.1410093350069863,"lower":0.8,"upper":0.9,"upper_inclusive":false},{"index":9,"count":10,"confidence":0.9182182908058166,"empirical_correctness":0.5,"signed_gap":0.41821829080581663,"absolute_gap":0.41821829080581663,"lower":0.9,"upper":1.0,"upper_inclusive":true}]},"equal_mass":{"n":56,"bin_count":10,"ece":0.2763793920832021,"signed_mean_gap":0.26896181542958536,"absolute_mean_gap":0.26896181542958536,"bins":[{"index":0,"count":5,"confidence":0.2537809908390045,"empirical_correctness":0.0,"signed_gap":0.2537809908390045,"absolute_gap":0.2537809908390045,"min_confidence":0.21950747072696686,"max_confidence":0.29090914130210876},{"index":1,"count":6,"confidence":0.2987179756164551,"empirical_correctness":0.3333333333333333,"signed_gap":-0.03461535771687824,"absolute_gap":0.03461535771687824,"min_confidence":0.29377642273902893,"max_confidence":0.3088676333427429},{"index":2,"count":5,"confidence":0.3258474290370941,"empirical_correctness":0.0,"signed_gap":0.3258474290370941,"absolute_gap":0.3258474290370941,"min_confidence":0.3125802278518677,"max_confidence":0.33409881591796875},{"index":3,"count":6,"confidence":0.4002920339504878,"empirical_correctness":0.0,"signed_gap":0.4002920339504878,"absolute_gap":0.4002920339504878,"min_confidence":0.34032735228538513,"max_confidence":0.4942655563354492},{"index":4,"count":6,"confidence":0.6237975160280863,"empirical_correctness":0.3333333333333333,"signed_gap":0.290464182694753,"absolute_gap":0.290464182694753,"min_confidence":0.5105740427970886,"max_confidence":0.7215518355369568},{"index":5,"count":5,"confidence":0.7748021960258484,"empirical_correctness":0.4,"signed_gap":0.3748021960258484,"absolute_gap":0.3748021960258484,"min_confidence":0.7425537705421448,"max_confidence":0.8034501671791077},{"index":6,"count":6,"confidence":0.8389711578687032,"empirical_correctness":0.6666666666666666,"signed_gap":0.1723044912020366,"absolute_gap":0.1723044912020366,"min_confidence":0.8270289301872253,"max_confidence":0.8544089198112488},{"index":7,"count":5,"confidence":0.8717630982398987,"empirical_correctness":0.8,"signed_gap":0.07176309823989868,"absolute_gap":0.07176309823989868,"min_confidence":0.8586470484733582,"max_confidence":0.8857632875442505},{"index":8,"count":6,"confidence":0.9001896778742472,"empirical_correctness":0.6666666666666666,"signed_gap":0.23352301120758057,"absolute_gap":0.23352301120758057,"min_confidence":0.8865945339202881,"max_confidence":0.9113597273826599},{"index":9,"count":6,"confidence":0.9265138208866119,"empirical_correctness":0.3333333333333333,"signed_gap":0.5931804875532787,"absolute_gap":0.5931804875532787,"min_confidence":0.9221100807189941,"max_confidence":0.9315935969352722}]}},"latency_ms":{"n":56,"median":8.506359503371641,"p95":8.613662503194064},"per_action":{"keep":{"n":8,"raw":{"n":8,"acceptable":0,"unsafe":0,"review":0,"automated_count":8,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.32440756488388023,"confidence_level":0.95}},"effective":{"n":8,"acceptable":0,"unsafe":0,"review":2,"automated_count":6,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":2,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":0.25,"automated_coverage":0.75,"conditional_correctness":0.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.25,"accuracy_wilson_95":{"lower":0.0,"upper":0.32440756488388023,"confidence_level":0.95}}},"drop":{"n":22,"raw":{"n":22,"acceptable":19,"unsafe":2,"review":1,"automated_count":21,"automated_acceptable":19,"automated_unsafe":2,"unnecessary_reviews":1,"accuracy":0.8636363636363636,"unsafe_rate":0.09090909090909091,"review_rate":0.045454545454545456,"automated_coverage":0.9545454545454546,"conditional_correctness":0.9047619047619048,"conditional_unsafe":0.09523809523809523,"unnecessary_review_rate":0.045454545454545456,"accuracy_wilson_95":{"lower":0.6666501283342828,"upper":0.9525099628594474,"confidence_level":0.95}},"effective":{"n":22,"acceptable":17,"unsafe":0,"review":5,"automated_count":17,"automated_acceptable":17,"automated_unsafe":0,"unnecessary_reviews":5,"accuracy":0.7727272727272727,"unsafe_rate":0.0,"review_rate":0.22727272727272727,"automated_coverage":0.7727272727272727,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.22727272727272727,"accuracy_wilson_95":{"lower":0.5656004682477829,"upper":0.8987696001475147,"confidence_level":0.95}}},"mask":{"n":8,"raw":{"n":8,"acceptable":0,"unsafe":5,"review":0,"automated_count":8,"automated_acceptable":0,"automated_unsafe":5,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":0.625,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":0.625,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.32440756488388023,"confidence_level":0.95}},"effective":{"n":8,"acceptable":0,"unsafe":0,"review":5,"automated_count":3,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":5,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":0.625,"automated_coverage":0.375,"conditional_correctness":0.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.625,"accuracy_wilson_95":{"lower":0.0,"upper":0.32440756488388023,"confidence_level":0.95}}},"pseudonymize":{"n":8,"raw":{"n":8,"acceptable":0,"unsafe":1,"review":0,"automated_count":8,"automated_acceptable":0,"automated_unsafe":1,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":0.125,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":0.125,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.32440756488388023,"confidence_level":0.95}},"effective":{"n":8,"acceptable":0,"unsafe":0,"review":5,"automated_count":3,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":5,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":0.625,"automated_coverage":0.375,"conditional_correctness":0.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.625,"accuracy_wilson_95":{"lower":0.0,"upper":0.32440756488388023,"confidence_level":0.95}}},"generalize":{"n":8,"raw":{"n":8,"acceptable":1,"unsafe":1,"review":0,"automated_count":8,"automated_acceptable":1,"automated_unsafe":1,"unnecessary_reviews":0,"accuracy":0.125,"unsafe_rate":0.125,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.125,"conditional_unsafe":0.125,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.02241749145005667,"upper":0.4708881822128535,"confidence_level":0.95}},"effective":{"n":8,"acceptable":1,"unsafe":0,"review":3,"automated_count":5,"automated_acceptable":1,"automated_unsafe":0,"unnecessary_reviews":3,"accuracy":0.125,"unsafe_rate":0.0,"review_rate":0.375,"automated_coverage":0.625,"conditional_correctness":0.2,"conditional_unsafe":0.0,"unnecessary_review_rate":0.375,"accuracy_wilson_95":{"lower":0.02241749145005667,"upper":0.4708881822128535,"confidence_level":0.95}}},"review":{"n":2,"raw":{"n":2,"acceptable":0,"unsafe":0,"review":0,"automated_count":2,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.6576197724933469,"confidence_level":0.95}},"effective":{"n":2,"acceptable":0,"unsafe":0,"review":0,"automated_count":2,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.6576197724933469,"confidence_level":0.95}}}}}},"composite":{"n":198,"scope":"descriptive sum across four correlated panels","raw":{"n":198,"acceptable":82,"unsafe":18,"review":10,"automated_count":188,"automated_acceptable":80,"automated_unsafe":18,"unnecessary_reviews":8,"accuracy":0.41414141414141414,"unsafe_rate":0.09090909090909091,"review_rate":0.050505050505050504,"automated_coverage":0.9494949494949495,"conditional_correctness":0.425531914893617,"conditional_unsafe":0.09574468085106383,"unnecessary_review_rate":0.04040404040404041},"effective":{"n":198,"acceptable":72,"unsafe":0,"review":62,"automated_count":136,"automated_acceptable":68,"automated_unsafe":0,"unnecessary_reviews":58,"accuracy":0.36363636363636365,"unsafe_rate":0.0,"review_rate":0.31313131313131315,"automated_coverage":0.6868686868686869,"conditional_correctness":0.5,"conditional_unsafe":0.0,"unnecessary_review_rate":0.29292929292929293}}},{"id":"Suarify--laya-pdpa-my-v3-fp32","platform":"Local","model_id":"Suarify/laya-pdpa-my-v3","revision":"c88002a9ba10658a7ed70e12f02773f2708d4c86","loading":"float32 / none","scoring_kind":"Native decision readout","settings":{"dtype":"float32","quantization":"none","max_tokens":1024,"head_max_tokens":256,"temperature":5.0,"autocast":false,"truncation":false,"cpu_offload":false,"guard_threshold":0.45,"seed":0,"tf32":false},"scoring_metadata":{"backend":"laya_native_local_builtin","readout":"native_mask_slot_scorer_numpy_float32_softmax","probability_source":"native_choice_unrounded_published_temperature_not_task_calibrated","latency_scope":"synchronized serial complete encoder+head+act forward,input transfer and native readout; tokenization excluded","native_disclosures":["Stock installed torch attention in place of ONNX dynamic equivalent; no custom kernels or repo imports.","FP32/no-autocast explicit precision: convert every saved F16/BF16/F32 tensor toFP32; no quantization.","Unrounded native probability; guard uses winner p, not entropy-concentration confidence.","Act head fully loaded and evaluated; not used to alter six-action answer.","Native choice:6-10 bucket else choice scalar0, T=5.0. No task fitting, priors correction, prompt change or finetuning here."],"equal_mass_convention":"rank bins with floor boundaries at floor(i*n/k), k=min(10,n//2); minimum two rows"},"load_ms":null,"peak_allocated_bytes":1363195392,"peak_reserved_bytes":1396703232,"panels":{"diagnostic43-no_samples":{"n":43,"raw":{"n":43,"acceptable":22,"unsafe":3,"review":0,"automated_count":43,"automated_acceptable":22,"automated_unsafe":3,"unnecessary_reviews":0,"accuracy":0.5116279069767442,"unsafe_rate":0.06976744186046512,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.5116279069767442,"conditional_unsafe":0.06976744186046512,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.3675230846944931,"upper":0.6538255245328142,"confidence_level":0.95}},"effective":{"n":43,"acceptable":19,"unsafe":0,"review":11,"automated_count":32,"automated_acceptable":18,"automated_unsafe":0,"unnecessary_reviews":10,"accuracy":0.4418604651162791,"unsafe_rate":0.0,"review_rate":0.2558139534883721,"automated_coverage":0.7441860465116279,"conditional_correctness":0.5625,"conditional_unsafe":0.0,"unnecessary_review_rate":0.23255813953488372,"accuracy_wilson_95":{"lower":0.30433309839170514,"upper":0.5889238554717591,"confidence_level":0.95}},"confusion":{"raw":{"keep":{"keep":3,"drop":9,"mask":0,"pseudonymize":0,"generalize":0,"review":0},"drop":{"keep":1,"drop":15,"mask":0,"pseudonymize":0,"generalize":0,"review":0},"mask":{"keep":1,"drop":1,"mask":0,"pseudonymize":0,"generalize":0,"review":0},"pseudonymize":{"keep":0,"drop":2,"mask":0,"pseudonymize":0,"generalize":0,"review":0},"generalize":{"keep":0,"drop":6,"mask":0,"pseudonymize":0,"generalize":0,"review":0},"review":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0}},"effective":{"keep":{"keep":2,"drop":7,"mask":0,"pseudonymize":0,"generalize":0,"review":3},"drop":{"keep":0,"drop":12,"mask":0,"pseudonymize":0,"generalize":0,"review":4},"mask":{"keep":0,"drop":1,"mask":0,"pseudonymize":0,"generalize":0,"review":1},"pseudonymize":{"keep":0,"drop":2,"mask":0,"pseudonymize":0,"generalize":0,"review":0},"generalize":{"keep":0,"drop":4,"mask":0,"pseudonymize":0,"generalize":0,"review":2},"review":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0}}},"probability_metrics":{"singleton_n":38,"multiacceptable_n":5,"multiacceptable_coverage":0.11627906976744186,"nll":1.610947284394641,"nll_zero_mass_n":0,"brier":0.7748135029084712,"acceptable_set_n":43,"acceptable_set_nll":1.4579574796917822,"acceptable_set_zero_mass_n":0},"calibration_all_labels":{"equal_width":{"n":43,"bin_count":10,"ece":0.2090680477924125,"signed_mean_gap":0.18382418328939482,"absolute_mean_gap":0.18382418328939482,"bins":[{"index":0,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.0,"upper":0.1,"upper_inclusive":false},{"index":1,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.1,"upper":0.2,"upper_inclusive":false},{"index":2,"count":1,"confidence":0.2353195697069168,"empirical_correctness":0.0,"signed_gap":0.2353195697069168,"absolute_gap":0.2353195697069168,"lower":0.2,"upper":0.3,"upper_inclusive":false},{"index":3,"count":5,"confidence":0.35199740529060364,"empirical_correctness":0.4,"signed_gap":-0.048002594709396385,"absolute_gap":0.048002594709396385,"lower":0.3,"upper":0.4,"upper_inclusive":false},{"index":4,"count":6,"confidence":0.4495449811220169,"empirical_correctness":0.5,"signed_gap":-0.05045501887798309,"absolute_gap":0.05045501887798309,"lower":0.4,"upper":0.5,"upper_inclusive":false},{"index":5,"count":4,"confidence":0.5398375988006592,"empirical_correctness":0.5,"signed_gap":0.03983759880065918,"absolute_gap":0.03983759880065918,"lower":0.5,"upper":0.6,"upper_inclusive":false},{"index":6,"count":2,"confidence":0.6370919346809387,"empirical_correctness":0.5,"signed_gap":0.13709193468093872,"absolute_gap":0.13709193468093872,"lower":0.6,"upper":0.7,"upper_inclusive":false},{"index":7,"count":3,"confidence":0.7560777068138123,"empirical_correctness":0.3333333333333333,"signed_gap":0.42274437348047894,"absolute_gap":0.42274437348047894,"lower":0.7,"upper":0.8,"upper_inclusive":false},{"index":8,"count":14,"confidence":0.8628742226532528,"empirical_correctness":0.5,"signed_gap":0.36287422265325275,"absolute_gap":0.36287422265325275,"lower":0.8,"upper":0.9,"upper_inclusive":false},{"index":9,"count":8,"confidence":0.9287321120500565,"empirical_correctness":0.75,"signed_gap":0.17873211205005646,"absolute_gap":0.17873211205005646,"lower":0.9,"upper":1.0,"upper_inclusive":true}]},"equal_mass":{"n":43,"bin_count":10,"ece":0.2718026627634847,"signed_mean_gap":0.18382418328939482,"absolute_mean_gap":0.18382418328939482,"bins":[{"index":0,"count":4,"confidence":0.3056195266544819,"empirical_correctness":0.0,"signed_gap":0.3056195266544819,"absolute_gap":0.3056195266544819,"min_confidence":0.2353195697069168,"max_confidence":0.3475295305252075},{"index":1,"count":4,"confidence":0.398743212223053,"empirical_correctness":0.75,"signed_gap":-0.351256787776947,"absolute_gap":0.351256787776947,"min_confidence":0.3821585476398468,"max_confidence":0.41378283500671387},{"index":2,"count":4,"confidence":0.46878138184547424,"empirical_correctness":0.5,"signed_gap":-0.031218618154525757,"absolute_gap":0.031218618154525757,"min_confidence":0.4470135569572449,"max_confidence":0.4784477949142456},{"index":3,"count":5,"confidence":0.5551054835319519,"empirical_correctness":0.4,"signed_gap":0.15510548353195186,"absolute_gap":0.15510548353195186,"min_confidence":0.5002322196960449,"max_confidence":0.6161770224571228},{"index":4,"count":4,"confidence":0.7315599918365479,"empirical_correctness":0.5,"signed_gap":0.23155999183654785,"absolute_gap":0.23155999183654785,"min_confidence":0.6580068469047546,"max_confidence":0.7751131653785706},{"index":5,"count":4,"confidence":0.8241623342037201,"empirical_correctness":0.25,"signed_gap":0.5741623342037201,"absolute_gap":0.5741623342037201,"min_confidence":0.8181347250938416,"max_confidence":0.8370962738990784},{"index":6,"count":5,"confidence":0.8662449359893799,"empirical_correctness":0.4,"signed_gap":0.4662449359893799,"absolute_gap":0.4662449359893799,"min_confidence":0.8416163921356201,"max_confidence":0.8803648948669434},{"index":7,"count":4,"confidence":0.8887848556041718,"empirical_correctness":0.75,"signed_gap":0.13878485560417175,"absolute_gap":0.13878485560417175,"min_confidence":0.8833198547363281,"max_confidence":0.8941332697868347},{"index":8,"count":4,"confidence":0.9095910787582397,"empirical_correctness":1.0,"signed_gap":-0.09040892124176025,"absolute_gap":0.09040892124176025,"min_confidence":0.8972256779670715,"max_confidence":0.9148873090744019},{"index":9,"count":5,"confidence":0.9377436518669129,"empirical_correctness":0.6,"signed_gap":0.3377436518669129,"absolute_gap":0.3377436518669129,"min_confidence":0.9270004630088806,"max_confidence":0.9496970176696777}]}},"calibration_singleton_labels":{"equal_width":{"n":38,"bin_count":10,"ece":0.26439177950746134,"signed_mean_gap":0.2143336420780734,"absolute_mean_gap":0.2143336420780734,"bins":[{"index":0,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.0,"upper":0.1,"upper_inclusive":false},{"index":1,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.1,"upper":0.2,"upper_inclusive":false},{"index":2,"count":1,"confidence":0.2353195697069168,"empirical_correctness":0.0,"signed_gap":0.2353195697069168,"absolute_gap":0.2353195697069168,"lower":0.2,"upper":0.3,"upper_inclusive":false},{"index":3,"count":5,"confidence":0.35199740529060364,"empirical_correctness":0.4,"signed_gap":-0.048002594709396385,"absolute_gap":0.048002594709396385,"lower":0.3,"upper":0.4,"upper_inclusive":false},{"index":4,"count":5,"confidence":0.45778167247772217,"empirical_correctness":0.6,"signed_gap":-0.1422183275222778,"absolute_gap":0.1422183275222778,"lower":0.4,"upper":0.5,"upper_inclusive":false},{"index":5,"count":4,"confidence":0.5398375988006592,"empirical_correctness":0.5,"signed_gap":0.03983759880065918,"absolute_gap":0.03983759880065918,"lower":0.5,"upper":0.6,"upper_inclusive":false},{"index":6,"count":1,"confidence":0.6161770224571228,"empirical_correctness":0.0,"signed_gap":0.6161770224571228,"absolute_gap":0.6161770224571228,"lower":0.6,"upper":0.7,"upper_inclusive":false},{"index":7,"count":3,"confidence":0.7560777068138123,"empirical_correctness":0.3333333333333333,"signed_gap":0.42274437348047894,"absolute_gap":0.42274437348047894,"lower":0.7,"upper":0.8,"upper_inclusive":false},{"index":8,"count":12,"confidence":0.8599633028109869,"empirical_correctness":0.4166666666666667,"signed_gap":0.4432966361443202,"absolute_gap":0.4432966361443202,"lower":0.8,"upper":0.9,"upper_inclusive":false},{"index":9,"count":7,"confidence":0.9281633240836007,"empirical_correctness":0.7142857142857143,"signed_gap":0.2138776097978864,"absolute_gap":0.2138776097978864,"lower":0.9,"upper":1.0,"upper_inclusive":true}]},"equal_mass":{"n":38,"bin_count":10,"ece":0.31708992115761103,"signed_mean_gap":0.2143336420780734,"absolute_mean_gap":0.2143336420780734,"bins":[{"index":0,"count":3,"confidence":0.29164952536424,"empirical_correctness":0.0,"signed_gap":0.29164952536424,"absolute_gap":0.29164952536424,"min_confidence":0.2353195697069168,"max_confidence":0.32357141375541687},{"index":1,"count":4,"confidence":0.3835352137684822,"empirical_correctness":0.75,"signed_gap":-0.3664647862315178,"absolute_gap":0.3664647862315178,"min_confidence":0.3475295305252075,"max_confidence":0.41378283500671387},{"index":2,"count":4,"confidence":0.46878138184547424,"empirical_correctness":0.5,"signed_gap":-0.031218618154525757,"absolute_gap":0.031218618154525757,"min_confidence":0.4470135569572449,"max_confidence":0.4784477949142456},{"index":3,"count":4,"confidence":0.5398375988006592,"empirical_correctness":0.5,"signed_gap":0.03983759880065918,"absolute_gap":0.03983759880065918,"min_confidence":0.5002322196960449,"max_confidence":0.5703932642936707},{"index":4,"count":4,"confidence":0.7211025357246399,"empirical_correctness":0.25,"signed_gap":0.4711025357246399,"absolute_gap":0.4711025357246399,"min_confidence":0.6161770224571228,"max_confidence":0.7751131653785706},{"index":5,"count":3,"confidence":0.819851020971934,"empirical_correctness":0.3333333333333333,"signed_gap":0.48651768763860065,"absolute_gap":0.48651768763860065,"min_confidence":0.8181347250938416,"max_confidence":0.8220638036727905},{"index":6,"count":4,"confidence":0.8550855666399002,"empirical_correctness":0.25,"signed_gap":0.6050855666399002,"absolute_gap":0.6050855666399002,"min_confidence":0.8370962738990784,"max_confidence":0.8786287903785706},{"index":7,"count":4,"confidence":0.8856096565723419,"empirical_correctness":0.5,"signed_gap":0.3856096565723419,"absolute_gap":0.3856096565723419,"min_confidence":0.8803648948669434,"max_confidence":0.8941332697868347},{"index":8,"count":4,"confidence":0.9095910787582397,"empirical_correctness":1.0,"signed_gap":-0.09040892124176025,"absolute_gap":0.09040892124176025,"min_confidence":0.8972256779670715,"max_confidence":0.9148873090744019},{"index":9,"count":4,"confidence":0.9390011578798294,"empirical_correctness":0.5,"signed_gap":0.4390011578798294,"absolute_gap":0.4390011578798294,"min_confidence":0.9270004630088806,"max_confidence":0.9496970176696777}]}},"latency_ms":{"n":43,"median":30.177900101989508,"p95":31.609119893983006},"per_action":{"keep":{"n":12,"raw":{"n":12,"acceptable":3,"unsafe":0,"review":0,"automated_count":12,"automated_acceptable":3,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":0.25,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.25,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.08894166839405471,"upper":0.5323053349335657,"confidence_level":0.95}},"effective":{"n":12,"acceptable":2,"unsafe":0,"review":3,"automated_count":9,"automated_acceptable":2,"automated_unsafe":0,"unnecessary_reviews":3,"accuracy":0.16666666666666666,"unsafe_rate":0.0,"review_rate":0.25,"automated_coverage":0.75,"conditional_correctness":0.2222222222222222,"conditional_unsafe":0.0,"unnecessary_review_rate":0.25,"accuracy_wilson_95":{"lower":0.04696514218385378,"upper":0.4480308622529734,"confidence_level":0.95}}},"drop":{"n":16,"raw":{"n":16,"acceptable":15,"unsafe":1,"review":0,"automated_count":16,"automated_acceptable":15,"automated_unsafe":1,"unnecessary_reviews":0,"accuracy":0.9375,"unsafe_rate":0.0625,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.9375,"conditional_unsafe":0.0625,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.7167126242970107,"upper":0.9888806552353576,"confidence_level":0.95}},"effective":{"n":16,"acceptable":12,"unsafe":0,"review":4,"automated_count":12,"automated_acceptable":12,"automated_unsafe":0,"unnecessary_reviews":4,"accuracy":0.75,"unsafe_rate":0.0,"review_rate":0.25,"automated_coverage":0.75,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.25,"accuracy_wilson_95":{"lower":0.5050168346449122,"upper":0.8981793250878696,"confidence_level":0.95}}},"mask":{"n":2,"raw":{"n":2,"acceptable":0,"unsafe":1,"review":0,"automated_count":2,"automated_acceptable":0,"automated_unsafe":1,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":0.5,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":0.5,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.6576197724933469,"confidence_level":0.95}},"effective":{"n":2,"acceptable":0,"unsafe":0,"review":1,"automated_count":1,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":1,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":0.5,"automated_coverage":0.5,"conditional_correctness":0.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.5,"accuracy_wilson_95":{"lower":0.0,"upper":0.6576197724933469,"confidence_level":0.95}}},"pseudonymize":{"n":2,"raw":{"n":2,"acceptable":0,"unsafe":0,"review":0,"automated_count":2,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.6576197724933469,"confidence_level":0.95}},"effective":{"n":2,"acceptable":0,"unsafe":0,"review":0,"automated_count":2,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.6576197724933469,"confidence_level":0.95}}},"generalize":{"n":6,"raw":{"n":6,"acceptable":0,"unsafe":0,"review":0,"automated_count":6,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":2.7755575615628914e-17,"upper":0.3903342879021653,"confidence_level":0.95}},"effective":{"n":6,"acceptable":0,"unsafe":0,"review":2,"automated_count":4,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":2,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":0.3333333333333333,"automated_coverage":0.6666666666666666,"conditional_correctness":0.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.3333333333333333,"accuracy_wilson_95":{"lower":2.7755575615628914e-17,"upper":0.3903342879021653,"confidence_level":0.95}}},"review":{"n":0,"raw":{"n":0,"acceptable":0,"unsafe":0,"review":0,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":null,"unsafe_rate":null,"review_rate":null,"automated_coverage":null,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":null},"effective":{"n":0,"acceptable":0,"unsafe":0,"review":0,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":null,"unsafe_rate":null,"review_rate":null,"automated_coverage":null,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":null}}}},"diagnostic43-compact_no_samples":{"n":43,"raw":{"n":43,"acceptable":21,"unsafe":0,"review":0,"automated_count":43,"automated_acceptable":21,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":0.4883720930232558,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.4883720930232558,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.3461744754671859,"upper":0.632476915305507,"confidence_level":0.95}},"effective":{"n":43,"acceptable":21,"unsafe":0,"review":0,"automated_count":43,"automated_acceptable":21,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":0.4883720930232558,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.4883720930232558,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.3461744754671859,"upper":0.632476915305507,"confidence_level":0.95}},"confusion":{"raw":{"keep":{"keep":0,"drop":12,"mask":0,"pseudonymize":0,"generalize":0,"review":0},"drop":{"keep":0,"drop":16,"mask":0,"pseudonymize":0,"generalize":0,"review":0},"mask":{"keep":0,"drop":2,"mask":0,"pseudonymize":0,"generalize":0,"review":0},"pseudonymize":{"keep":0,"drop":2,"mask":0,"pseudonymize":0,"generalize":0,"review":0},"generalize":{"keep":0,"drop":6,"mask":0,"pseudonymize":0,"generalize":0,"review":0},"review":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0}},"effective":{"keep":{"keep":0,"drop":12,"mask":0,"pseudonymize":0,"generalize":0,"review":0},"drop":{"keep":0,"drop":16,"mask":0,"pseudonymize":0,"generalize":0,"review":0},"mask":{"keep":0,"drop":2,"mask":0,"pseudonymize":0,"generalize":0,"review":0},"pseudonymize":{"keep":0,"drop":2,"mask":0,"pseudonymize":0,"generalize":0,"review":0},"generalize":{"keep":0,"drop":6,"mask":0,"pseudonymize":0,"generalize":0,"review":0},"review":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0}}},"probability_metrics":{"singleton_n":38,"multiacceptable_n":5,"multiacceptable_coverage":0.11627906976744186,"nll":3.3266288468427616,"nll_zero_mass_n":0,"brier":1.099127066387453,"acceptable_set_n":43,"acceptable_set_nll":2.940803081065035,"acceptable_set_zero_mass_n":0},"calibration_all_labels":{"equal_width":{"n":43,"bin_count":10,"ece":0.47334459216095687,"signed_mean_gap":0.47334459216095687,"absolute_mean_gap":0.47334459216095687,"bins":[{"index":0,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.0,"upper":0.1,"upper_inclusive":false},{"index":1,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.1,"upper":0.2,"upper_inclusive":false},{"index":2,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.2,"upper":0.3,"upper_inclusive":false},{"index":3,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.3,"upper":0.4,"upper_inclusive":false},{"index":4,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.4,"upper":0.5,"upper_inclusive":false},{"index":5,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.5,"upper":0.6,"upper_inclusive":false},{"index":6,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.6,"upper":0.7,"upper_inclusive":false},{"index":7,"count":1,"confidence":0.7857770323753357,"empirical_correctness":0.0,"signed_gap":0.7857770323753357,"absolute_gap":0.7857770323753357,"lower":0.7,"upper":0.8,"upper_inclusive":false},{"index":8,"count":5,"confidence":0.851430070400238,"empirical_correctness":0.4,"signed_gap":0.45143007040023797,"absolute_gap":0.45143007040023797,"lower":0.8,"upper":0.9,"upper_inclusive":false},{"index":9,"count":37,"confidence":0.9813754075282329,"empirical_correctness":0.5135135135135135,"signed_gap":0.46786189401471945,"absolute_gap":0.46786189401471945,"lower":0.9,"upper":1.0,"upper_inclusive":true}]},"equal_mass":{"n":43,"bin_count":10,"ece":0.4733445921609568,"signed_mean_gap":0.4733445921609568,"absolute_mean_gap":0.4733445921609568,"bins":[{"index":0,"count":4,"confidence":0.8252502679824829,"empirical_correctness":0.25,"signed_gap":0.5752502679824829,"absolute_gap":0.5752502679824829,"min_confidence":0.7857770323753357,"max_confidence":0.8606126308441162},{"index":1,"count":4,"confidence":0.891208291053772,"empirical_correctness":0.5,"signed_gap":0.391208291053772,"absolute_gap":0.391208291053772,"min_confidence":0.862457811832428,"max_confidence":0.9197311997413635},{"index":2,"count":4,"confidence":0.9329236894845963,"empirical_correctness":0.5,"signed_gap":0.43292368948459625,"absolute_gap":0.43292368948459625,"min_confidence":0.9259294867515564,"max_confidence":0.9447764754295349},{"index":3,"count":5,"confidence":0.9732324719429016,"empirical_correctness":0.6,"signed_gap":0.37323247194290166,"absolute_gap":0.37323247194290166,"min_confidence":0.9627522826194763,"max_confidence":0.9846195578575134},{"index":4,"count":4,"confidence":0.9919432550668716,"empirical_correctness":0.25,"signed_gap":0.7419432550668716,"absolute_gap":0.7419432550668716,"min_confidence":0.990547239780426,"max_confidence":0.9939524531364441},{"index":5,"count":4,"confidence":0.9949216693639755,"empirical_correctness":0.5,"signed_gap":0.4949216693639755,"absolute_gap":0.4949216693639755,"min_confidence":0.9943832159042358,"max_confidence":0.9953900575637817},{"index":6,"count":5,"confidence":0.9961891412734986,"empirical_correctness":0.6,"signed_gap":0.3961891412734986,"absolute_gap":0.3961891412734986,"min_confidence":0.9958574175834656,"max_confidence":0.9964599013328552},{"index":7,"count":4,"confidence":0.9967128783464432,"empirical_correctness":0.25,"signed_gap":0.7467128783464432,"absolute_gap":0.7467128783464432,"min_confidence":0.9965862035751343,"max_confidence":0.9968283772468567},{"index":8,"count":4,"confidence":0.9969872087240219,"empirical_correctness":0.5,"signed_gap":0.4969872087240219,"absolute_gap":0.4969872087240219,"min_confidence":0.9968327879905701,"max_confidence":0.9971132278442383},{"index":9,"count":5,"confidence":0.9973840713500977,"empirical_correctness":0.8,"signed_gap":0.1973840713500976,"absolute_gap":0.1973840713500976,"min_confidence":0.9971750974655151,"max_confidence":0.9974716901779175}]}},"calibration_singleton_labels":{"equal_width":{"n":38,"bin_count":10,"ece":0.5373305678367615,"signed_mean_gap":0.5373305678367615,"absolute_mean_gap":0.5373305678367615,"bins":[{"index":0,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.0,"upper":0.1,"upper_inclusive":false},{"index":1,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.1,"upper":0.2,"upper_inclusive":false},{"index":2,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.2,"upper":0.3,"upper_inclusive":false},{"index":3,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.3,"upper":0.4,"upper_inclusive":false},{"index":4,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.4,"upper":0.5,"upper_inclusive":false},{"index":5,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.5,"upper":0.6,"upper_inclusive":false},{"index":6,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.6,"upper":0.7,"upper_inclusive":false},{"index":7,"count":1,"confidence":0.7857770323753357,"empirical_correctness":0.0,"signed_gap":0.7857770323753357,"absolute_gap":0.7857770323753357,"lower":0.7,"upper":0.8,"upper_inclusive":false},{"index":8,"count":5,"confidence":0.851430070400238,"empirical_correctness":0.4,"signed_gap":0.45143007040023797,"absolute_gap":0.45143007040023797,"lower":0.8,"upper":0.9,"upper_inclusive":false},{"index":9,"count":32,"confidence":0.9804885685443878,"empirical_correctness":0.4375,"signed_gap":0.5429885685443878,"absolute_gap":0.5429885685443878,"lower":0.9,"upper":1.0,"upper_inclusive":true}]},"equal_mass":{"n":38,"bin_count":10,"ece":0.5373305678367615,"signed_mean_gap":0.5373305678367615,"absolute_mean_gap":0.5373305678367615,"bins":[{"index":0,"count":3,"confidence":0.8134628136952718,"empirical_correctness":0.3333333333333333,"signed_gap":0.48012948036193853,"absolute_gap":0.48012948036193853,"min_confidence":0.7857770323753357,"max_confidence":0.8438723087310791},{"index":1,"count":4,"confidence":0.8764286488294601,"empirical_correctness":0.25,"signed_gap":0.6264286488294601,"absolute_gap":0.6264286488294601,"min_confidence":0.8606126308441162,"max_confidence":0.9031756520271301},{"index":2,"count":4,"confidence":0.9266623705625534,"empirical_correctness":0.75,"signed_gap":0.1766623705625534,"absolute_gap":0.1766623705625534,"min_confidence":0.9197311997413635,"max_confidence":0.9345871210098267},{"index":3,"count":4,"confidence":0.9658917486667633,"empirical_correctness":0.25,"signed_gap":0.7158917486667633,"absolute_gap":0.7158917486667633,"min_confidence":0.9447764754295349,"max_confidence":0.9816458225250244},{"index":4,"count":4,"confidence":0.9920509457588196,"empirical_correctness":0.0,"signed_gap":0.9920509457588196,"absolute_gap":0.9920509457588196,"min_confidence":0.990547239780426,"max_confidence":0.9943832159042358},{"index":5,"count":3,"confidence":0.9951011538505554,"empirical_correctness":0.6666666666666666,"signed_gap":0.3284344871838888,"absolute_gap":0.3284344871838888,"min_confidence":0.9945687055587769,"max_confidence":0.9953900575637817},{"index":6,"count":4,"confidence":0.9961214512586594,"empirical_correctness":0.5,"signed_gap":0.49612145125865936,"absolute_gap":0.49612145125865936,"min_confidence":0.9958574175834656,"max_confidence":0.9963621497154236},{"index":7,"count":4,"confidence":0.9967128783464432,"empirical_correctness":0.25,"signed_gap":0.7467128783464432,"absolute_gap":0.7467128783464432,"min_confidence":0.9965862035751343,"max_confidence":0.9968283772468567},{"index":8,"count":4,"confidence":0.9969872087240219,"empirical_correctness":0.5,"signed_gap":0.4969872087240219,"absolute_gap":0.4969872087240219,"min_confidence":0.9968327879905701,"max_confidence":0.9971132278442383},{"index":9,"count":4,"confidence":0.9973621666431427,"empirical_correctness":0.75,"signed_gap":0.2473621666431427,"absolute_gap":0.2473621666431427,"min_confidence":0.9971750974655151,"max_confidence":0.9974662065505981}]}},"latency_ms":{"n":43,"median":31.485799932852387,"p95":32.40411998704076},"per_action":{"keep":{"n":12,"raw":{"n":12,"acceptable":0,"unsafe":0,"review":0,"automated_count":12,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":2.7755575615628914e-17,"upper":0.24249400665524085,"confidence_level":0.95}},"effective":{"n":12,"acceptable":0,"unsafe":0,"review":0,"automated_count":12,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":2.7755575615628914e-17,"upper":0.24249400665524085,"confidence_level":0.95}}},"drop":{"n":16,"raw":{"n":16,"acceptable":16,"unsafe":0,"review":0,"automated_count":16,"automated_acceptable":16,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":1.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.8063923194655637,"upper":1.0,"confidence_level":0.95}},"effective":{"n":16,"acceptable":16,"unsafe":0,"review":0,"automated_count":16,"automated_acceptable":16,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":1.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.8063923194655637,"upper":1.0,"confidence_level":0.95}}},"mask":{"n":2,"raw":{"n":2,"acceptable":0,"unsafe":0,"review":0,"automated_count":2,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.6576197724933469,"confidence_level":0.95}},"effective":{"n":2,"acceptable":0,"unsafe":0,"review":0,"automated_count":2,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.6576197724933469,"confidence_level":0.95}}},"pseudonymize":{"n":2,"raw":{"n":2,"acceptable":0,"unsafe":0,"review":0,"automated_count":2,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.6576197724933469,"confidence_level":0.95}},"effective":{"n":2,"acceptable":0,"unsafe":0,"review":0,"automated_count":2,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.6576197724933469,"confidence_level":0.95}}},"generalize":{"n":6,"raw":{"n":6,"acceptable":0,"unsafe":0,"review":0,"automated_count":6,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":2.7755575615628914e-17,"upper":0.3903342879021653,"confidence_level":0.95}},"effective":{"n":6,"acceptable":0,"unsafe":0,"review":0,"automated_count":6,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":2.7755575615628914e-17,"upper":0.3903342879021653,"confidence_level":0.95}}},"review":{"n":0,"raw":{"n":0,"acceptable":0,"unsafe":0,"review":0,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":null,"unsafe_rate":null,"review_rate":null,"automated_coverage":null,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":null},"effective":{"n":0,"acceptable":0,"unsafe":0,"review":0,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":null,"unsafe_rate":null,"review_rate":null,"automated_coverage":null,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":null}}}},"challenge56-no_samples":{"n":56,"raw":{"n":56,"acceptable":24,"unsafe":10,"review":0,"automated_count":56,"automated_acceptable":24,"automated_unsafe":10,"unnecessary_reviews":0,"accuracy":0.42857142857142855,"unsafe_rate":0.17857142857142858,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.42857142857142855,"conditional_unsafe":0.17857142857142858,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.3076896268393824,"upper":0.5586237926918696,"confidence_level":0.95}},"effective":{"n":56,"acceptable":21,"unsafe":0,"review":28,"automated_count":28,"automated_acceptable":20,"automated_unsafe":0,"unnecessary_reviews":27,"accuracy":0.375,"unsafe_rate":0.0,"review_rate":0.5,"automated_coverage":0.5,"conditional_correctness":0.7142857142857143,"conditional_unsafe":0.0,"unnecessary_review_rate":0.48214285714285715,"accuracy_wilson_95":{"lower":0.260102143264709,"upper":0.5059463409149819,"confidence_level":0.95}},"confusion":{"raw":{"keep":{"keep":0,"drop":6,"mask":1,"pseudonymize":1,"generalize":0,"review":0},"drop":{"keep":2,"drop":20,"mask":0,"pseudonymize":0,"generalize":0,"review":0},"mask":{"keep":0,"drop":8,"mask":0,"pseudonymize":0,"generalize":0,"review":0},"pseudonymize":{"keep":4,"drop":2,"mask":0,"pseudonymize":0,"generalize":2,"review":0},"generalize":{"keep":4,"drop":0,"mask":0,"pseudonymize":0,"generalize":4,"review":0},"review":{"keep":0,"drop":2,"mask":0,"pseudonymize":0,"generalize":0,"review":0}},"effective":{"keep":{"keep":0,"drop":5,"mask":0,"pseudonymize":0,"generalize":0,"review":3},"drop":{"keep":0,"drop":17,"mask":0,"pseudonymize":0,"generalize":0,"review":5},"mask":{"keep":0,"drop":2,"mask":0,"pseudonymize":0,"generalize":0,"review":6},"pseudonymize":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":8},"generalize":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":3,"review":5},"review":{"keep":0,"drop":1,"mask":0,"pseudonymize":0,"generalize":0,"review":1}}},"probability_metrics":{"singleton_n":56,"multiacceptable_n":0,"multiacceptable_coverage":0.0,"nll":1.4831692627580555,"nll_zero_mass_n":0,"brier":0.6673784265296867,"acceptable_set_n":56,"acceptable_set_nll":1.4831692627580555,"acceptable_set_zero_mass_n":0},"calibration_all_labels":{"equal_width":{"n":56,"bin_count":10,"ece":0.17618930605905397,"signed_mean_gap":0.13194496557116506,"absolute_mean_gap":0.13194496557116506,"bins":[{"index":0,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.0,"upper":0.1,"upper_inclusive":false},{"index":1,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.1,"upper":0.2,"upper_inclusive":false},{"index":2,"count":5,"confidence":0.27929396033287046,"empirical_correctness":0.2,"signed_gap":0.07929396033287045,"absolute_gap":0.07929396033287045,"lower":0.2,"upper":0.3,"upper_inclusive":false},{"index":3,"count":11,"confidence":0.34147156368602405,"empirical_correctness":0.18181818181818182,"signed_gap":0.15965338186784223,"absolute_gap":0.15965338186784223,"lower":0.3,"upper":0.4,"upper_inclusive":false},{"index":4,"count":12,"confidence":0.4454132119814555,"empirical_correctness":0.08333333333333333,"signed_gap":0.3620798786481222,"absolute_gap":0.3620798786481222,"lower":0.4,"upper":0.5,"upper_inclusive":false},{"index":5,"count":5,"confidence":0.568546450138092,"empirical_correctness":0.2,"signed_gap":0.368546450138092,"absolute_gap":0.368546450138092,"lower":0.5,"upper":0.6,"upper_inclusive":false},{"index":6,"count":5,"confidence":0.657482361793518,"empirical_correctness":0.6,"signed_gap":0.057482361793518066,"absolute_gap":0.057482361793518066,"lower":0.6,"upper":0.7,"upper_inclusive":false},{"index":7,"count":6,"confidence":0.7482509911060333,"empirical_correctness":0.8333333333333334,"signed_gap":-0.08508234222730005,"absolute_gap":0.08508234222730005,"lower":0.7,"upper":0.8,"upper_inclusive":false},{"index":8,"count":10,"confidence":0.8446754932403564,"empirical_correctness":0.9,"signed_gap":-0.05532450675964362,"absolute_gap":0.05532450675964362,"lower":0.8,"upper":0.9,"upper_inclusive":false},{"index":9,"count":2,"confidence":0.9124487936496735,"empirical_correctness":1.0,"signed_gap":-0.08755120635032654,"absolute_gap":0.08755120635032654,"lower":0.9,"upper":1.0,"upper_inclusive":true}]},"equal_mass":{"n":56,"bin_count":10,"ece":0.17744506948760577,"signed_mean_gap":0.13194496557116506,"absolute_mean_gap":0.13194496557116506,"bins":[{"index":0,"count":5,"confidence":0.27929396033287046,"empirical_correctness":0.2,"signed_gap":0.07929396033287045,"absolute_gap":0.07929396033287045,"min_confidence":0.2524385452270508,"max_confidence":0.2993970811367035},{"index":1,"count":6,"confidence":0.32322104771931964,"empirical_correctness":0.16666666666666666,"signed_gap":0.15655438105265299,"absolute_gap":0.15655438105265299,"min_confidence":0.3043217957019806,"max_confidence":0.3387906849384308},{"index":2,"count":5,"confidence":0.3633721828460693,"empirical_correctness":0.2,"signed_gap":0.1633721828460693,"absolute_gap":0.1633721828460693,"min_confidence":0.3421577513217926,"max_confidence":0.39567074179649353},{"index":3,"count":6,"confidence":0.4350762218236923,"empirical_correctness":0.16666666666666666,"signed_gap":0.2684095551570257,"absolute_gap":0.2684095551570257,"min_confidence":0.4250757396221161,"max_confidence":0.4403200149536133},{"index":4,"count":6,"confidence":0.45575020213921863,"empirical_correctness":0.0,"signed_gap":0.45575020213921863,"absolute_gap":0.45575020213921863,"min_confidence":0.441942423582077,"max_confidence":0.4839661717414856},{"index":5,"count":5,"confidence":0.568546450138092,"empirical_correctness":0.2,"signed_gap":0.368546450138092,"absolute_gap":0.368546450138092,"min_confidence":0.5045910477638245,"max_confidence":0.5934259295463562},{"index":6,"count":6,"confidence":0.6679895520210266,"empirical_correctness":0.6666666666666666,"signed_gap":0.0013228853543599817,"absolute_gap":0.0013228853543599817,"min_confidence":0.6035141944885254,"max_confidence":0.7205255031585693},{"index":7,"count":5,"confidence":0.7537960886955262,"empirical_correctness":0.8,"signed_gap":-0.04620391130447388,"absolute_gap":0.04620391130447388,"min_confidence":0.7267486453056335,"max_confidence":0.7984062433242798},{"index":8,"count":6,"confidence":0.8261694411436716,"empirical_correctness":1.0,"signed_gap":-0.17383055885632837,"absolute_gap":0.17383055885632837,"min_confidence":0.8135843873023987,"max_confidence":0.8381526470184326},{"index":9,"count":6,"confidence":0.8857726454734802,"empirical_correctness":0.8333333333333334,"signed_gap":0.052439312140146854,"absolute_gap":0.052439312140146854,"min_confidence":0.8547865152359009,"max_confidence":0.9212165474891663}]}},"calibration_singleton_labels":{"equal_width":{"n":56,"bin_count":10,"ece":0.17618930605905397,"signed_mean_gap":0.13194496557116506,"absolute_mean_gap":0.13194496557116506,"bins":[{"index":0,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.0,"upper":0.1,"upper_inclusive":false},{"index":1,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.1,"upper":0.2,"upper_inclusive":false},{"index":2,"count":5,"confidence":0.27929396033287046,"empirical_correctness":0.2,"signed_gap":0.07929396033287045,"absolute_gap":0.07929396033287045,"lower":0.2,"upper":0.3,"upper_inclusive":false},{"index":3,"count":11,"confidence":0.34147156368602405,"empirical_correctness":0.18181818181818182,"signed_gap":0.15965338186784223,"absolute_gap":0.15965338186784223,"lower":0.3,"upper":0.4,"upper_inclusive":false},{"index":4,"count":12,"confidence":0.4454132119814555,"empirical_correctness":0.08333333333333333,"signed_gap":0.3620798786481222,"absolute_gap":0.3620798786481222,"lower":0.4,"upper":0.5,"upper_inclusive":false},{"index":5,"count":5,"confidence":0.568546450138092,"empirical_correctness":0.2,"signed_gap":0.368546450138092,"absolute_gap":0.368546450138092,"lower":0.5,"upper":0.6,"upper_inclusive":false},{"index":6,"count":5,"confidence":0.657482361793518,"empirical_correctness":0.6,"signed_gap":0.057482361793518066,"absolute_gap":0.057482361793518066,"lower":0.6,"upper":0.7,"upper_inclusive":false},{"index":7,"count":6,"confidence":0.7482509911060333,"empirical_correctness":0.8333333333333334,"signed_gap":-0.08508234222730005,"absolute_gap":0.08508234222730005,"lower":0.7,"upper":0.8,"upper_inclusive":false},{"index":8,"count":10,"confidence":0.8446754932403564,"empirical_correctness":0.9,"signed_gap":-0.05532450675964362,"absolute_gap":0.05532450675964362,"lower":0.8,"upper":0.9,"upper_inclusive":false},{"index":9,"count":2,"confidence":0.9124487936496735,"empirical_correctness":1.0,"signed_gap":-0.08755120635032654,"absolute_gap":0.08755120635032654,"lower":0.9,"upper":1.0,"upper_inclusive":true}]},"equal_mass":{"n":56,"bin_count":10,"ece":0.17744506948760577,"signed_mean_gap":0.13194496557116506,"absolute_mean_gap":0.13194496557116506,"bins":[{"index":0,"count":5,"confidence":0.27929396033287046,"empirical_correctness":0.2,"signed_gap":0.07929396033287045,"absolute_gap":0.07929396033287045,"min_confidence":0.2524385452270508,"max_confidence":0.2993970811367035},{"index":1,"count":6,"confidence":0.32322104771931964,"empirical_correctness":0.16666666666666666,"signed_gap":0.15655438105265299,"absolute_gap":0.15655438105265299,"min_confidence":0.3043217957019806,"max_confidence":0.3387906849384308},{"index":2,"count":5,"confidence":0.3633721828460693,"empirical_correctness":0.2,"signed_gap":0.1633721828460693,"absolute_gap":0.1633721828460693,"min_confidence":0.3421577513217926,"max_confidence":0.39567074179649353},{"index":3,"count":6,"confidence":0.4350762218236923,"empirical_correctness":0.16666666666666666,"signed_gap":0.2684095551570257,"absolute_gap":0.2684095551570257,"min_confidence":0.4250757396221161,"max_confidence":0.4403200149536133},{"index":4,"count":6,"confidence":0.45575020213921863,"empirical_correctness":0.0,"signed_gap":0.45575020213921863,"absolute_gap":0.45575020213921863,"min_confidence":0.441942423582077,"max_confidence":0.4839661717414856},{"index":5,"count":5,"confidence":0.568546450138092,"empirical_correctness":0.2,"signed_gap":0.368546450138092,"absolute_gap":0.368546450138092,"min_confidence":0.5045910477638245,"max_confidence":0.5934259295463562},{"index":6,"count":6,"confidence":0.6679895520210266,"empirical_correctness":0.6666666666666666,"signed_gap":0.0013228853543599817,"absolute_gap":0.0013228853543599817,"min_confidence":0.6035141944885254,"max_confidence":0.7205255031585693},{"index":7,"count":5,"confidence":0.7537960886955262,"empirical_correctness":0.8,"signed_gap":-0.04620391130447388,"absolute_gap":0.04620391130447388,"min_confidence":0.7267486453056335,"max_confidence":0.7984062433242798},{"index":8,"count":6,"confidence":0.8261694411436716,"empirical_correctness":1.0,"signed_gap":-0.17383055885632837,"absolute_gap":0.17383055885632837,"min_confidence":0.8135843873023987,"max_confidence":0.8381526470184326},{"index":9,"count":6,"confidence":0.8857726454734802,"empirical_correctness":0.8333333333333334,"signed_gap":0.052439312140146854,"absolute_gap":0.052439312140146854,"min_confidence":0.8547865152359009,"max_confidence":0.9212165474891663}]}},"latency_ms":{"n":56,"median":26.009899913333356,"p95":26.5557998791337},"per_action":{"keep":{"n":8,"raw":{"n":8,"acceptable":0,"unsafe":0,"review":0,"automated_count":8,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.32440756488388023,"confidence_level":0.95}},"effective":{"n":8,"acceptable":0,"unsafe":0,"review":3,"automated_count":5,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":3,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":0.375,"automated_coverage":0.625,"conditional_correctness":0.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.375,"accuracy_wilson_95":{"lower":0.0,"upper":0.32440756488388023,"confidence_level":0.95}}},"drop":{"n":22,"raw":{"n":22,"acceptable":20,"unsafe":2,"review":0,"automated_count":22,"automated_acceptable":20,"automated_unsafe":2,"unnecessary_reviews":0,"accuracy":0.9090909090909091,"unsafe_rate":0.09090909090909091,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.9090909090909091,"conditional_unsafe":0.09090909090909091,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.7218505315839417,"upper":0.9747045710090046,"confidence_level":0.95}},"effective":{"n":22,"acceptable":17,"unsafe":0,"review":5,"automated_count":17,"automated_acceptable":17,"automated_unsafe":0,"unnecessary_reviews":5,"accuracy":0.7727272727272727,"unsafe_rate":0.0,"review_rate":0.22727272727272727,"automated_coverage":0.7727272727272727,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.22727272727272727,"accuracy_wilson_95":{"lower":0.5656004682477829,"upper":0.8987696001475147,"confidence_level":0.95}}},"mask":{"n":8,"raw":{"n":8,"acceptable":0,"unsafe":0,"review":0,"automated_count":8,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.32440756488388023,"confidence_level":0.95}},"effective":{"n":8,"acceptable":0,"unsafe":0,"review":6,"automated_count":2,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":6,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":0.75,"automated_coverage":0.25,"conditional_correctness":0.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.75,"accuracy_wilson_95":{"lower":0.0,"upper":0.32440756488388023,"confidence_level":0.95}}},"pseudonymize":{"n":8,"raw":{"n":8,"acceptable":0,"unsafe":4,"review":0,"automated_count":8,"automated_acceptable":0,"automated_unsafe":4,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":0.5,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":0.5,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.32440756488388023,"confidence_level":0.95}},"effective":{"n":8,"acceptable":0,"unsafe":0,"review":8,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":8,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.32440756488388023,"confidence_level":0.95}}},"generalize":{"n":8,"raw":{"n":8,"acceptable":4,"unsafe":4,"review":0,"automated_count":8,"automated_acceptable":4,"automated_unsafe":4,"unnecessary_reviews":0,"accuracy":0.5,"unsafe_rate":0.5,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.5,"conditional_unsafe":0.5,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.21521606221387757,"upper":0.7847839377861224,"confidence_level":0.95}},"effective":{"n":8,"acceptable":3,"unsafe":0,"review":5,"automated_count":3,"automated_acceptable":3,"automated_unsafe":0,"unnecessary_reviews":5,"accuracy":0.375,"unsafe_rate":0.0,"review_rate":0.625,"automated_coverage":0.375,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.625,"accuracy_wilson_95":{"lower":0.13684428582359742,"upper":0.6942576053973727,"confidence_level":0.95}}},"review":{"n":2,"raw":{"n":2,"acceptable":0,"unsafe":0,"review":0,"automated_count":2,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.6576197724933469,"confidence_level":0.95}},"effective":{"n":2,"acceptable":1,"unsafe":0,"review":1,"automated_count":1,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":0.5,"unsafe_rate":0.0,"review_rate":0.5,"automated_coverage":0.5,"conditional_correctness":0.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.09453120573423074,"upper":0.9054687942657693,"confidence_level":0.95}}}}},"challenge56-compact_no_samples":{"n":56,"raw":{"n":56,"acceptable":29,"unsafe":6,"review":0,"automated_count":56,"automated_acceptable":29,"automated_unsafe":6,"unnecessary_reviews":0,"accuracy":0.5178571428571429,"unsafe_rate":0.10714285714285714,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.5178571428571429,"conditional_unsafe":0.10714285714285714,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.39010383924176983,"upper":0.6433178058754173,"confidence_level":0.95}},"effective":{"n":56,"acceptable":25,"unsafe":0,"review":18,"automated_count":38,"automated_acceptable":25,"automated_unsafe":0,"unnecessary_reviews":18,"accuracy":0.44642857142857145,"unsafe_rate":0.0,"review_rate":0.32142857142857145,"automated_coverage":0.6785714285714286,"conditional_correctness":0.6578947368421053,"conditional_unsafe":0.0,"unnecessary_review_rate":0.32142857142857145,"accuracy_wilson_95":{"lower":0.3238672125478337,"upper":0.5758678521006054,"confidence_level":0.95}},"confusion":{"raw":{"keep":{"keep":1,"drop":6,"mask":0,"pseudonymize":1,"generalize":0,"review":0},"drop":{"keep":0,"drop":22,"mask":0,"pseudonymize":0,"generalize":0,"review":0},"mask":{"keep":2,"drop":5,"mask":1,"pseudonymize":0,"generalize":0,"review":0},"pseudonymize":{"keep":4,"drop":4,"mask":0,"pseudonymize":0,"generalize":0,"review":0},"generalize":{"keep":0,"drop":3,"mask":0,"pseudonymize":0,"generalize":5,"review":0},"review":{"keep":0,"drop":2,"mask":0,"pseudonymize":0,"generalize":0,"review":0}},"effective":{"keep":{"keep":0,"drop":5,"mask":0,"pseudonymize":0,"generalize":0,"review":3},"drop":{"keep":0,"drop":22,"mask":0,"pseudonymize":0,"generalize":0,"review":0},"mask":{"keep":0,"drop":2,"mask":0,"pseudonymize":0,"generalize":0,"review":6},"pseudonymize":{"keep":0,"drop":3,"mask":0,"pseudonymize":0,"generalize":0,"review":5},"generalize":{"keep":0,"drop":1,"mask":0,"pseudonymize":0,"generalize":3,"review":4},"review":{"keep":0,"drop":2,"mask":0,"pseudonymize":0,"generalize":0,"review":0}}},"probability_metrics":{"singleton_n":56,"multiacceptable_n":0,"multiacceptable_coverage":0.0,"nll":1.342748993269995,"nll_zero_mass_n":0,"brier":0.6277792315357861,"acceptable_set_n":56,"acceptable_set_nll":1.342748993269995,"acceptable_set_zero_mass_n":0},"calibration_all_labels":{"equal_width":{"n":56,"bin_count":10,"ece":0.21780231461993282,"signed_mean_gap":0.13631437532603738,"absolute_mean_gap":0.13631437532603738,"bins":[{"index":0,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.0,"upper":0.1,"upper_inclusive":false},{"index":1,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.1,"upper":0.2,"upper_inclusive":false},{"index":2,"count":3,"confidence":0.27798477311929065,"empirical_correctness":0.6666666666666666,"signed_gap":-0.388681893547376,"absolute_gap":0.388681893547376,"lower":0.2,"upper":0.3,"upper_inclusive":false},{"index":3,"count":7,"confidence":0.357783636876515,"empirical_correctness":0.14285714285714285,"signed_gap":0.21492649401937214,"absolute_gap":0.21492649401937214,"lower":0.3,"upper":0.4,"upper_inclusive":false},{"index":4,"count":11,"confidence":0.4548523480241949,"empirical_correctness":0.09090909090909091,"signed_gap":0.363943257115104,"absolute_gap":0.363943257115104,"lower":0.4,"upper":0.5,"upper_inclusive":false},{"index":5,"count":2,"confidence":0.5573991537094116,"empirical_correctness":0.0,"signed_gap":0.5573991537094116,"absolute_gap":0.5573991537094116,"lower":0.5,"upper":0.6,"upper_inclusive":false},{"index":6,"count":6,"confidence":0.6680030028025309,"empirical_correctness":0.5,"signed_gap":0.1680030028025309,"absolute_gap":0.1680030028025309,"lower":0.6,"upper":0.7,"upper_inclusive":false},{"index":7,"count":8,"confidence":0.7355479225516319,"empirical_correctness":0.875,"signed_gap":-0.13945207744836807,"absolute_gap":0.13945207744836807,"lower":0.7,"upper":0.8,"upper_inclusive":false},{"index":8,"count":8,"confidence":0.8501963466405869,"empirical_correctness":0.75,"signed_gap":0.10019634664058685,"absolute_gap":0.10019634664058685,"lower":0.8,"upper":0.9,"upper_inclusive":false},{"index":9,"count":11,"confidence":0.9530017213387922,"empirical_correctness":0.8181818181818182,"signed_gap":0.134819903156974,"absolute_gap":0.134819903156974,"lower":0.9,"upper":1.0,"upper_inclusive":true}]},"equal_mass":{"n":56,"bin_count":10,"ece":0.17480551637709144,"signed_mean_gap":0.1363143753260374,"absolute_mean_gap":0.1363143753260374,"bins":[{"index":0,"count":5,"confidence":0.2973869413137436,"empirical_correctness":0.4,"signed_gap":-0.10261305868625642,"absolute_gap":0.10261305868625642,"min_confidence":0.24627210199832916,"max_confidence":0.3334280550479889},{"index":1,"count":6,"confidence":0.3774038602908452,"empirical_correctness":0.3333333333333333,"signed_gap":0.0440705269575119,"absolute_gap":0.0440705269575119,"min_confidence":0.3359266519546509,"max_confidence":0.4129180908203125},{"index":2,"count":5,"confidence":0.4363486170768738,"empirical_correctness":0.0,"signed_gap":0.4363486170768738,"absolute_gap":0.4363486170768738,"min_confidence":0.4157581031322479,"max_confidence":0.4592623710632324},{"index":3,"count":6,"confidence":0.48998840649922687,"empirical_correctness":0.0,"signed_gap":0.48998840649922687,"absolute_gap":0.48998840649922687,"min_confidence":0.4670828878879547,"max_confidence":0.5312157869338989},{"index":4,"count":6,"confidence":0.6511423190434774,"empirical_correctness":0.5,"signed_gap":0.1511423190434774,"absolute_gap":0.1511423190434774,"min_confidence":0.5835825204849243,"max_confidence":0.6797832250595093},{"index":5,"count":5,"confidence":0.7115273714065552,"empirical_correctness":0.8,"signed_gap":-0.08847262859344485,"absolute_gap":0.08847262859344485,"min_confidence":0.6847466230392456,"max_confidence":0.7346122860908508},{"index":6,"count":6,"confidence":0.7720510264237722,"empirical_correctness":0.6666666666666666,"signed_gap":0.10538435975710558,"absolute_gap":0.10538435975710558,"min_confidence":0.7415578961372375,"max_confidence":0.8191949725151062},{"index":7,"count":5,"confidence":0.857473087310791,"empirical_correctness":0.8,"signed_gap":0.05747308731079093,"absolute_gap":0.05747308731079093,"min_confidence":0.8411946892738342,"max_confidence":0.8775652050971985},{"index":8,"count":6,"confidence":0.9164557953675588,"empirical_correctness":0.6666666666666666,"signed_gap":0.2497891287008922,"absolute_gap":0.2497891287008922,"min_confidence":0.8933923244476318,"max_confidence":0.9428204298019409},{"index":9,"count":6,"confidence":0.9796127478281657,"empirical_correctness":1.0,"signed_gap":-0.020387252171834347,"absolute_gap":0.020387252171834347,"min_confidence":0.9498463869094849,"max_confidence":0.9975053668022156}]}},"calibration_singleton_labels":{"equal_width":{"n":56,"bin_count":10,"ece":0.21780231461993282,"signed_mean_gap":0.13631437532603738,"absolute_mean_gap":0.13631437532603738,"bins":[{"index":0,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.0,"upper":0.1,"upper_inclusive":false},{"index":1,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.1,"upper":0.2,"upper_inclusive":false},{"index":2,"count":3,"confidence":0.27798477311929065,"empirical_correctness":0.6666666666666666,"signed_gap":-0.388681893547376,"absolute_gap":0.388681893547376,"lower":0.2,"upper":0.3,"upper_inclusive":false},{"index":3,"count":7,"confidence":0.357783636876515,"empirical_correctness":0.14285714285714285,"signed_gap":0.21492649401937214,"absolute_gap":0.21492649401937214,"lower":0.3,"upper":0.4,"upper_inclusive":false},{"index":4,"count":11,"confidence":0.4548523480241949,"empirical_correctness":0.09090909090909091,"signed_gap":0.363943257115104,"absolute_gap":0.363943257115104,"lower":0.4,"upper":0.5,"upper_inclusive":false},{"index":5,"count":2,"confidence":0.5573991537094116,"empirical_correctness":0.0,"signed_gap":0.5573991537094116,"absolute_gap":0.5573991537094116,"lower":0.5,"upper":0.6,"upper_inclusive":false},{"index":6,"count":6,"confidence":0.6680030028025309,"empirical_correctness":0.5,"signed_gap":0.1680030028025309,"absolute_gap":0.1680030028025309,"lower":0.6,"upper":0.7,"upper_inclusive":false},{"index":7,"count":8,"confidence":0.7355479225516319,"empirical_correctness":0.875,"signed_gap":-0.13945207744836807,"absolute_gap":0.13945207744836807,"lower":0.7,"upper":0.8,"upper_inclusive":false},{"index":8,"count":8,"confidence":0.8501963466405869,"empirical_correctness":0.75,"signed_gap":0.10019634664058685,"absolute_gap":0.10019634664058685,"lower":0.8,"upper":0.9,"upper_inclusive":false},{"index":9,"count":11,"confidence":0.9530017213387922,"empirical_correctness":0.8181818181818182,"signed_gap":0.134819903156974,"absolute_gap":0.134819903156974,"lower":0.9,"upper":1.0,"upper_inclusive":true}]},"equal_mass":{"n":56,"bin_count":10,"ece":0.17480551637709144,"signed_mean_gap":0.1363143753260374,"absolute_mean_gap":0.1363143753260374,"bins":[{"index":0,"count":5,"confidence":0.2973869413137436,"empirical_correctness":0.4,"signed_gap":-0.10261305868625642,"absolute_gap":0.10261305868625642,"min_confidence":0.24627210199832916,"max_confidence":0.3334280550479889},{"index":1,"count":6,"confidence":0.3774038602908452,"empirical_correctness":0.3333333333333333,"signed_gap":0.0440705269575119,"absolute_gap":0.0440705269575119,"min_confidence":0.3359266519546509,"max_confidence":0.4129180908203125},{"index":2,"count":5,"confidence":0.4363486170768738,"empirical_correctness":0.0,"signed_gap":0.4363486170768738,"absolute_gap":0.4363486170768738,"min_confidence":0.4157581031322479,"max_confidence":0.4592623710632324},{"index":3,"count":6,"confidence":0.48998840649922687,"empirical_correctness":0.0,"signed_gap":0.48998840649922687,"absolute_gap":0.48998840649922687,"min_confidence":0.4670828878879547,"max_confidence":0.5312157869338989},{"index":4,"count":6,"confidence":0.6511423190434774,"empirical_correctness":0.5,"signed_gap":0.1511423190434774,"absolute_gap":0.1511423190434774,"min_confidence":0.5835825204849243,"max_confidence":0.6797832250595093},{"index":5,"count":5,"confidence":0.7115273714065552,"empirical_correctness":0.8,"signed_gap":-0.08847262859344485,"absolute_gap":0.08847262859344485,"min_confidence":0.6847466230392456,"max_confidence":0.7346122860908508},{"index":6,"count":6,"confidence":0.7720510264237722,"empirical_correctness":0.6666666666666666,"signed_gap":0.10538435975710558,"absolute_gap":0.10538435975710558,"min_confidence":0.7415578961372375,"max_confidence":0.8191949725151062},{"index":7,"count":5,"confidence":0.857473087310791,"empirical_correctness":0.8,"signed_gap":0.05747308731079093,"absolute_gap":0.05747308731079093,"min_confidence":0.8411946892738342,"max_confidence":0.8775652050971985},{"index":8,"count":6,"confidence":0.9164557953675588,"empirical_correctness":0.6666666666666666,"signed_gap":0.2497891287008922,"absolute_gap":0.2497891287008922,"min_confidence":0.8933923244476318,"max_confidence":0.9428204298019409},{"index":9,"count":6,"confidence":0.9796127478281657,"empirical_correctness":1.0,"signed_gap":-0.020387252171834347,"absolute_gap":0.020387252171834347,"min_confidence":0.9498463869094849,"max_confidence":0.9975053668022156}]}},"latency_ms":{"n":56,"median":30.598999932408333,"p95":31.29247488686815},"per_action":{"keep":{"n":8,"raw":{"n":8,"acceptable":1,"unsafe":0,"review":0,"automated_count":8,"automated_acceptable":1,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":0.125,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.125,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.02241749145005667,"upper":0.4708881822128535,"confidence_level":0.95}},"effective":{"n":8,"acceptable":0,"unsafe":0,"review":3,"automated_count":5,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":3,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":0.375,"automated_coverage":0.625,"conditional_correctness":0.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.375,"accuracy_wilson_95":{"lower":0.0,"upper":0.32440756488388023,"confidence_level":0.95}}},"drop":{"n":22,"raw":{"n":22,"acceptable":22,"unsafe":0,"review":0,"automated_count":22,"automated_acceptable":22,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":1.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.8513451253913791,"upper":1.0,"confidence_level":0.95}},"effective":{"n":22,"acceptable":22,"unsafe":0,"review":0,"automated_count":22,"automated_acceptable":22,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":1.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.8513451253913791,"upper":1.0,"confidence_level":0.95}}},"mask":{"n":8,"raw":{"n":8,"acceptable":1,"unsafe":2,"review":0,"automated_count":8,"automated_acceptable":1,"automated_unsafe":2,"unnecessary_reviews":0,"accuracy":0.125,"unsafe_rate":0.25,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.125,"conditional_unsafe":0.25,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.02241749145005667,"upper":0.4708881822128535,"confidence_level":0.95}},"effective":{"n":8,"acceptable":0,"unsafe":0,"review":6,"automated_count":2,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":6,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":0.75,"automated_coverage":0.25,"conditional_correctness":0.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.75,"accuracy_wilson_95":{"lower":0.0,"upper":0.32440756488388023,"confidence_level":0.95}}},"pseudonymize":{"n":8,"raw":{"n":8,"acceptable":0,"unsafe":4,"review":0,"automated_count":8,"automated_acceptable":0,"automated_unsafe":4,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":0.5,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":0.5,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.32440756488388023,"confidence_level":0.95}},"effective":{"n":8,"acceptable":0,"unsafe":0,"review":5,"automated_count":3,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":5,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":0.625,"automated_coverage":0.375,"conditional_correctness":0.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.625,"accuracy_wilson_95":{"lower":0.0,"upper":0.32440756488388023,"confidence_level":0.95}}},"generalize":{"n":8,"raw":{"n":8,"acceptable":5,"unsafe":0,"review":0,"automated_count":8,"automated_acceptable":5,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":0.625,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.625,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.30574239460262737,"upper":0.8631557141764026,"confidence_level":0.95}},"effective":{"n":8,"acceptable":3,"unsafe":0,"review":4,"automated_count":4,"automated_acceptable":3,"automated_unsafe":0,"unnecessary_reviews":4,"accuracy":0.375,"unsafe_rate":0.0,"review_rate":0.5,"automated_coverage":0.5,"conditional_correctness":0.75,"conditional_unsafe":0.0,"unnecessary_review_rate":0.5,"accuracy_wilson_95":{"lower":0.13684428582359742,"upper":0.6942576053973727,"confidence_level":0.95}}},"review":{"n":2,"raw":{"n":2,"acceptable":0,"unsafe":0,"review":0,"automated_count":2,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.6576197724933469,"confidence_level":0.95}},"effective":{"n":2,"acceptable":0,"unsafe":0,"review":0,"automated_count":2,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.6576197724933469,"confidence_level":0.95}}}}}},"composite":{"n":198,"scope":"descriptive sum across four correlated panels","raw":{"n":198,"acceptable":96,"unsafe":19,"review":0,"automated_count":198,"automated_acceptable":96,"automated_unsafe":19,"unnecessary_reviews":0,"accuracy":0.48484848484848486,"unsafe_rate":0.09595959595959595,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.48484848484848486,"conditional_unsafe":0.09595959595959595,"unnecessary_review_rate":0.0},"effective":{"n":198,"acceptable":86,"unsafe":0,"review":57,"automated_count":141,"automated_acceptable":84,"automated_unsafe":0,"unnecessary_reviews":55,"accuracy":0.43434343434343436,"unsafe_rate":0.0,"review_rate":0.2878787878787879,"automated_coverage":0.7121212121212122,"conditional_correctness":0.5957446808510638,"conditional_unsafe":0.0,"unnecessary_review_rate":0.2777777777777778}}},{"id":"impacte--mimir-laya-router-fp32","platform":"Local","model_id":"impacte/mimir-laya-router","revision":"441057d7578bf6a6ee8cf207a44b00cf01a0738e","loading":"float32 / none","scoring_kind":"Native decision readout","settings":{"dtype":"float32","quantization":"none","max_tokens":512,"head_max_tokens":192,"temperature":4.323858261108398,"autocast":false,"truncation":false,"cpu_offload":false,"guard_threshold":0.45,"seed":0,"tf32":false},"scoring_metadata":{"backend":"laya_native_local_builtin","readout":"native_mask_slot_scorer_numpy_float32_softmax","probability_source":"native_choice_unrounded_published_temperature_not_task_calibrated","latency_scope":"synchronized serial complete encoder+head+act forward,input transfer and native readout; tokenization excluded","native_disclosures":["Stock installed torch attention in place of ONNX dynamic equivalent; no custom kernels or repo imports.","FP32/no-autocast explicit precision: convert every saved F16/BF16/F32 tensor toFP32; no quantization.","Unrounded native probability; guard uses winner p, not entropy-concentration confidence.","Act head fully loaded and evaluated; not used to alter six-action answer.","Native choice:6-10 bucket else choice scalar0, T=4.323858261108398. No task fitting, priors correction, prompt change or finetuning here."],"equal_mass_convention":"rank bins with floor boundaries at floor(i*n/k), k=min(10,n//2); minimum two rows"},"load_ms":null,"peak_allocated_bytes":1748760576,"peak_reserved_bytes":1870659584,"panels":{"diagnostic43-no_samples":{"n":43,"raw":{"n":43,"acceptable":15,"unsafe":17,"review":0,"automated_count":43,"automated_acceptable":15,"automated_unsafe":17,"unnecessary_reviews":0,"accuracy":0.3488372093023256,"unsafe_rate":0.3953488372093023,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.3488372093023256,"conditional_unsafe":0.3953488372093023,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.22418589997352706,"upper":0.49828218007147956,"confidence_level":0.95}},"effective":{"n":43,"acceptable":17,"unsafe":3,"review":14,"automated_count":29,"automated_acceptable":15,"automated_unsafe":3,"unnecessary_reviews":12,"accuracy":0.3953488372093023,"unsafe_rate":0.06976744186046512,"review_rate":0.32558139534883723,"automated_coverage":0.6744186046511628,"conditional_correctness":0.5172413793103449,"conditional_unsafe":0.10344827586206896,"unnecessary_review_rate":0.27906976744186046,"accuracy_wilson_95":{"lower":0.2636531475829744,"upper":0.5442093693712609,"confidence_level":0.95}},"confusion":{"raw":{"keep":{"keep":5,"drop":7,"mask":0,"pseudonymize":0,"generalize":0,"review":0},"drop":{"keep":9,"drop":7,"mask":0,"pseudonymize":0,"generalize":0,"review":0},"mask":{"keep":0,"drop":2,"mask":0,"pseudonymize":0,"generalize":0,"review":0},"pseudonymize":{"keep":2,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0},"generalize":{"keep":4,"drop":2,"mask":0,"pseudonymize":0,"generalize":0,"review":0},"review":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0}},"effective":{"keep":{"keep":5,"drop":7,"mask":0,"pseudonymize":0,"generalize":0,"review":0},"drop":{"keep":2,"drop":7,"mask":0,"pseudonymize":0,"generalize":0,"review":7},"mask":{"keep":0,"drop":2,"mask":0,"pseudonymize":0,"generalize":0,"review":0},"pseudonymize":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":2},"generalize":{"keep":1,"drop":2,"mask":0,"pseudonymize":0,"generalize":0,"review":3},"review":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0}}},"probability_metrics":{"singleton_n":38,"multiacceptable_n":5,"multiacceptable_coverage":0.11627906976744186,"nll":2.3132848876822525,"nll_zero_mass_n":0,"brier":1.0509876603582553,"acceptable_set_n":43,"acceptable_set_nll":2.108481866341781,"acceptable_set_zero_mass_n":0},"calibration_all_labels":{"equal_width":{"n":43,"bin_count":10,"ece":0.4176325895065485,"signed_mean_gap":0.4176325895065485,"absolute_mean_gap":0.4176325895065485,"bins":[{"index":0,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.0,"upper":0.1,"upper_inclusive":false},{"index":1,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.1,"upper":0.2,"upper_inclusive":false},{"index":2,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.2,"upper":0.3,"upper_inclusive":false},{"index":3,"count":1,"confidence":0.3420577943325043,"empirical_correctness":0.0,"signed_gap":0.3420577943325043,"absolute_gap":0.3420577943325043,"lower":0.3,"upper":0.4,"upper_inclusive":false},{"index":4,"count":2,"confidence":0.4961491972208023,"empirical_correctness":0.0,"signed_gap":0.4961491972208023,"absolute_gap":0.4961491972208023,"lower":0.4,"upper":0.5,"upper_inclusive":false},{"index":5,"count":6,"confidence":0.5491306682427725,"empirical_correctness":0.5,"signed_gap":0.04913066824277246,"absolute_gap":0.04913066824277246,"lower":0.5,"upper":0.6,"upper_inclusive":false},{"index":6,"count":6,"confidence":0.6563655336697897,"empirical_correctness":0.5,"signed_gap":0.15636553366978967,"absolute_gap":0.15636553366978967,"lower":0.6,"upper":0.7,"upper_inclusive":false},{"index":7,"count":5,"confidence":0.7597237586975097,"empirical_correctness":0.2,"signed_gap":0.5597237586975097,"absolute_gap":0.5597237586975097,"lower":0.7,"upper":0.8,"upper_inclusive":false},{"index":8,"count":12,"confidence":0.8657783766587576,"empirical_correctness":0.4166666666666667,"signed_gap":0.4491117099920909,"absolute_gap":0.4491117099920909,"lower":0.8,"upper":0.9,"upper_inclusive":false},{"index":9,"count":11,"confidence":0.9275371486490424,"empirical_correctness":0.2727272727272727,"signed_gap":0.6548098759217696,"absolute_gap":0.6548098759217696,"lower":0.9,"upper":1.0,"upper_inclusive":true}]},"equal_mass":{"n":43,"bin_count":10,"ece":0.4546644036159959,"signed_mean_gap":0.4176325895065485,"absolute_mean_gap":0.4176325895065485,"bins":[{"index":0,"count":4,"confidence":0.46404772996902466,"empirical_correctness":0.0,"signed_gap":0.46404772996902466,"absolute_gap":0.46404772996902466,"min_confidence":0.3420577943325043,"max_confidence":0.5218347311019897},{"index":1,"count":4,"confidence":0.5509539991617203,"empirical_correctness":0.75,"signed_gap":-0.19904600083827972,"absolute_gap":0.19904600083827972,"min_confidence":0.5379091501235962,"max_confidence":0.5646947622299194},{"index":2,"count":4,"confidence":0.616007924079895,"empirical_correctness":0.5,"signed_gap":0.11600792407989502,"absolute_gap":0.11600792407989502,"min_confidence":0.5691332817077637,"max_confidence":0.6501633524894714},{"index":3,"count":5,"confidence":0.7067438125610351,"empirical_correctness":0.2,"signed_gap":0.5067438125610351,"absolute_gap":0.5067438125610351,"min_confidence":0.6619250774383545,"max_confidence":0.7572488188743591},{"index":4,"count":4,"confidence":0.7775685042142868,"empirical_correctness":0.25,"signed_gap":0.5275685042142868,"absolute_gap":0.5275685042142868,"min_confidence":0.7591113448143005,"max_confidence":0.8020794987678528},{"index":5,"count":4,"confidence":0.8552632480859756,"empirical_correctness":0.5,"signed_gap":0.35526324808597565,"absolute_gap":0.35526324808597565,"min_confidence":0.839147686958313,"max_confidence":0.8682568669319153},{"index":6,"count":5,"confidence":0.8770645380020141,"empirical_correctness":0.4,"signed_gap":0.4770645380020141,"absolute_gap":0.4770645380020141,"min_confidence":0.8700669407844543,"max_confidence":0.8860844969749451},{"index":7,"count":4,"confidence":0.9005301296710968,"empirical_correctness":0.5,"signed_gap":0.4005301296710968,"absolute_gap":0.4005301296710968,"min_confidence":0.88762366771698,"max_confidence":0.9123690724372864},{"index":8,"count":4,"confidence":0.9218508899211884,"empirical_correctness":0.25,"signed_gap":0.6718508899211884,"absolute_gap":0.6718508899211884,"min_confidence":0.9124030470848083,"max_confidence":0.9315119385719299},{"index":9,"count":5,"confidence":0.9388539791107178,"empirical_correctness":0.2,"signed_gap":0.7388539791107178,"absolute_gap":0.7388539791107178,"min_confidence":0.9364524483680725,"max_confidence":0.9421296119689941}]}},"calibration_singleton_labels":{"equal_width":{"n":38,"bin_count":10,"ece":0.4782690515643672,"signed_mean_gap":0.4782690515643672,"absolute_mean_gap":0.4782690515643672,"bins":[{"index":0,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.0,"upper":0.1,"upper_inclusive":false},{"index":1,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.1,"upper":0.2,"upper_inclusive":false},{"index":2,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.2,"upper":0.3,"upper_inclusive":false},{"index":3,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.3,"upper":0.4,"upper_inclusive":false},{"index":4,"count":1,"confidence":0.49362611770629883,"empirical_correctness":0.0,"signed_gap":0.49362611770629883,"absolute_gap":0.49362611770629883,"lower":0.4,"upper":0.5,"upper_inclusive":false},{"index":5,"count":5,"confidence":0.5489724278450012,"empirical_correctness":0.4,"signed_gap":0.1489724278450012,"absolute_gap":0.1489724278450012,"lower":0.5,"upper":0.6,"upper_inclusive":false},{"index":6,"count":5,"confidence":0.6607958197593689,"empirical_correctness":0.4,"signed_gap":0.26079581975936883,"absolute_gap":0.26079581975936883,"lower":0.6,"upper":0.7,"upper_inclusive":false},{"index":7,"count":4,"confidence":0.7598768621683121,"empirical_correctness":0.0,"signed_gap":0.7598768621683121,"absolute_gap":0.7598768621683121,"lower":0.7,"upper":0.8,"upper_inclusive":false},{"index":8,"count":12,"confidence":0.8657783766587576,"empirical_correctness":0.4166666666666667,"signed_gap":0.4491117099920909,"absolute_gap":0.4491117099920909,"lower":0.8,"upper":0.9,"upper_inclusive":false},{"index":9,"count":11,"confidence":0.9275371486490424,"empirical_correctness":0.2727272727272727,"signed_gap":0.6548098759217696,"absolute_gap":0.6548098759217696,"lower":0.9,"upper":1.0,"upper_inclusive":true}]},"equal_mass":{"n":38,"bin_count":10,"ece":0.5356284991690987,"signed_mean_gap":0.4782690515643672,"absolute_mean_gap":0.4782690515643672,"bins":[{"index":0,"count":3,"confidence":0.5177899996439616,"empirical_correctness":0.0,"signed_gap":0.5177899996439616,"absolute_gap":0.5177899996439616,"min_confidence":0.49362611770629883,"max_confidence":0.5379091501235962},{"index":1,"count":4,"confidence":0.573909804224968,"empirical_correctness":0.75,"signed_gap":-0.17609019577503204,"absolute_gap":0.17609019577503204,"min_confidence":0.5512902140617371,"max_confidence":0.6105209589004517},{"index":2,"count":4,"confidence":0.6733645349740982,"empirical_correctness":0.25,"signed_gap":0.4233645349740982,"absolute_gap":0.4233645349740982,"min_confidence":0.6501633524894714,"max_confidence":0.6919435262680054},{"index":3,"count":4,"confidence":0.7598768621683121,"empirical_correctness":0.0,"signed_gap":0.7598768621683121,"absolute_gap":0.7598768621683121,"min_confidence":0.7331754565238953,"max_confidence":0.7815287709236145},{"index":4,"count":4,"confidence":0.83871890604496,"empirical_correctness":0.25,"signed_gap":0.58871890604496,"absolute_gap":0.58871890604496,"min_confidence":0.8020794987678528,"max_confidence":0.8642519116401672},{"index":5,"count":3,"confidence":0.8715104262034098,"empirical_correctness":1.0,"signed_gap":-0.1284895737965902,"absolute_gap":0.1284895737965902,"min_confidence":0.8682568669319153,"max_confidence":0.8762074708938599},{"index":6,"count":4,"confidence":0.8816679865121841,"empirical_correctness":0.25,"signed_gap":0.6316679865121841,"absolute_gap":0.6316679865121841,"min_confidence":0.876295804977417,"max_confidence":0.88762366771698},{"index":7,"count":4,"confidence":0.9067249745130539,"empirical_correctness":0.5,"signed_gap":0.4067249745130539,"absolute_gap":0.4067249745130539,"min_confidence":0.8932616710662842,"max_confidence":0.9124030470848083},{"index":8,"count":4,"confidence":0.9278632402420044,"empirical_correctness":0.0,"signed_gap":0.9278632402420044,"absolute_gap":0.9278632402420044,"min_confidence":0.9208023548126221,"max_confidence":0.9364524483680725},{"index":9,"count":4,"confidence":0.9394543617963791,"empirical_correctness":0.25,"signed_gap":0.6894543617963791,"absolute_gap":0.6894543617963791,"min_confidence":0.9370092153549194,"max_confidence":0.9421296119689941}]}},"latency_ms":{"n":43,"median":18.113500031176955,"p95":18.795909022446722},"per_action":{"keep":{"n":12,"raw":{"n":12,"acceptable":5,"unsafe":0,"review":0,"automated_count":12,"automated_acceptable":5,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":0.4166666666666667,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.4166666666666667,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.19326031365875665,"upper":0.6804886874504503,"confidence_level":0.95}},"effective":{"n":12,"acceptable":5,"unsafe":0,"review":0,"automated_count":12,"automated_acceptable":5,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":0.4166666666666667,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.4166666666666667,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.19326031365875665,"upper":0.6804886874504503,"confidence_level":0.95}}},"drop":{"n":16,"raw":{"n":16,"acceptable":7,"unsafe":9,"review":0,"automated_count":16,"automated_acceptable":7,"automated_unsafe":9,"unnecessary_reviews":0,"accuracy":0.4375,"unsafe_rate":0.5625,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.4375,"conditional_unsafe":0.5625,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.23098652405492354,"upper":0.6682144360118811,"confidence_level":0.95}},"effective":{"n":16,"acceptable":7,"unsafe":2,"review":7,"automated_count":9,"automated_acceptable":7,"automated_unsafe":2,"unnecessary_reviews":7,"accuracy":0.4375,"unsafe_rate":0.125,"review_rate":0.4375,"automated_coverage":0.5625,"conditional_correctness":0.7777777777777778,"conditional_unsafe":0.2222222222222222,"unnecessary_review_rate":0.4375,"accuracy_wilson_95":{"lower":0.23098652405492354,"upper":0.6682144360118811,"confidence_level":0.95}}},"mask":{"n":2,"raw":{"n":2,"acceptable":0,"unsafe":0,"review":0,"automated_count":2,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.6576197724933469,"confidence_level":0.95}},"effective":{"n":2,"acceptable":0,"unsafe":0,"review":0,"automated_count":2,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.6576197724933469,"confidence_level":0.95}}},"pseudonymize":{"n":2,"raw":{"n":2,"acceptable":0,"unsafe":2,"review":0,"automated_count":2,"automated_acceptable":0,"automated_unsafe":2,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":1.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":1.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.6576197724933469,"confidence_level":0.95}},"effective":{"n":2,"acceptable":0,"unsafe":0,"review":2,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":2,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.6576197724933469,"confidence_level":0.95}}},"generalize":{"n":6,"raw":{"n":6,"acceptable":0,"unsafe":4,"review":0,"automated_count":6,"automated_acceptable":0,"automated_unsafe":4,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":0.6666666666666666,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":0.6666666666666666,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":2.7755575615628914e-17,"upper":0.3903342879021653,"confidence_level":0.95}},"effective":{"n":6,"acceptable":0,"unsafe":1,"review":3,"automated_count":3,"automated_acceptable":0,"automated_unsafe":1,"unnecessary_reviews":3,"accuracy":0.0,"unsafe_rate":0.16666666666666666,"review_rate":0.5,"automated_coverage":0.5,"conditional_correctness":0.0,"conditional_unsafe":0.3333333333333333,"unnecessary_review_rate":0.5,"accuracy_wilson_95":{"lower":2.7755575615628914e-17,"upper":0.3903342879021653,"confidence_level":0.95}}},"review":{"n":0,"raw":{"n":0,"acceptable":0,"unsafe":0,"review":0,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":null,"unsafe_rate":null,"review_rate":null,"automated_coverage":null,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":null},"effective":{"n":0,"acceptable":0,"unsafe":0,"review":0,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":null,"unsafe_rate":null,"review_rate":null,"automated_coverage":null,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":null}}}},"diagnostic43-compact_no_samples":{"n":43,"raw":{"n":43,"acceptable":14,"unsafe":25,"review":0,"automated_count":43,"automated_acceptable":14,"automated_unsafe":25,"unnecessary_reviews":0,"accuracy":0.32558139534883723,"unsafe_rate":0.5813953488372093,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.32558139534883723,"conditional_unsafe":0.5813953488372093,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.20493297126568708,"upper":0.47483789032470525,"confidence_level":0.95}},"effective":{"n":43,"acceptable":20,"unsafe":3,"review":20,"automated_count":23,"automated_acceptable":15,"automated_unsafe":3,"unnecessary_reviews":15,"accuracy":0.46511627906976744,"unsafe_rate":0.06976744186046512,"review_rate":0.46511627906976744,"automated_coverage":0.5348837209302325,"conditional_correctness":0.6521739130434783,"conditional_unsafe":0.13043478260869565,"unnecessary_review_rate":0.3488372093023256,"accuracy_wilson_95":{"lower":0.32511057704121815,"upper":0.6108435952768602,"confidence_level":0.95}},"confusion":{"raw":{"keep":{"keep":9,"drop":0,"mask":0,"pseudonymize":0,"generalize":3,"review":0},"drop":{"keep":13,"drop":0,"mask":0,"pseudonymize":0,"generalize":3,"review":0},"mask":{"keep":2,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0},"pseudonymize":{"keep":2,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0},"generalize":{"keep":1,"drop":0,"mask":0,"pseudonymize":0,"generalize":5,"review":0},"review":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0}},"effective":{"keep":{"keep":9,"drop":0,"mask":0,"pseudonymize":0,"generalize":2,"review":1},"drop":{"keep":1,"drop":0,"mask":3,"pseudonymize":0,"generalize":2,"review":10},"mask":{"keep":0,"drop":0,"mask":1,"pseudonymize":0,"generalize":0,"review":1},"pseudonymize":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":2},"generalize":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":5,"review":1},"review":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0}}},"probability_metrics":{"singleton_n":38,"multiacceptable_n":5,"multiacceptable_coverage":0.11627906976744186,"nll":2.0485938407699207,"nll_zero_mass_n":0,"brier":0.9636219289686642,"acceptable_set_n":43,"acceptable_set_nll":2.022952528726824,"acceptable_set_zero_mass_n":0},"calibration_all_labels":{"equal_width":{"n":43,"bin_count":10,"ece":0.47774653725845867,"signed_mean_gap":0.47774653725845867,"absolute_mean_gap":0.47774653725845867,"bins":[{"index":0,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.0,"upper":0.1,"upper_inclusive":false},{"index":1,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.1,"upper":0.2,"upper_inclusive":false},{"index":2,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.2,"upper":0.3,"upper_inclusive":false},{"index":3,"count":1,"confidence":0.36736610531806946,"empirical_correctness":0.0,"signed_gap":0.36736610531806946,"absolute_gap":0.36736610531806946,"lower":0.3,"upper":0.4,"upper_inclusive":false},{"index":4,"count":4,"confidence":0.49097415804862976,"empirical_correctness":0.0,"signed_gap":0.49097415804862976,"absolute_gap":0.49097415804862976,"lower":0.4,"upper":0.5,"upper_inclusive":false},{"index":5,"count":4,"confidence":0.5482825338840485,"empirical_correctness":0.25,"signed_gap":0.29828253388404846,"absolute_gap":0.29828253388404846,"lower":0.5,"upper":0.6,"upper_inclusive":false},{"index":6,"count":4,"confidence":0.6748917698860168,"empirical_correctness":0.25,"signed_gap":0.42489176988601685,"absolute_gap":0.42489176988601685,"lower":0.6,"upper":0.7,"upper_inclusive":false},{"index":7,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.7,"upper":0.8,"upper_inclusive":false},{"index":8,"count":7,"confidence":0.8681850859097072,"empirical_correctness":0.42857142857142855,"signed_gap":0.4396136573382787,"absolute_gap":0.4396136573382787,"lower":0.8,"upper":0.9,"upper_inclusive":false},{"index":9,"count":23,"confidence":0.9235585020936053,"empirical_correctness":0.391304347826087,"signed_gap":0.5322541542675183,"absolute_gap":0.5322541542675183,"lower":0.9,"upper":1.0,"upper_inclusive":true}]},"equal_mass":{"n":43,"bin_count":10,"ece":0.47774653725845867,"signed_mean_gap":0.47774653725845867,"absolute_mean_gap":0.47774653725845867,"bins":[{"index":0,"count":4,"confidence":0.4593168646097183,"empirical_correctness":0.0,"signed_gap":0.4593168646097183,"absolute_gap":0.4593168646097183,"min_confidence":0.36736610531806946,"max_confidence":0.4925368130207062},{"index":1,"count":4,"confidence":0.5282729938626289,"empirical_correctness":0.25,"signed_gap":0.27827299386262894,"absolute_gap":0.27827299386262894,"min_confidence":0.4939952790737152,"max_confidence":0.5462015271186829},{"index":2,"count":4,"confidence":0.6446223706007004,"empirical_correctness":0.25,"signed_gap":0.3946223706007004,"absolute_gap":0.3946223706007004,"min_confidence":0.5740334391593933,"max_confidence":0.678918719291687},{"index":3,"count":5,"confidence":0.8217464327812195,"empirical_correctness":0.2,"signed_gap":0.6217464327812194,"absolute_gap":0.6217464327812194,"min_confidence":0.6951110363006592,"max_confidence":0.8682615160942078},{"index":4,"count":4,"confidence":0.8920451253652573,"empirical_correctness":0.75,"signed_gap":0.14204512536525726,"absolute_gap":0.14204512536525726,"min_confidence":0.8726578950881958,"max_confidence":0.9045060276985168},{"index":5,"count":4,"confidence":0.9085651487112045,"empirical_correctness":0.25,"signed_gap":0.6585651487112045,"absolute_gap":0.6585651487112045,"min_confidence":0.906955361366272,"max_confidence":0.9105781316757202},{"index":6,"count":5,"confidence":0.9206136107444763,"empirical_correctness":0.2,"signed_gap":0.7206136107444763,"absolute_gap":0.7206136107444763,"min_confidence":0.916549563407898,"max_confidence":0.9226888418197632},{"index":7,"count":4,"confidence":0.9254985749721527,"empirical_correctness":0.0,"signed_gap":0.9254985749721527,"absolute_gap":0.9254985749721527,"min_confidence":0.9235663414001465,"max_confidence":0.9270742535591125},{"index":8,"count":4,"confidence":0.931832566857338,"empirical_correctness":0.75,"signed_gap":0.18183256685733795,"absolute_gap":0.18183256685733795,"min_confidence":0.9313854575157166,"max_confidence":0.9323329329490662},{"index":9,"count":5,"confidence":0.9341372609138489,"empirical_correctness":0.6,"signed_gap":0.3341372609138489,"absolute_gap":0.3341372609138489,"min_confidence":0.9331580996513367,"max_confidence":0.9353649020195007}]}},"calibration_singleton_labels":{"equal_width":{"n":38,"bin_count":10,"ece":0.45292064704393087,"signed_mean_gap":0.45292064704393087,"absolute_mean_gap":0.45292064704393087,"bins":[{"index":0,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.0,"upper":0.1,"upper_inclusive":false},{"index":1,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.1,"upper":0.2,"upper_inclusive":false},{"index":2,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.2,"upper":0.3,"upper_inclusive":false},{"index":3,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.3,"upper":0.4,"upper_inclusive":false},{"index":4,"count":4,"confidence":0.49097415804862976,"empirical_correctness":0.0,"signed_gap":0.49097415804862976,"absolute_gap":0.49097415804862976,"lower":0.4,"upper":0.5,"upper_inclusive":false},{"index":5,"count":2,"confidence":0.5390897393226624,"empirical_correctness":0.5,"signed_gap":0.039089739322662354,"absolute_gap":0.039089739322662354,"lower":0.5,"upper":0.6,"upper_inclusive":false},{"index":6,"count":4,"confidence":0.6748917698860168,"empirical_correctness":0.25,"signed_gap":0.42489176988601685,"absolute_gap":0.42489176988601685,"lower":0.6,"upper":0.7,"upper_inclusive":false},{"index":7,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.7,"upper":0.8,"upper_inclusive":false},{"index":8,"count":7,"confidence":0.8681850859097072,"empirical_correctness":0.42857142857142855,"signed_gap":0.4396136573382787,"absolute_gap":0.4396136573382787,"lower":0.8,"upper":0.9,"upper_inclusive":false},{"index":9,"count":21,"confidence":0.9234307521865481,"empirical_correctness":0.42857142857142855,"signed_gap":0.4948593236151196,"absolute_gap":0.4948593236151196,"lower":0.9,"upper":1.0,"upper_inclusive":true}]},"equal_mass":{"n":38,"bin_count":10,"ece":0.45292064704393087,"signed_mean_gap":0.45292064704393087,"absolute_mean_gap":0.45292064704393087,"bins":[{"index":0,"count":3,"confidence":0.48996711770693463,"empirical_correctness":0.0,"signed_gap":0.48996711770693463,"absolute_gap":0.48996711770693463,"min_confidence":0.4886256456375122,"max_confidence":0.4925368130207062},{"index":1,"count":4,"confidence":0.5568832084536552,"empirical_correctness":0.25,"signed_gap":0.30688320845365524,"absolute_gap":0.30688320845365524,"min_confidence":0.4939952790737152,"max_confidence":0.655358076095581},{"index":2,"count":4,"confidence":0.7189524471759796,"empirical_correctness":0.25,"signed_gap":0.4689524471759796,"absolute_gap":0.4689524471759796,"min_confidence":0.6701792478561401,"max_confidence":0.8316007852554321},{"index":3,"count":4,"confidence":0.8636695593595505,"empirical_correctness":0.25,"signed_gap":0.6136695593595505,"absolute_gap":0.6136695593595505,"min_confidence":0.8457894325256348,"max_confidence":0.8726578950881958},{"index":4,"count":4,"confidence":0.9006194919347763,"empirical_correctness":0.75,"signed_gap":0.1506194919347763,"absolute_gap":0.1506194919347763,"min_confidence":0.8930768966674805,"max_confidence":0.906955361366272},{"index":5,"count":3,"confidence":0.9091017444928488,"empirical_correctness":0.3333333333333333,"signed_gap":0.5757684111595154,"absolute_gap":0.5757684111595154,"min_confidence":0.9081347584724426,"max_confidence":0.9105781316757202},{"index":6,"count":4,"confidence":0.9216296225786209,"empirical_correctness":0.25,"signed_gap":0.6716296225786209,"absolute_gap":0.6716296225786209,"min_confidence":0.9201556444168091,"max_confidence":0.9226888418197632},{"index":7,"count":4,"confidence":0.9254985749721527,"empirical_correctness":0.0,"signed_gap":0.9254985749721527,"absolute_gap":0.9254985749721527,"min_confidence":0.9235663414001465,"max_confidence":0.9270742535591125},{"index":8,"count":4,"confidence":0.931832566857338,"empirical_correctness":0.75,"signed_gap":0.18183256685733795,"absolute_gap":0.18183256685733795,"min_confidence":0.9313854575157166,"max_confidence":0.9323329329490662},{"index":9,"count":4,"confidence":0.9343590289354324,"empirical_correctness":0.75,"signed_gap":0.18435902893543243,"absolute_gap":0.18435902893543243,"min_confidence":0.9331580996513367,"max_confidence":0.9353649020195007}]}},"latency_ms":{"n":43,"median":20.72297700215131,"p95":21.15895397728309},"per_action":{"keep":{"n":12,"raw":{"n":12,"acceptable":9,"unsafe":0,"review":0,"automated_count":12,"automated_acceptable":9,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":0.75,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.75,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.46769466506643426,"upper":0.9110583316059453,"confidence_level":0.95}},"effective":{"n":12,"acceptable":9,"unsafe":0,"review":1,"automated_count":11,"automated_acceptable":9,"automated_unsafe":0,"unnecessary_reviews":1,"accuracy":0.75,"unsafe_rate":0.0,"review_rate":0.08333333333333333,"automated_coverage":0.9166666666666666,"conditional_correctness":0.8181818181818182,"conditional_unsafe":0.0,"unnecessary_review_rate":0.08333333333333333,"accuracy_wilson_95":{"lower":0.46769466506643426,"upper":0.9110583316059453,"confidence_level":0.95}}},"drop":{"n":16,"raw":{"n":16,"acceptable":0,"unsafe":15,"review":0,"automated_count":16,"automated_acceptable":0,"automated_unsafe":15,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":0.9375,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":0.9375,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.1936076805344365,"confidence_level":0.95}},"effective":{"n":16,"acceptable":0,"unsafe":3,"review":10,"automated_count":6,"automated_acceptable":0,"automated_unsafe":3,"unnecessary_reviews":10,"accuracy":0.0,"unsafe_rate":0.1875,"review_rate":0.625,"automated_coverage":0.375,"conditional_correctness":0.0,"conditional_unsafe":0.5,"unnecessary_review_rate":0.625,"accuracy_wilson_95":{"lower":0.0,"upper":0.1936076805344365,"confidence_level":0.95}}},"mask":{"n":2,"raw":{"n":2,"acceptable":0,"unsafe":2,"review":0,"automated_count":2,"automated_acceptable":0,"automated_unsafe":2,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":1.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":1.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.6576197724933469,"confidence_level":0.95}},"effective":{"n":2,"acceptable":1,"unsafe":0,"review":1,"automated_count":1,"automated_acceptable":1,"automated_unsafe":0,"unnecessary_reviews":1,"accuracy":0.5,"unsafe_rate":0.0,"review_rate":0.5,"automated_coverage":0.5,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.5,"accuracy_wilson_95":{"lower":0.09453120573423074,"upper":0.9054687942657693,"confidence_level":0.95}}},"pseudonymize":{"n":2,"raw":{"n":2,"acceptable":0,"unsafe":2,"review":0,"automated_count":2,"automated_acceptable":0,"automated_unsafe":2,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":1.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":1.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.6576197724933469,"confidence_level":0.95}},"effective":{"n":2,"acceptable":0,"unsafe":0,"review":2,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":2,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.6576197724933469,"confidence_level":0.95}}},"generalize":{"n":6,"raw":{"n":6,"acceptable":5,"unsafe":1,"review":0,"automated_count":6,"automated_acceptable":5,"automated_unsafe":1,"unnecessary_reviews":0,"accuracy":0.8333333333333334,"unsafe_rate":0.16666666666666666,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.8333333333333334,"conditional_unsafe":0.16666666666666666,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.43649717781352987,"upper":0.9699466302516935,"confidence_level":0.95}},"effective":{"n":6,"acceptable":5,"unsafe":0,"review":1,"automated_count":5,"automated_acceptable":5,"automated_unsafe":0,"unnecessary_reviews":1,"accuracy":0.8333333333333334,"unsafe_rate":0.0,"review_rate":0.16666666666666666,"automated_coverage":0.8333333333333334,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.16666666666666666,"accuracy_wilson_95":{"lower":0.43649717781352987,"upper":0.9699466302516935,"confidence_level":0.95}}},"review":{"n":0,"raw":{"n":0,"acceptable":0,"unsafe":0,"review":0,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":null,"unsafe_rate":null,"review_rate":null,"automated_coverage":null,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":null},"effective":{"n":0,"acceptable":0,"unsafe":0,"review":0,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":null,"unsafe_rate":null,"review_rate":null,"automated_coverage":null,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":null}}}},"challenge56-no_samples":{"n":56,"raw":{"n":56,"acceptable":25,"unsafe":24,"review":0,"automated_count":56,"automated_acceptable":25,"automated_unsafe":24,"unnecessary_reviews":0,"accuracy":0.44642857142857145,"unsafe_rate":0.42857142857142855,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.44642857142857145,"conditional_unsafe":0.42857142857142855,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.3238672125478337,"upper":0.5758678521006054,"confidence_level":0.95}},"effective":{"n":56,"acceptable":13,"unsafe":0,"review":45,"automated_count":11,"automated_acceptable":11,"automated_unsafe":0,"unnecessary_reviews":43,"accuracy":0.23214285714285715,"unsafe_rate":0.0,"review_rate":0.8035714285714286,"automated_coverage":0.19642857142857142,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.7678571428571429,"accuracy_wilson_95":{"lower":0.1409937910920862,"upper":0.35768153215010856,"confidence_level":0.95}},"confusion":{"raw":{"keep":{"keep":7,"drop":0,"mask":1,"pseudonymize":0,"generalize":0,"review":0},"drop":{"keep":10,"drop":8,"mask":0,"pseudonymize":0,"generalize":4,"review":0},"mask":{"keep":0,"drop":1,"mask":7,"pseudonymize":0,"generalize":0,"review":0},"pseudonymize":{"keep":7,"drop":0,"mask":0,"pseudonymize":0,"generalize":1,"review":0},"generalize":{"keep":5,"drop":0,"mask":0,"pseudonymize":0,"generalize":3,"review":0},"review":{"keep":2,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0}},"effective":{"keep":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":8},"drop":{"keep":0,"drop":5,"mask":0,"pseudonymize":0,"generalize":0,"review":17},"mask":{"keep":0,"drop":0,"mask":4,"pseudonymize":0,"generalize":0,"review":4},"pseudonymize":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":8},"generalize":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":2,"review":6},"review":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":2}}},"probability_metrics":{"singleton_n":56,"multiacceptable_n":0,"multiacceptable_coverage":0.0,"nll":1.4389267468028966,"nll_zero_mass_n":0,"brier":0.6760884011652422,"acceptable_set_n":56,"acceptable_set_nll":1.4389267468028966,"acceptable_set_zero_mass_n":0},"calibration_all_labels":{"equal_width":{"n":56,"bin_count":10,"ece":0.1520521454513073,"signed_mean_gap":-0.09761692477124077,"absolute_mean_gap":0.09761692477124077,"bins":[{"index":0,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.0,"upper":0.1,"upper_inclusive":false},{"index":1,"count":4,"confidence":0.19486549124121666,"empirical_correctness":0.0,"signed_gap":0.19486549124121666,"absolute_gap":0.19486549124121666,"lower":0.1,"upper":0.2,"upper_inclusive":false},{"index":2,"count":35,"confidence":0.2356975006205695,"empirical_correctness":0.34285714285714286,"signed_gap":-0.10715964223657337,"absolute_gap":0.10715964223657337,"lower":0.2,"upper":0.3,"upper_inclusive":false},{"index":3,"count":2,"confidence":0.3208279460668564,"empirical_correctness":0.5,"signed_gap":-0.17917205393314362,"absolute_gap":0.17917205393314362,"lower":0.3,"upper":0.4,"upper_inclusive":false},{"index":4,"count":4,"confidence":0.44512009620666504,"empirical_correctness":0.75,"signed_gap":-0.30487990379333496,"absolute_gap":0.30487990379333496,"lower":0.4,"upper":0.5,"upper_inclusive":false},{"index":5,"count":4,"confidence":0.5545012652873993,"empirical_correctness":0.75,"signed_gap":-0.1954987347126007,"absolute_gap":0.1954987347126007,"lower":0.5,"upper":0.6,"upper_inclusive":false},{"index":6,"count":1,"confidence":0.6035153865814209,"empirical_correctness":1.0,"signed_gap":-0.3964846134185791,"absolute_gap":0.3964846134185791,"lower":0.6,"upper":0.7,"upper_inclusive":false},{"index":7,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.7,"upper":0.8,"upper_inclusive":false},{"index":8,"count":3,"confidence":0.8387322624524435,"empirical_correctness":1.0,"signed_gap":-0.16126773754755652,"absolute_gap":0.16126773754755652,"lower":0.8,"upper":0.9,"upper_inclusive":false},{"index":9,"count":3,"confidence":0.9149080713589987,"empirical_correctness":0.6666666666666666,"signed_gap":0.24824140469233202,"absolute_gap":0.24824140469233202,"lower":0.9,"upper":1.0,"upper_inclusive":true}]},"equal_mass":{"n":56,"bin_count":10,"ece":0.18864660231130465,"signed_mean_gap":-0.09761692477124077,"absolute_mean_gap":0.09761692477124077,"bins":[{"index":0,"count":5,"confidence":0.19632319509983062,"empirical_correctness":0.0,"signed_gap":0.19632319509983062,"absolute_gap":0.19632319509983062,"min_confidence":0.19315896928310394,"max_confidence":0.2021540105342865},{"index":1,"count":6,"confidence":0.2066378245751063,"empirical_correctness":0.0,"signed_gap":0.2066378245751063,"absolute_gap":0.2066378245751063,"min_confidence":0.20330601930618286,"max_confidence":0.20832206308841705},{"index":2,"count":5,"confidence":0.21329340934753419,"empirical_correctness":0.2,"signed_gap":0.013293409347534174,"absolute_gap":0.013293409347534174,"min_confidence":0.20914532244205475,"max_confidence":0.21657317876815796},{"index":3,"count":6,"confidence":0.22281873722871146,"empirical_correctness":0.3333333333333333,"signed_gap":-0.11051459610462186,"absolute_gap":0.11051459610462186,"min_confidence":0.2174784392118454,"max_confidence":0.22854577004909515},{"index":4,"count":6,"confidence":0.23487713684638342,"empirical_correctness":0.5,"signed_gap":-0.26512286315361655,"absolute_gap":0.26512286315361655,"min_confidence":0.22909313440322876,"max_confidence":0.24397607147693634},{"index":5,"count":5,"confidence":0.26200639009475707,"empirical_correctness":0.6,"signed_gap":-0.3379936099052429,"absolute_gap":0.3379936099052429,"min_confidence":0.2578413486480713,"max_confidence":0.26852017641067505},{"index":6,"count":6,"confidence":0.2807928870121638,"empirical_correctness":0.5,"signed_gap":-0.21920711298783618,"absolute_gap":0.21920711298783618,"min_confidence":0.2701481282711029,"max_confidence":0.2940748333930969},{"index":7,"count":5,"confidence":0.3901501536369324,"empirical_correctness":0.6,"signed_gap":-0.20984984636306758,"absolute_gap":0.20984984636306758,"min_confidence":0.3099544644355774,"max_confidence":0.4554674029350281},{"index":8,"count":6,"confidence":0.5488176594177882,"empirical_correctness":0.8333333333333334,"signed_gap":-0.2845156739155452,"absolute_gap":0.2845156739155452,"min_confidence":0.47138550877571106,"max_confidence":0.6035153865814209},{"index":9,"count":6,"confidence":0.8768201669057211,"empirical_correctness":0.8333333333333334,"signed_gap":0.043486833572387695,"absolute_gap":0.043486833572387695,"min_confidence":0.8013089299201965,"max_confidence":0.9184117913246155}]}},"calibration_singleton_labels":{"equal_width":{"n":56,"bin_count":10,"ece":0.1520521454513073,"signed_mean_gap":-0.09761692477124077,"absolute_mean_gap":0.09761692477124077,"bins":[{"index":0,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.0,"upper":0.1,"upper_inclusive":false},{"index":1,"count":4,"confidence":0.19486549124121666,"empirical_correctness":0.0,"signed_gap":0.19486549124121666,"absolute_gap":0.19486549124121666,"lower":0.1,"upper":0.2,"upper_inclusive":false},{"index":2,"count":35,"confidence":0.2356975006205695,"empirical_correctness":0.34285714285714286,"signed_gap":-0.10715964223657337,"absolute_gap":0.10715964223657337,"lower":0.2,"upper":0.3,"upper_inclusive":false},{"index":3,"count":2,"confidence":0.3208279460668564,"empirical_correctness":0.5,"signed_gap":-0.17917205393314362,"absolute_gap":0.17917205393314362,"lower":0.3,"upper":0.4,"upper_inclusive":false},{"index":4,"count":4,"confidence":0.44512009620666504,"empirical_correctness":0.75,"signed_gap":-0.30487990379333496,"absolute_gap":0.30487990379333496,"lower":0.4,"upper":0.5,"upper_inclusive":false},{"index":5,"count":4,"confidence":0.5545012652873993,"empirical_correctness":0.75,"signed_gap":-0.1954987347126007,"absolute_gap":0.1954987347126007,"lower":0.5,"upper":0.6,"upper_inclusive":false},{"index":6,"count":1,"confidence":0.6035153865814209,"empirical_correctness":1.0,"signed_gap":-0.3964846134185791,"absolute_gap":0.3964846134185791,"lower":0.6,"upper":0.7,"upper_inclusive":false},{"index":7,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.7,"upper":0.8,"upper_inclusive":false},{"index":8,"count":3,"confidence":0.8387322624524435,"empirical_correctness":1.0,"signed_gap":-0.16126773754755652,"absolute_gap":0.16126773754755652,"lower":0.8,"upper":0.9,"upper_inclusive":false},{"index":9,"count":3,"confidence":0.9149080713589987,"empirical_correctness":0.6666666666666666,"signed_gap":0.24824140469233202,"absolute_gap":0.24824140469233202,"lower":0.9,"upper":1.0,"upper_inclusive":true}]},"equal_mass":{"n":56,"bin_count":10,"ece":0.18864660231130465,"signed_mean_gap":-0.09761692477124077,"absolute_mean_gap":0.09761692477124077,"bins":[{"index":0,"count":5,"confidence":0.19632319509983062,"empirical_correctness":0.0,"signed_gap":0.19632319509983062,"absolute_gap":0.19632319509983062,"min_confidence":0.19315896928310394,"max_confidence":0.2021540105342865},{"index":1,"count":6,"confidence":0.2066378245751063,"empirical_correctness":0.0,"signed_gap":0.2066378245751063,"absolute_gap":0.2066378245751063,"min_confidence":0.20330601930618286,"max_confidence":0.20832206308841705},{"index":2,"count":5,"confidence":0.21329340934753419,"empirical_correctness":0.2,"signed_gap":0.013293409347534174,"absolute_gap":0.013293409347534174,"min_confidence":0.20914532244205475,"max_confidence":0.21657317876815796},{"index":3,"count":6,"confidence":0.22281873722871146,"empirical_correctness":0.3333333333333333,"signed_gap":-0.11051459610462186,"absolute_gap":0.11051459610462186,"min_confidence":0.2174784392118454,"max_confidence":0.22854577004909515},{"index":4,"count":6,"confidence":0.23487713684638342,"empirical_correctness":0.5,"signed_gap":-0.26512286315361655,"absolute_gap":0.26512286315361655,"min_confidence":0.22909313440322876,"max_confidence":0.24397607147693634},{"index":5,"count":5,"confidence":0.26200639009475707,"empirical_correctness":0.6,"signed_gap":-0.3379936099052429,"absolute_gap":0.3379936099052429,"min_confidence":0.2578413486480713,"max_confidence":0.26852017641067505},{"index":6,"count":6,"confidence":0.2807928870121638,"empirical_correctness":0.5,"signed_gap":-0.21920711298783618,"absolute_gap":0.21920711298783618,"min_confidence":0.2701481282711029,"max_confidence":0.2940748333930969},{"index":7,"count":5,"confidence":0.3901501536369324,"empirical_correctness":0.6,"signed_gap":-0.20984984636306758,"absolute_gap":0.20984984636306758,"min_confidence":0.3099544644355774,"max_confidence":0.4554674029350281},{"index":8,"count":6,"confidence":0.5488176594177882,"empirical_correctness":0.8333333333333334,"signed_gap":-0.2845156739155452,"absolute_gap":0.2845156739155452,"min_confidence":0.47138550877571106,"max_confidence":0.6035153865814209},{"index":9,"count":6,"confidence":0.8768201669057211,"empirical_correctness":0.8333333333333334,"signed_gap":0.043486833572387695,"absolute_gap":0.043486833572387695,"min_confidence":0.8013089299201965,"max_confidence":0.9184117913246155}]}},"latency_ms":{"n":56,"median":15.484255040064454,"p95":15.827403985895216},"per_action":{"keep":{"n":8,"raw":{"n":8,"acceptable":7,"unsafe":0,"review":0,"automated_count":8,"automated_acceptable":7,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":0.875,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.875,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.5291118177871464,"upper":0.9775825085499433,"confidence_level":0.95}},"effective":{"n":8,"acceptable":0,"unsafe":0,"review":8,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":8,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.32440756488388023,"confidence_level":0.95}}},"drop":{"n":22,"raw":{"n":22,"acceptable":8,"unsafe":10,"review":0,"automated_count":22,"automated_acceptable":8,"automated_unsafe":10,"unnecessary_reviews":0,"accuracy":0.36363636363636365,"unsafe_rate":0.45454545454545453,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.36363636363636365,"conditional_unsafe":0.45454545454545453,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.19733204670366125,"upper":0.5704829190986899,"confidence_level":0.95}},"effective":{"n":22,"acceptable":5,"unsafe":0,"review":17,"automated_count":5,"automated_acceptable":5,"automated_unsafe":0,"unnecessary_reviews":17,"accuracy":0.22727272727272727,"unsafe_rate":0.0,"review_rate":0.7727272727272727,"automated_coverage":0.22727272727272727,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.7727272727272727,"accuracy_wilson_95":{"lower":0.10123039985248516,"upper":0.4343995317522171,"confidence_level":0.95}}},"mask":{"n":8,"raw":{"n":8,"acceptable":7,"unsafe":0,"review":0,"automated_count":8,"automated_acceptable":7,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":0.875,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.875,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.5291118177871464,"upper":0.9775825085499433,"confidence_level":0.95}},"effective":{"n":8,"acceptable":4,"unsafe":0,"review":4,"automated_count":4,"automated_acceptable":4,"automated_unsafe":0,"unnecessary_reviews":4,"accuracy":0.5,"unsafe_rate":0.0,"review_rate":0.5,"automated_coverage":0.5,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.5,"accuracy_wilson_95":{"lower":0.21521606221387757,"upper":0.7847839377861224,"confidence_level":0.95}}},"pseudonymize":{"n":8,"raw":{"n":8,"acceptable":0,"unsafe":7,"review":0,"automated_count":8,"automated_acceptable":0,"automated_unsafe":7,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":0.875,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":0.875,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.32440756488388023,"confidence_level":0.95}},"effective":{"n":8,"acceptable":0,"unsafe":0,"review":8,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":8,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.32440756488388023,"confidence_level":0.95}}},"generalize":{"n":8,"raw":{"n":8,"acceptable":3,"unsafe":5,"review":0,"automated_count":8,"automated_acceptable":3,"automated_unsafe":5,"unnecessary_reviews":0,"accuracy":0.375,"unsafe_rate":0.625,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.375,"conditional_unsafe":0.625,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.13684428582359742,"upper":0.6942576053973727,"confidence_level":0.95}},"effective":{"n":8,"acceptable":2,"unsafe":0,"review":6,"automated_count":2,"automated_acceptable":2,"automated_unsafe":0,"unnecessary_reviews":6,"accuracy":0.25,"unsafe_rate":0.0,"review_rate":0.75,"automated_coverage":0.25,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.75,"accuracy_wilson_95":{"lower":0.071479212752109,"upper":0.5907245696898311,"confidence_level":0.95}}},"review":{"n":2,"raw":{"n":2,"acceptable":0,"unsafe":2,"review":0,"automated_count":2,"automated_acceptable":0,"automated_unsafe":2,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":1.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":1.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.6576197724933469,"confidence_level":0.95}},"effective":{"n":2,"acceptable":2,"unsafe":0,"review":2,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":1.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.34238022750665303,"upper":1.0,"confidence_level":0.95}}}}},"challenge56-compact_no_samples":{"n":56,"raw":{"n":56,"acceptable":14,"unsafe":42,"review":0,"automated_count":56,"automated_acceptable":14,"automated_unsafe":42,"unnecessary_reviews":0,"accuracy":0.25,"unsafe_rate":0.75,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.25,"conditional_unsafe":0.75,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.15517054688270684,"upper":0.376926421476675,"confidence_level":0.95}},"effective":{"n":56,"acceptable":13,"unsafe":0,"review":45,"automated_count":11,"automated_acceptable":11,"automated_unsafe":0,"unnecessary_reviews":43,"accuracy":0.23214285714285715,"unsafe_rate":0.0,"review_rate":0.8035714285714286,"automated_coverage":0.19642857142857142,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.7678571428571429,"accuracy_wilson_95":{"lower":0.1409937910920862,"upper":0.35768153215010856,"confidence_level":0.95}},"confusion":{"raw":{"keep":{"keep":8,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0},"drop":{"keep":22,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0},"mask":{"keep":8,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0},"pseudonymize":{"keep":8,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0},"generalize":{"keep":2,"drop":0,"mask":0,"pseudonymize":0,"generalize":6,"review":0},"review":{"keep":2,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0}},"effective":{"keep":{"keep":5,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":3},"drop":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":22},"mask":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":8},"pseudonymize":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":8},"generalize":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":6,"review":2},"review":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":2}}},"probability_metrics":{"singleton_n":56,"multiacceptable_n":0,"multiacceptable_coverage":0.0,"nll":2.794136741760311,"nll_zero_mass_n":0,"brier":1.246593049487006,"acceptable_set_n":56,"acceptable_set_nll":2.794136741760311,"acceptable_set_zero_mass_n":0},"calibration_all_labels":{"equal_width":{"n":56,"bin_count":10,"ece":0.6065519909773555,"signed_mean_gap":0.6065519909773555,"absolute_mean_gap":0.6065519909773555,"bins":[{"index":0,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.0,"upper":0.1,"upper_inclusive":false},{"index":1,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.1,"upper":0.2,"upper_inclusive":false},{"index":2,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.2,"upper":0.3,"upper_inclusive":false},{"index":3,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.3,"upper":0.4,"upper_inclusive":false},{"index":4,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.4,"upper":0.5,"upper_inclusive":false},{"index":5,"count":8,"confidence":0.5311411172151566,"empirical_correctness":0.25,"signed_gap":0.28114111721515656,"absolute_gap":0.28114111721515656,"lower":0.5,"upper":0.6,"upper_inclusive":false},{"index":6,"count":2,"confidence":0.6721029281616211,"empirical_correctness":0.0,"signed_gap":0.6721029281616211,"absolute_gap":0.6721029281616211,"lower":0.6,"upper":0.7,"upper_inclusive":false},{"index":7,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.7,"upper":0.8,"upper_inclusive":false},{"index":8,"count":5,"confidence":0.8651413679122925,"empirical_correctness":0.4,"signed_gap":0.46514136791229244,"absolute_gap":0.46514136791229244,"lower":0.8,"upper":0.9,"upper_inclusive":false},{"index":9,"count":41,"confidence":0.9279968258811206,"empirical_correctness":0.24390243902439024,"signed_gap":0.6840943868567304,"absolute_gap":0.6840943868567304,"lower":0.9,"upper":1.0,"upper_inclusive":true}]},"equal_mass":{"n":56,"bin_count":10,"ece":0.6065519909773555,"signed_mean_gap":0.6065519909773555,"absolute_mean_gap":0.6065519909773555,"bins":[{"index":0,"count":5,"confidence":0.5110403537750244,"empirical_correctness":0.2,"signed_gap":0.3110403537750244,"absolute_gap":0.3110403537750244,"min_confidence":0.5031729936599731,"max_confidence":0.5237137675285339},{"index":1,"count":6,"confidence":0.6424906154473623,"empirical_correctness":0.16666666666666666,"signed_gap":0.47582394878069567,"absolute_gap":0.47582394878069567,"min_confidence":0.5333175659179688,"max_confidence":0.816810667514801},{"index":2,"count":5,"confidence":0.8821810007095336,"empirical_correctness":0.6,"signed_gap":0.28218100070953367,"absolute_gap":0.28218100070953367,"min_confidence":0.8660868406295776,"max_confidence":0.9020088315010071},{"index":3,"count":6,"confidence":0.9139071802298228,"empirical_correctness":0.0,"signed_gap":0.9139071802298228,"absolute_gap":0.9139071802298228,"min_confidence":0.9062347412109375,"max_confidence":0.9201921820640564},{"index":4,"count":6,"confidence":0.9232203861077627,"empirical_correctness":0.16666666666666666,"signed_gap":0.7565537194410961,"absolute_gap":0.7565537194410961,"min_confidence":0.9212928414344788,"max_confidence":0.9248540997505188},{"index":5,"count":5,"confidence":0.9267906308174133,"empirical_correctness":0.2,"signed_gap":0.7267906308174132,"absolute_gap":0.7267906308174132,"min_confidence":0.9249811768531799,"max_confidence":0.9290846586227417},{"index":6,"count":6,"confidence":0.9310581882794698,"empirical_correctness":0.0,"signed_gap":0.9310581882794698,"absolute_gap":0.9310581882794698,"min_confidence":0.9294241666793823,"max_confidence":0.9323678612709045},{"index":7,"count":5,"confidence":0.9329224348068237,"empirical_correctness":0.4,"signed_gap":0.5329224348068237,"absolute_gap":0.5329224348068237,"min_confidence":0.9325147271156311,"max_confidence":0.9333078265190125},{"index":8,"count":6,"confidence":0.9347822070121765,"empirical_correctness":0.16666666666666666,"signed_gap":0.7681155403455099,"absolute_gap":0.7681155403455099,"min_confidence":0.933673083782196,"max_confidence":0.9361917972564697},{"index":9,"count":6,"confidence":0.9382479886213938,"empirical_correctness":0.6666666666666666,"signed_gap":0.2715813219547272,"absolute_gap":0.2715813219547272,"min_confidence":0.9368823766708374,"max_confidence":0.9427427649497986}]}},"calibration_singleton_labels":{"equal_width":{"n":56,"bin_count":10,"ece":0.6065519909773555,"signed_mean_gap":0.6065519909773555,"absolute_mean_gap":0.6065519909773555,"bins":[{"index":0,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.0,"upper":0.1,"upper_inclusive":false},{"index":1,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.1,"upper":0.2,"upper_inclusive":false},{"index":2,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.2,"upper":0.3,"upper_inclusive":false},{"index":3,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.3,"upper":0.4,"upper_inclusive":false},{"index":4,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.4,"upper":0.5,"upper_inclusive":false},{"index":5,"count":8,"confidence":0.5311411172151566,"empirical_correctness":0.25,"signed_gap":0.28114111721515656,"absolute_gap":0.28114111721515656,"lower":0.5,"upper":0.6,"upper_inclusive":false},{"index":6,"count":2,"confidence":0.6721029281616211,"empirical_correctness":0.0,"signed_gap":0.6721029281616211,"absolute_gap":0.6721029281616211,"lower":0.6,"upper":0.7,"upper_inclusive":false},{"index":7,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.7,"upper":0.8,"upper_inclusive":false},{"index":8,"count":5,"confidence":0.8651413679122925,"empirical_correctness":0.4,"signed_gap":0.46514136791229244,"absolute_gap":0.46514136791229244,"lower":0.8,"upper":0.9,"upper_inclusive":false},{"index":9,"count":41,"confidence":0.9279968258811206,"empirical_correctness":0.24390243902439024,"signed_gap":0.6840943868567304,"absolute_gap":0.6840943868567304,"lower":0.9,"upper":1.0,"upper_inclusive":true}]},"equal_mass":{"n":56,"bin_count":10,"ece":0.6065519909773555,"signed_mean_gap":0.6065519909773555,"absolute_mean_gap":0.6065519909773555,"bins":[{"index":0,"count":5,"confidence":0.5110403537750244,"empirical_correctness":0.2,"signed_gap":0.3110403537750244,"absolute_gap":0.3110403537750244,"min_confidence":0.5031729936599731,"max_confidence":0.5237137675285339},{"index":1,"count":6,"confidence":0.6424906154473623,"empirical_correctness":0.16666666666666666,"signed_gap":0.47582394878069567,"absolute_gap":0.47582394878069567,"min_confidence":0.5333175659179688,"max_confidence":0.816810667514801},{"index":2,"count":5,"confidence":0.8821810007095336,"empirical_correctness":0.6,"signed_gap":0.28218100070953367,"absolute_gap":0.28218100070953367,"min_confidence":0.8660868406295776,"max_confidence":0.9020088315010071},{"index":3,"count":6,"confidence":0.9139071802298228,"empirical_correctness":0.0,"signed_gap":0.9139071802298228,"absolute_gap":0.9139071802298228,"min_confidence":0.9062347412109375,"max_confidence":0.9201921820640564},{"index":4,"count":6,"confidence":0.9232203861077627,"empirical_correctness":0.16666666666666666,"signed_gap":0.7565537194410961,"absolute_gap":0.7565537194410961,"min_confidence":0.9212928414344788,"max_confidence":0.9248540997505188},{"index":5,"count":5,"confidence":0.9267906308174133,"empirical_correctness":0.2,"signed_gap":0.7267906308174132,"absolute_gap":0.7267906308174132,"min_confidence":0.9249811768531799,"max_confidence":0.9290846586227417},{"index":6,"count":6,"confidence":0.9310581882794698,"empirical_correctness":0.0,"signed_gap":0.9310581882794698,"absolute_gap":0.9310581882794698,"min_confidence":0.9294241666793823,"max_confidence":0.9323678612709045},{"index":7,"count":5,"confidence":0.9329224348068237,"empirical_correctness":0.4,"signed_gap":0.5329224348068237,"absolute_gap":0.5329224348068237,"min_confidence":0.9325147271156311,"max_confidence":0.9333078265190125},{"index":8,"count":6,"confidence":0.9347822070121765,"empirical_correctness":0.16666666666666666,"signed_gap":0.7681155403455099,"absolute_gap":0.7681155403455099,"min_confidence":0.933673083782196,"max_confidence":0.9361917972564697},{"index":9,"count":6,"confidence":0.9382479886213938,"empirical_correctness":0.6666666666666666,"signed_gap":0.2715813219547272,"absolute_gap":0.2715813219547272,"min_confidence":0.9368823766708374,"max_confidence":0.9427427649497986}]}},"latency_ms":{"n":56,"median":18.046660989057273,"p95":18.39231747726444},"per_action":{"keep":{"n":8,"raw":{"n":8,"acceptable":8,"unsafe":0,"review":0,"automated_count":8,"automated_acceptable":8,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":1.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.6755924351161198,"upper":1.0,"confidence_level":0.95}},"effective":{"n":8,"acceptable":5,"unsafe":0,"review":3,"automated_count":5,"automated_acceptable":5,"automated_unsafe":0,"unnecessary_reviews":3,"accuracy":0.625,"unsafe_rate":0.0,"review_rate":0.375,"automated_coverage":0.625,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.375,"accuracy_wilson_95":{"lower":0.30574239460262737,"upper":0.8631557141764026,"confidence_level":0.95}}},"drop":{"n":22,"raw":{"n":22,"acceptable":0,"unsafe":22,"review":0,"automated_count":22,"automated_acceptable":0,"automated_unsafe":22,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":1.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":1.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.14865487460862087,"confidence_level":0.95}},"effective":{"n":22,"acceptable":0,"unsafe":0,"review":22,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":22,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.14865487460862087,"confidence_level":0.95}}},"mask":{"n":8,"raw":{"n":8,"acceptable":0,"unsafe":8,"review":0,"automated_count":8,"automated_acceptable":0,"automated_unsafe":8,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":1.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":1.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.32440756488388023,"confidence_level":0.95}},"effective":{"n":8,"acceptable":0,"unsafe":0,"review":8,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":8,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.32440756488388023,"confidence_level":0.95}}},"pseudonymize":{"n":8,"raw":{"n":8,"acceptable":0,"unsafe":8,"review":0,"automated_count":8,"automated_acceptable":0,"automated_unsafe":8,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":1.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":1.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.32440756488388023,"confidence_level":0.95}},"effective":{"n":8,"acceptable":0,"unsafe":0,"review":8,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":8,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.32440756488388023,"confidence_level":0.95}}},"generalize":{"n":8,"raw":{"n":8,"acceptable":6,"unsafe":2,"review":0,"automated_count":8,"automated_acceptable":6,"automated_unsafe":2,"unnecessary_reviews":0,"accuracy":0.75,"unsafe_rate":0.25,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.75,"conditional_unsafe":0.25,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.40927543031016883,"upper":0.9285207872478909,"confidence_level":0.95}},"effective":{"n":8,"acceptable":6,"unsafe":0,"review":2,"automated_count":6,"automated_acceptable":6,"automated_unsafe":0,"unnecessary_reviews":2,"accuracy":0.75,"unsafe_rate":0.0,"review_rate":0.25,"automated_coverage":0.75,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.25,"accuracy_wilson_95":{"lower":0.40927543031016883,"upper":0.9285207872478909,"confidence_level":0.95}}},"review":{"n":2,"raw":{"n":2,"acceptable":0,"unsafe":2,"review":0,"automated_count":2,"automated_acceptable":0,"automated_unsafe":2,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":1.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":1.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.6576197724933469,"confidence_level":0.95}},"effective":{"n":2,"acceptable":2,"unsafe":0,"review":2,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":1.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.34238022750665303,"upper":1.0,"confidence_level":0.95}}}}}},"composite":{"n":198,"scope":"descriptive sum across four correlated panels","raw":{"n":198,"acceptable":68,"unsafe":108,"review":0,"automated_count":198,"automated_acceptable":68,"automated_unsafe":108,"unnecessary_reviews":0,"accuracy":0.3434343434343434,"unsafe_rate":0.5454545454545454,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.3434343434343434,"conditional_unsafe":0.5454545454545454,"unnecessary_review_rate":0.0},"effective":{"n":198,"acceptable":63,"unsafe":6,"review":124,"automated_count":74,"automated_acceptable":52,"automated_unsafe":6,"unnecessary_reviews":113,"accuracy":0.3181818181818182,"unsafe_rate":0.030303030303030304,"review_rate":0.6262626262626263,"automated_coverage":0.37373737373737376,"conditional_correctness":0.7027027027027027,"conditional_unsafe":0.08108108108108109,"unnecessary_review_rate":0.5707070707070707}}},{"id":"3p3r--sentinel-laya-multilingual-fp32","platform":"Local","model_id":"3p3r/sentinel-laya-multilingual","revision":"a7539113482f2631fd733d3f20a5b027ce1b617e","loading":"float32 / none","scoring_kind":"Native decision readout","settings":{"dtype":"float32","quantization":"none","max_tokens":512,"head_max_tokens":256,"temperature":1.2,"autocast":false,"truncation":false,"cpu_offload":false,"guard_threshold":0.45,"seed":0,"tf32":false},"scoring_metadata":{"backend":"laya_native_local_builtin","readout":"native_mask_slot_scorer_numpy_float32_softmax","probability_source":"native_choice_unrounded_published_temperature_not_task_calibrated","latency_scope":"synchronized serial complete encoder+head+act forward,input transfer and native readout; tokenization excluded","native_disclosures":["Stock installed torch attention in place of ONNX dynamic equivalent; no custom kernels or repo imports.","FP32/no-autocast explicit precision: convert every saved F16/BF16/F32 tensor toFP32; no quantization.","Unrounded native probability; guard uses winner p, not entropy-concentration confidence.","Act head fully loaded and evaluated; not used to alter six-action answer.","Native choice:6-10 bucket else choice scalar0, T=1.2. No task fitting, priors correction, prompt change or finetuning here."],"equal_mass_convention":"rank bins with floor boundaries at floor(i*n/k), k=min(10,n//2); minimum two rows"},"load_ms":null,"peak_allocated_bytes":1363194368,"peak_reserved_bytes":1396703232,"panels":{"diagnostic43-no_samples":{"n":43,"raw":{"n":43,"acceptable":21,"unsafe":1,"review":0,"automated_count":43,"automated_acceptable":21,"automated_unsafe":1,"unnecessary_reviews":0,"accuracy":0.4883720930232558,"unsafe_rate":0.023255813953488372,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.4883720930232558,"conditional_unsafe":0.023255813953488372,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.3461744754671859,"upper":0.632476915305507,"confidence_level":0.95}},"effective":{"n":43,"acceptable":9,"unsafe":0,"review":34,"automated_count":9,"automated_acceptable":4,"automated_unsafe":0,"unnecessary_reviews":29,"accuracy":0.20930232558139536,"unsafe_rate":0.0,"review_rate":0.7906976744186046,"automated_coverage":0.20930232558139536,"conditional_correctness":0.4444444444444444,"conditional_unsafe":0.0,"unnecessary_review_rate":0.6744186046511628,"accuracy_wilson_95":{"lower":0.11422834220995602,"upper":0.3520564271073644,"confidence_level":0.95}},"confusion":{"raw":{"keep":{"keep":0,"drop":11,"mask":1,"pseudonymize":0,"generalize":0,"review":0},"drop":{"keep":0,"drop":16,"mask":0,"pseudonymize":0,"generalize":0,"review":0},"mask":{"keep":0,"drop":1,"mask":1,"pseudonymize":0,"generalize":0,"review":0},"pseudonymize":{"keep":0,"drop":2,"mask":0,"pseudonymize":0,"generalize":0,"review":0},"generalize":{"keep":0,"drop":6,"mask":0,"pseudonymize":0,"generalize":0,"review":0},"review":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0}},"effective":{"keep":{"keep":0,"drop":3,"mask":0,"pseudonymize":0,"generalize":0,"review":9},"drop":{"keep":0,"drop":4,"mask":0,"pseudonymize":0,"generalize":0,"review":12},"mask":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":2},"pseudonymize":{"keep":0,"drop":1,"mask":0,"pseudonymize":0,"generalize":0,"review":1},"generalize":{"keep":0,"drop":1,"mask":0,"pseudonymize":0,"generalize":0,"review":5},"review":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0}}},"probability_metrics":{"singleton_n":38,"multiacceptable_n":5,"multiacceptable_coverage":0.11627906976744186,"nll":1.5809935426670512,"nll_zero_mass_n":0,"brier":0.7579235589937,"acceptable_set_n":43,"acceptable_set_nll":1.4757823420798146,"acceptable_set_zero_mass_n":0},"calibration_all_labels":{"equal_width":{"n":43,"bin_count":10,"ece":0.14768052343712296,"signed_mean_gap":-0.11101817218370212,"absolute_mean_gap":0.11101817218370212,"bins":[{"index":0,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.0,"upper":0.1,"upper_inclusive":false},{"index":1,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.1,"upper":0.2,"upper_inclusive":false},{"index":2,"count":6,"confidence":0.2687547877430916,"empirical_correctness":0.5,"signed_gap":-0.23124521225690842,"absolute_gap":0.23124521225690842,"lower":0.2,"upper":0.3,"upper_inclusive":false},{"index":3,"count":27,"confidence":0.34964686080261514,"empirical_correctness":0.48148148148148145,"signed_gap":-0.1318346206788663,"absolute_gap":0.1318346206788663,"lower":0.3,"upper":0.4,"upper_inclusive":false},{"index":4,"count":3,"confidence":0.46166135867436725,"empirical_correctness":0.6666666666666666,"signed_gap":-0.20500530799229938,"absolute_gap":0.20500530799229938,"lower":0.4,"upper":0.5,"upper_inclusive":false},{"index":5,"count":7,"confidence":0.5411772217069354,"empirical_correctness":0.42857142857142855,"signed_gap":0.11260579313550684,"absolute_gap":0.11260579313550684,"lower":0.5,"upper":0.6,"upper_inclusive":false},{"index":6,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.6,"upper":0.7,"upper_inclusive":false},{"index":7,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.7,"upper":0.8,"upper_inclusive":false},{"index":8,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.8,"upper":0.9,"upper_inclusive":false},{"index":9,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.9,"upper":1.0,"upper_inclusive":true}]},"equal_mass":{"n":43,"bin_count":10,"ece":0.21328479432782463,"signed_mean_gap":-0.11101817218370216,"absolute_mean_gap":0.11101817218370216,"bins":[{"index":0,"count":4,"confidence":0.25889894738793373,"empirical_correctness":0.5,"signed_gap":-0.24110105261206627,"absolute_gap":0.24110105261206627,"min_confidence":0.24707935750484467,"max_confidence":0.2695411741733551},{"index":1,"count":4,"confidence":0.2969917804002762,"empirical_correctness":0.5,"signed_gap":-0.20300821959972382,"absolute_gap":0.20300821959972382,"min_confidence":0.28468000888824463,"max_confidence":0.30794551968574524},{"index":2,"count":4,"confidence":0.3175762668251991,"empirical_correctness":0.5,"signed_gap":-0.18242373317480087,"absolute_gap":0.18242373317480087,"min_confidence":0.3101488947868347,"max_confidence":0.3221662640571594},{"index":3,"count":5,"confidence":0.3345526993274689,"empirical_correctness":0.6,"signed_gap":-0.2654473006725311,"absolute_gap":0.2654473006725311,"min_confidence":0.3245738446712494,"max_confidence":0.3469155430793762},{"index":4,"count":4,"confidence":0.3516329452395439,"empirical_correctness":0.5,"signed_gap":-0.14836705476045609,"absolute_gap":0.14836705476045609,"min_confidence":0.3477258086204529,"max_confidence":0.3538963496685028},{"index":5,"count":4,"confidence":0.35887283086776733,"empirical_correctness":0.0,"signed_gap":0.35887283086776733,"absolute_gap":0.35887283086776733,"min_confidence":0.3550763428211212,"max_confidence":0.3645850121974945},{"index":6,"count":5,"confidence":0.37405636310577395,"empirical_correctness":0.4,"signed_gap":-0.025943636894226074,"absolute_gap":0.025943636894226074,"min_confidence":0.3696991801261902,"max_confidence":0.3838791251182556},{"index":7,"count":4,"confidence":0.39951012283563614,"empirical_correctness":1.0,"signed_gap":-0.6004898771643639,"absolute_gap":0.6004898771643639,"min_confidence":0.38625866174697876,"max_confidence":0.4239829182624817},{"index":8,"count":4,"confidence":0.496500164270401,"empirical_correctness":0.5,"signed_gap":-0.003499835729598999,"absolute_gap":0.003499835729598999,"min_confidence":0.47025367617607117,"max_confidence":0.5213280916213989},{"index":9,"count":5,"confidence":0.5526482105255127,"empirical_correctness":0.4,"signed_gap":0.1526482105255127,"absolute_gap":0.1526482105255127,"min_confidence":0.522541344165802,"max_confidence":0.5986150503158569}]}},"calibration_singleton_labels":{"equal_width":{"n":38,"bin_count":10,"ece":0.10856102446192191,"signed_mean_gap":-0.06707467962252467,"absolute_mean_gap":0.06707467962252467,"bins":[{"index":0,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.0,"upper":0.1,"upper_inclusive":false},{"index":1,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.1,"upper":0.2,"upper_inclusive":false},{"index":2,"count":5,"confidence":0.265569743514061,"empirical_correctness":0.4,"signed_gap":-0.13443025648593904,"absolute_gap":0.13443025648593904,"lower":0.2,"upper":0.3,"upper_inclusive":false},{"index":3,"count":23,"confidence":0.34565603603487427,"empirical_correctness":0.43478260869565216,"signed_gap":-0.08912657266077789,"absolute_gap":0.08912657266077789,"lower":0.3,"upper":0.4,"upper_inclusive":false},{"index":4,"count":3,"confidence":0.46166135867436725,"empirical_correctness":0.6666666666666666,"signed_gap":-0.20500530799229938,"absolute_gap":0.20500530799229938,"lower":0.4,"upper":0.5,"upper_inclusive":false},{"index":5,"count":7,"confidence":0.5411772217069354,"empirical_correctness":0.42857142857142855,"signed_gap":0.11260579313550684,"absolute_gap":0.11260579313550684,"lower":0.5,"upper":0.6,"upper_inclusive":false},{"index":6,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.6,"upper":0.7,"upper_inclusive":false},{"index":7,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.7,"upper":0.8,"upper_inclusive":false},{"index":8,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.8,"upper":0.9,"upper_inclusive":false},{"index":9,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.9,"upper":1.0,"upper_inclusive":true}]},"equal_mass":{"n":38,"bin_count":10,"ece":0.21591292831458544,"signed_mean_gap":-0.06707467962252467,"absolute_mean_gap":0.06707467962252467,"bins":[{"index":0,"count":3,"confidence":0.25535153845945996,"empirical_correctness":0.3333333333333333,"signed_gap":-0.07798179487387336,"absolute_gap":0.07798179487387336,"min_confidence":0.24707935750484467,"max_confidence":0.26445621252059937},{"index":1,"count":4,"confidence":0.2932070717215538,"empirical_correctness":0.5,"signed_gap":-0.2067929282784462,"absolute_gap":0.2067929282784462,"min_confidence":0.2695411741733551,"max_confidence":0.30794551968574524},{"index":2,"count":4,"confidence":0.3175762668251991,"empirical_correctness":0.5,"signed_gap":-0.18242373317480087,"absolute_gap":0.18242373317480087,"min_confidence":0.3101488947868347,"max_confidence":0.3221662640571594},{"index":3,"count":4,"confidence":0.3357853293418884,"empirical_correctness":0.75,"signed_gap":-0.4142146706581116,"absolute_gap":0.4142146706581116,"min_confidence":0.3245738446712494,"max_confidence":0.3469155430793762},{"index":4,"count":4,"confidence":0.3516329452395439,"empirical_correctness":0.5,"signed_gap":-0.14836705476045609,"absolute_gap":0.14836705476045609,"min_confidence":0.3477258086204529,"max_confidence":0.3538963496685028},{"index":5,"count":3,"confidence":0.35696877042452496,"empirical_correctness":0.0,"signed_gap":0.35696877042452496,"absolute_gap":0.35696877042452496,"min_confidence":0.3550763428211212,"max_confidence":0.35899072885513306},{"index":6,"count":4,"confidence":0.3695080950856209,"empirical_correctness":0.0,"signed_gap":0.3695080950856209,"absolute_gap":0.3695080950856209,"min_confidence":0.3645850121974945,"max_confidence":0.3719959855079651},{"index":7,"count":4,"confidence":0.4160935953259468,"empirical_correctness":0.75,"signed_gap":-0.3339064046740532,"absolute_gap":0.3339064046740532,"min_confidence":0.3838791251182556,"max_confidence":0.47025367617607117},{"index":8,"count":4,"confidence":0.5095720812678337,"empirical_correctness":0.5,"signed_gap":0.00957208126783371,"absolute_gap":0.00957208126783371,"min_confidence":0.49074748158454895,"max_confidence":0.522541344165802},{"index":9,"count":4,"confidence":0.5601749271154404,"empirical_correctness":0.5,"signed_gap":0.06017492711544037,"absolute_gap":0.06017492711544037,"min_confidence":0.5246698260307312,"max_confidence":0.5986150503158569}]}},"latency_ms":{"n":43,"median":29.73849978297949,"p95":31.36885992716998},"per_action":{"keep":{"n":12,"raw":{"n":12,"acceptable":0,"unsafe":0,"review":0,"automated_count":12,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":2.7755575615628914e-17,"upper":0.24249400665524085,"confidence_level":0.95}},"effective":{"n":12,"acceptable":0,"unsafe":0,"review":9,"automated_count":3,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":9,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":0.75,"automated_coverage":0.25,"conditional_correctness":0.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.75,"accuracy_wilson_95":{"lower":2.7755575615628914e-17,"upper":0.24249400665524085,"confidence_level":0.95}}},"drop":{"n":16,"raw":{"n":16,"acceptable":16,"unsafe":0,"review":0,"automated_count":16,"automated_acceptable":16,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":1.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.8063923194655637,"upper":1.0,"confidence_level":0.95}},"effective":{"n":16,"acceptable":4,"unsafe":0,"review":12,"automated_count":4,"automated_acceptable":4,"automated_unsafe":0,"unnecessary_reviews":12,"accuracy":0.25,"unsafe_rate":0.0,"review_rate":0.75,"automated_coverage":0.25,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.75,"accuracy_wilson_95":{"lower":0.10182067491213048,"upper":0.49498316535508774,"confidence_level":0.95}}},"mask":{"n":2,"raw":{"n":2,"acceptable":1,"unsafe":0,"review":0,"automated_count":2,"automated_acceptable":1,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":0.5,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.5,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.09453120573423074,"upper":0.9054687942657693,"confidence_level":0.95}},"effective":{"n":2,"acceptable":0,"unsafe":0,"review":2,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":2,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.6576197724933469,"confidence_level":0.95}}},"pseudonymize":{"n":2,"raw":{"n":2,"acceptable":0,"unsafe":0,"review":0,"automated_count":2,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.6576197724933469,"confidence_level":0.95}},"effective":{"n":2,"acceptable":0,"unsafe":0,"review":1,"automated_count":1,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":1,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":0.5,"automated_coverage":0.5,"conditional_correctness":0.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.5,"accuracy_wilson_95":{"lower":0.0,"upper":0.6576197724933469,"confidence_level":0.95}}},"generalize":{"n":6,"raw":{"n":6,"acceptable":0,"unsafe":0,"review":0,"automated_count":6,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":2.7755575615628914e-17,"upper":0.3903342879021653,"confidence_level":0.95}},"effective":{"n":6,"acceptable":0,"unsafe":0,"review":5,"automated_count":1,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":5,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":0.8333333333333334,"automated_coverage":0.16666666666666666,"conditional_correctness":0.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.8333333333333334,"accuracy_wilson_95":{"lower":2.7755575615628914e-17,"upper":0.3903342879021653,"confidence_level":0.95}}},"review":{"n":0,"raw":{"n":0,"acceptable":0,"unsafe":0,"review":0,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":null,"unsafe_rate":null,"review_rate":null,"automated_coverage":null,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":null},"effective":{"n":0,"acceptable":0,"unsafe":0,"review":0,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":null,"unsafe_rate":null,"review_rate":null,"automated_coverage":null,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":null}}}},"diagnostic43-compact_no_samples":{"n":43,"raw":{"n":43,"acceptable":22,"unsafe":7,"review":0,"automated_count":43,"automated_acceptable":22,"automated_unsafe":7,"unnecessary_reviews":0,"accuracy":0.5116279069767442,"unsafe_rate":0.16279069767441862,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.5116279069767442,"conditional_unsafe":0.16279069767441862,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.3675230846944931,"upper":0.6538255245328142,"confidence_level":0.95}},"effective":{"n":43,"acceptable":6,"unsafe":0,"review":40,"automated_count":3,"automated_acceptable":1,"automated_unsafe":0,"unnecessary_reviews":35,"accuracy":0.13953488372093023,"unsafe_rate":0.0,"review_rate":0.9302325581395349,"automated_coverage":0.06976744186046512,"conditional_correctness":0.3333333333333333,"conditional_unsafe":0.0,"unnecessary_review_rate":0.813953488372093,"accuracy_wilson_95":{"lower":0.06555723427964878,"upper":0.2726358796738286,"confidence_level":0.95}},"confusion":{"raw":{"keep":{"keep":6,"drop":6,"mask":0,"pseudonymize":0,"generalize":0,"review":0},"drop":{"keep":3,"drop":11,"mask":0,"pseudonymize":1,"generalize":1,"review":0},"mask":{"keep":2,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0},"pseudonymize":{"keep":0,"drop":2,"mask":0,"pseudonymize":0,"generalize":0,"review":0},"generalize":{"keep":0,"drop":5,"mask":0,"pseudonymize":0,"generalize":1,"review":0},"review":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0}},"effective":{"keep":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":12},"drop":{"keep":0,"drop":0,"mask":2,"pseudonymize":0,"generalize":0,"review":14},"mask":{"keep":0,"drop":0,"mask":1,"pseudonymize":0,"generalize":0,"review":1},"pseudonymize":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":2},"generalize":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":6},"review":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0}}},"probability_metrics":{"singleton_n":38,"multiacceptable_n":5,"multiacceptable_coverage":0.11627906976744186,"nll":1.4676985517260956,"nll_zero_mass_n":0,"brier":0.72341771473757,"acceptable_set_n":43,"acceptable_set_nll":1.3800764413857416,"acceptable_set_zero_mass_n":0},"calibration_all_labels":{"equal_width":{"n":43,"bin_count":10,"ece":0.18604114027910454,"signed_mean_gap":-0.18604114027910454,"absolute_mean_gap":0.18604114027910454,"bins":[{"index":0,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.0,"upper":0.1,"upper_inclusive":false},{"index":1,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.1,"upper":0.2,"upper_inclusive":false},{"index":2,"count":18,"confidence":0.26114236149522996,"empirical_correctness":0.5,"signed_gap":-0.23885763850477004,"absolute_gap":0.23885763850477004,"lower":0.2,"upper":0.3,"upper_inclusive":false},{"index":3,"count":18,"confidence":0.35147761967447066,"empirical_correctness":0.5,"signed_gap":-0.14852238032552934,"absolute_gap":0.14852238032552934,"lower":0.3,"upper":0.4,"upper_inclusive":false},{"index":4,"count":7,"confidence":0.4247244724205562,"empirical_correctness":0.5714285714285714,"signed_gap":-0.14670409900801518,"absolute_gap":0.14670409900801518,"lower":0.4,"upper":0.5,"upper_inclusive":false},{"index":5,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.5,"upper":0.6,"upper_inclusive":false},{"index":6,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.6,"upper":0.7,"upper_inclusive":false},{"index":7,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.7,"upper":0.8,"upper_inclusive":false},{"index":8,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.8,"upper":0.9,"upper_inclusive":false},{"index":9,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.9,"upper":1.0,"upper_inclusive":true}]},"equal_mass":{"n":43,"bin_count":10,"ece":0.21474810395129892,"signed_mean_gap":-0.18604114027910454,"absolute_mean_gap":0.18604114027910454,"bins":[{"index":0,"count":4,"confidence":0.23549848049879074,"empirical_correctness":0.5,"signed_gap":-0.26450151950120926,"absolute_gap":0.26450151950120926,"min_confidence":0.22428084909915924,"max_confidence":0.24947340786457062},{"index":1,"count":4,"confidence":0.2523569017648697,"empirical_correctness":0.5,"signed_gap":-0.2476430982351303,"absolute_gap":0.2476430982351303,"min_confidence":0.25161662697792053,"max_confidence":0.25394201278686523},{"index":2,"count":4,"confidence":0.2620752081274986,"empirical_correctness":0.5,"signed_gap":-0.23792479187250137,"absolute_gap":0.23792479187250137,"min_confidence":0.2546480894088745,"max_confidence":0.2661600112915039},{"index":3,"count":5,"confidence":0.28241094946861267,"empirical_correctness":0.6,"signed_gap":-0.3175890505313873,"absolute_gap":0.3175890505313873,"min_confidence":0.2686562240123749,"max_confidence":0.2877246141433716},{"index":4,"count":4,"confidence":0.3030046820640564,"empirical_correctness":0.5,"signed_gap":-0.1969953179359436,"absolute_gap":0.1969953179359436,"min_confidence":0.2887853980064392,"max_confidence":0.312656968832016},{"index":5,"count":4,"confidence":0.32911188155412674,"empirical_correctness":0.75,"signed_gap":-0.42088811844587326,"absolute_gap":0.42088811844587326,"min_confidence":0.3138015866279602,"max_confidence":0.3381534814834595},{"index":6,"count":5,"confidence":0.352533882856369,"empirical_correctness":0.4,"signed_gap":-0.047466117143631026,"absolute_gap":0.047466117143631026,"min_confidence":0.34466055035591125,"max_confidence":0.36109408736228943},{"index":7,"count":4,"confidence":0.38181790709495544,"empirical_correctness":0.5,"signed_gap":-0.11818209290504456,"absolute_gap":0.11818209290504456,"min_confidence":0.369198203086853,"max_confidence":0.38716039061546326},{"index":8,"count":4,"confidence":0.40429992973804474,"empirical_correctness":0.25,"signed_gap":0.15429992973804474,"absolute_gap":0.15429992973804474,"min_confidence":0.3973321318626404,"max_confidence":0.411659836769104},{"index":9,"count":5,"confidence":0.4305693686008453,"empirical_correctness":0.6,"signed_gap":-0.16943063139915465,"absolute_gap":0.16943063139915465,"min_confidence":0.4159790575504303,"max_confidence":0.448964923620224}]}},"calibration_singleton_labels":{"equal_width":{"n":38,"bin_count":10,"ece":0.14487073609703466,"signed_mean_gap":-0.14487073609703466,"absolute_mean_gap":0.14487073609703466,"bins":[{"index":0,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.0,"upper":0.1,"upper_inclusive":false},{"index":1,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.1,"upper":0.2,"upper_inclusive":false},{"index":2,"count":14,"confidence":0.25763595742838724,"empirical_correctness":0.42857142857142855,"signed_gap":-0.1709354711430413,"absolute_gap":0.1709354711430413,"lower":0.2,"upper":0.3,"upper_inclusive":false},{"index":3,"count":18,"confidence":0.35147761967447066,"empirical_correctness":0.5,"signed_gap":-0.14852238032552934,"absolute_gap":0.14852238032552934,"lower":0.3,"upper":0.4,"upper_inclusive":false},{"index":4,"count":6,"confidence":0.4269019116957982,"empirical_correctness":0.5,"signed_gap":-0.07309808830420178,"absolute_gap":0.07309808830420178,"lower":0.4,"upper":0.5,"upper_inclusive":false},{"index":5,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.5,"upper":0.6,"upper_inclusive":false},{"index":6,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.6,"upper":0.7,"upper_inclusive":false},{"index":7,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.7,"upper":0.8,"upper_inclusive":false},{"index":8,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.8,"upper":0.9,"upper_inclusive":false},{"index":9,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.9,"upper":1.0,"upper_inclusive":true}]},"equal_mass":{"n":38,"bin_count":10,"ece":0.18025578793726468,"signed_mean_gap":-0.14487073609703466,"absolute_mean_gap":0.14487073609703466,"bins":[{"index":0,"count":3,"confidence":0.2308401713768641,"empirical_correctness":0.3333333333333333,"signed_gap":-0.10249316195646921,"absolute_gap":0.10249316195646921,"min_confidence":0.22428084909915924,"max_confidence":0.2408909797668457},{"index":1,"count":4,"confidence":0.25123975053429604,"empirical_correctness":0.75,"signed_gap":-0.49876024946570396,"absolute_gap":0.49876024946570396,"min_confidence":0.24947340786457062,"max_confidence":0.25210949778556824},{"index":2,"count":4,"confidence":0.26269926130771637,"empirical_correctness":0.25,"signed_gap":0.01269926130771637,"absolute_gap":0.01269926130771637,"min_confidence":0.2546480894088745,"max_confidence":0.2686562240123749},{"index":3,"count":4,"confidence":0.2901763767004013,"empirical_correctness":0.5,"signed_gap":-0.2098236232995987,"absolute_gap":0.2098236232995987,"min_confidence":0.2835783064365387,"max_confidence":0.3020786643028259},{"index":4,"count":4,"confidence":0.3165348693728447,"empirical_correctness":0.5,"signed_gap":-0.1834651306271553,"absolute_gap":0.1834651306271553,"min_confidence":0.30849769711494446,"max_confidence":0.33118322491645813},{"index":5,"count":3,"confidence":0.3387077550093333,"empirical_correctness":0.6666666666666666,"signed_gap":-0.3279589116573333,"absolute_gap":0.3279589116573333,"min_confidence":0.33330923318862915,"max_confidence":0.34466055035591125},{"index":6,"count":4,"confidence":0.35450221598148346,"empirical_correctness":0.5,"signed_gap":-0.14549778401851654,"absolute_gap":0.14549778401851654,"min_confidence":0.3479155898094177,"max_confidence":0.36109408736228943},{"index":7,"count":4,"confidence":0.38181790709495544,"empirical_correctness":0.5,"signed_gap":-0.11818209290504456,"absolute_gap":0.11818209290504456,"min_confidence":0.369198203086853,"max_confidence":0.38716039061546326},{"index":8,"count":4,"confidence":0.4053797349333763,"empirical_correctness":0.25,"signed_gap":0.1553797349333763,"absolute_gap":0.1553797349333763,"min_confidence":0.3973321318626404,"max_confidence":0.4159790575504303},{"index":9,"count":4,"confidence":0.4342169463634491,"empirical_correctness":0.5,"signed_gap":-0.0657830536365509,"absolute_gap":0.0657830536365509,"min_confidence":0.41731253266334534,"max_confidence":0.448964923620224}]}},"latency_ms":{"n":43,"median":31.703900080174208,"p95":33.69487994350493},"per_action":{"keep":{"n":12,"raw":{"n":12,"acceptable":6,"unsafe":0,"review":0,"automated_count":12,"automated_acceptable":6,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":0.5,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.5,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.2537815976337061,"upper":0.7462184023662939,"confidence_level":0.95}},"effective":{"n":12,"acceptable":0,"unsafe":0,"review":12,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":12,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":2.7755575615628914e-17,"upper":0.24249400665524085,"confidence_level":0.95}}},"drop":{"n":16,"raw":{"n":16,"acceptable":11,"unsafe":4,"review":0,"automated_count":16,"automated_acceptable":11,"automated_unsafe":4,"unnecessary_reviews":0,"accuracy":0.6875,"unsafe_rate":0.25,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.6875,"conditional_unsafe":0.25,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.44404355834740666,"upper":0.8583535614521796,"confidence_level":0.95}},"effective":{"n":16,"acceptable":0,"unsafe":0,"review":14,"automated_count":2,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":14,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":0.875,"automated_coverage":0.125,"conditional_correctness":0.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.875,"accuracy_wilson_95":{"lower":0.0,"upper":0.1936076805344365,"confidence_level":0.95}}},"mask":{"n":2,"raw":{"n":2,"acceptable":0,"unsafe":2,"review":0,"automated_count":2,"automated_acceptable":0,"automated_unsafe":2,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":1.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":1.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.6576197724933469,"confidence_level":0.95}},"effective":{"n":2,"acceptable":1,"unsafe":0,"review":1,"automated_count":1,"automated_acceptable":1,"automated_unsafe":0,"unnecessary_reviews":1,"accuracy":0.5,"unsafe_rate":0.0,"review_rate":0.5,"automated_coverage":0.5,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.5,"accuracy_wilson_95":{"lower":0.09453120573423074,"upper":0.9054687942657693,"confidence_level":0.95}}},"pseudonymize":{"n":2,"raw":{"n":2,"acceptable":0,"unsafe":0,"review":0,"automated_count":2,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.6576197724933469,"confidence_level":0.95}},"effective":{"n":2,"acceptable":0,"unsafe":0,"review":2,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":2,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.6576197724933469,"confidence_level":0.95}}},"generalize":{"n":6,"raw":{"n":6,"acceptable":1,"unsafe":0,"review":0,"automated_count":6,"automated_acceptable":1,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":0.16666666666666666,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.16666666666666666,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.030053369748306635,"upper":0.5635028221864702,"confidence_level":0.95}},"effective":{"n":6,"acceptable":0,"unsafe":0,"review":6,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":6,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":2.7755575615628914e-17,"upper":0.3903342879021653,"confidence_level":0.95}}},"review":{"n":0,"raw":{"n":0,"acceptable":0,"unsafe":0,"review":0,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":null,"unsafe_rate":null,"review_rate":null,"automated_coverage":null,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":null},"effective":{"n":0,"acceptable":0,"unsafe":0,"review":0,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":null,"unsafe_rate":null,"review_rate":null,"automated_coverage":null,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":null}}}},"challenge56-no_samples":{"n":56,"raw":{"n":56,"acceptable":15,"unsafe":17,"review":0,"automated_count":56,"automated_acceptable":15,"automated_unsafe":17,"unnecessary_reviews":0,"accuracy":0.26785714285714285,"unsafe_rate":0.30357142857142855,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.26785714285714285,"conditional_unsafe":0.30357142857142855,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.16957305418501728,"upper":0.39594555929155156,"confidence_level":0.95}},"effective":{"n":56,"acceptable":2,"unsafe":0,"review":55,"automated_count":1,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":53,"accuracy":0.03571428571428571,"unsafe_rate":0.0,"review_rate":0.9821428571428571,"automated_coverage":0.017857142857142856,"conditional_correctness":0.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.9464285714285714,"accuracy_wilson_95":{"lower":0.00984942514823639,"upper":0.12118780180490107,"confidence_level":0.95}},"confusion":{"raw":{"keep":{"keep":0,"drop":0,"mask":8,"pseudonymize":0,"generalize":0,"review":0},"drop":{"keep":5,"drop":12,"mask":1,"pseudonymize":4,"generalize":0,"review":0},"mask":{"keep":0,"drop":6,"mask":0,"pseudonymize":2,"generalize":0,"review":0},"pseudonymize":{"keep":4,"drop":0,"mask":3,"pseudonymize":1,"generalize":0,"review":0},"generalize":{"keep":2,"drop":4,"mask":0,"pseudonymize":0,"generalize":2,"review":0},"review":{"keep":0,"drop":2,"mask":0,"pseudonymize":0,"generalize":0,"review":0}},"effective":{"keep":{"keep":0,"drop":0,"mask":1,"pseudonymize":0,"generalize":0,"review":7},"drop":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":22},"mask":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":8},"pseudonymize":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":8},"generalize":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":8},"review":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":2}}},"probability_metrics":{"singleton_n":56,"multiacceptable_n":0,"multiacceptable_coverage":0.0,"nll":1.8491480317322146,"nll_zero_mass_n":0,"brier":0.8534813606736276,"acceptable_set_n":56,"acceptable_set_nll":1.8491480317322146,"acceptable_set_zero_mass_n":0},"calibration_all_labels":{"equal_width":{"n":56,"bin_count":10,"ece":0.04389677861971513,"signed_mean_gap":0.04389677861971513,"absolute_mean_gap":0.04389677861971513,"bins":[{"index":0,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.0,"upper":0.1,"upper_inclusive":false},{"index":1,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.1,"upper":0.2,"upper_inclusive":false},{"index":2,"count":27,"confidence":0.270578666417687,"empirical_correctness":0.25925925925925924,"signed_gap":0.011319407158427774,"absolute_gap":0.011319407158427774,"lower":0.2,"upper":0.3,"upper_inclusive":false},{"index":3,"count":24,"confidence":0.33316615720589954,"empirical_correctness":0.2916666666666667,"signed_gap":0.04149949053923285,"absolute_gap":0.04149949053923285,"lower":0.3,"upper":0.4,"upper_inclusive":false},{"index":4,"count":5,"confidence":0.4313215672969818,"empirical_correctness":0.2,"signed_gap":0.23132156729698178,"absolute_gap":0.23132156729698178,"lower":0.4,"upper":0.5,"upper_inclusive":false},{"index":5,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.5,"upper":0.6,"upper_inclusive":false},{"index":6,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.6,"upper":0.7,"upper_inclusive":false},{"index":7,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.7,"upper":0.8,"upper_inclusive":false},{"index":8,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.8,"upper":0.9,"upper_inclusive":false},{"index":9,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.9,"upper":1.0,"upper_inclusive":true}]},"equal_mass":{"n":56,"bin_count":10,"ece":0.14184097280459745,"signed_mean_gap":0.043896778619715145,"absolute_mean_gap":0.043896778619715145,"bins":[{"index":0,"count":5,"confidence":0.2377529114484787,"empirical_correctness":0.4,"signed_gap":-0.1622470885515213,"absolute_gap":0.1622470885515213,"min_confidence":0.2244722992181778,"max_confidence":0.25089362263679504},{"index":1,"count":6,"confidence":0.2613683839639028,"empirical_correctness":0.0,"signed_gap":0.2613683839639028,"absolute_gap":0.2613683839639028,"min_confidence":0.25134310126304626,"max_confidence":0.2650676369667053},{"index":2,"count":5,"confidence":0.2731179535388947,"empirical_correctness":0.2,"signed_gap":0.07311795353889466,"absolute_gap":0.07311795353889466,"min_confidence":0.2675001919269562,"max_confidence":0.2789311110973358},{"index":3,"count":6,"confidence":0.2859063893556595,"empirical_correctness":0.3333333333333333,"signed_gap":-0.04742694397767383,"absolute_gap":0.04742694397767383,"min_confidence":0.2804229259490967,"max_confidence":0.28963974118232727},{"index":4,"count":6,"confidence":0.29467374583085376,"empirical_correctness":0.3333333333333333,"signed_gap":-0.03865958750247955,"absolute_gap":0.03865958750247955,"min_confidence":0.29070621728897095,"max_confidence":0.30042144656181335},{"index":5,"count":5,"confidence":0.3037588894367218,"empirical_correctness":0.0,"signed_gap":0.3037588894367218,"absolute_gap":0.3037588894367218,"min_confidence":0.30103394389152527,"max_confidence":0.30683887004852295},{"index":6,"count":6,"confidence":0.31409668922424316,"empirical_correctness":0.5,"signed_gap":-0.18590331077575684,"absolute_gap":0.18590331077575684,"min_confidence":0.30703410506248474,"max_confidence":0.32870879769325256},{"index":7,"count":5,"confidence":0.3401474118232727,"empirical_correctness":0.4,"signed_gap":-0.05985258817672734,"absolute_gap":0.05985258817672734,"min_confidence":0.33165907859802246,"max_confidence":0.34651196002960205},{"index":8,"count":6,"confidence":0.3664729942878087,"empirical_correctness":0.3333333333333333,"signed_gap":0.0331396609544754,"absolute_gap":0.0331396609544754,"min_confidence":0.35551708936691284,"max_confidence":0.37263622879981995},{"index":9,"count":6,"confidence":0.4248707592487335,"empirical_correctness":0.16666666666666666,"signed_gap":0.2582040925820669,"absolute_gap":0.2582040925820669,"min_confidence":0.39261671900749207,"max_confidence":0.4546016454696655}]}},"calibration_singleton_labels":{"equal_width":{"n":56,"bin_count":10,"ece":0.04389677861971513,"signed_mean_gap":0.04389677861971513,"absolute_mean_gap":0.04389677861971513,"bins":[{"index":0,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.0,"upper":0.1,"upper_inclusive":false},{"index":1,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.1,"upper":0.2,"upper_inclusive":false},{"index":2,"count":27,"confidence":0.270578666417687,"empirical_correctness":0.25925925925925924,"signed_gap":0.011319407158427774,"absolute_gap":0.011319407158427774,"lower":0.2,"upper":0.3,"upper_inclusive":false},{"index":3,"count":24,"confidence":0.33316615720589954,"empirical_correctness":0.2916666666666667,"signed_gap":0.04149949053923285,"absolute_gap":0.04149949053923285,"lower":0.3,"upper":0.4,"upper_inclusive":false},{"index":4,"count":5,"confidence":0.4313215672969818,"empirical_correctness":0.2,"signed_gap":0.23132156729698178,"absolute_gap":0.23132156729698178,"lower":0.4,"upper":0.5,"upper_inclusive":false},{"index":5,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.5,"upper":0.6,"upper_inclusive":false},{"index":6,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.6,"upper":0.7,"upper_inclusive":false},{"index":7,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.7,"upper":0.8,"upper_inclusive":false},{"index":8,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.8,"upper":0.9,"upper_inclusive":false},{"index":9,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.9,"upper":1.0,"upper_inclusive":true}]},"equal_mass":{"n":56,"bin_count":10,"ece":0.14184097280459745,"signed_mean_gap":0.043896778619715145,"absolute_mean_gap":0.043896778619715145,"bins":[{"index":0,"count":5,"confidence":0.2377529114484787,"empirical_correctness":0.4,"signed_gap":-0.1622470885515213,"absolute_gap":0.1622470885515213,"min_confidence":0.2244722992181778,"max_confidence":0.25089362263679504},{"index":1,"count":6,"confidence":0.2613683839639028,"empirical_correctness":0.0,"signed_gap":0.2613683839639028,"absolute_gap":0.2613683839639028,"min_confidence":0.25134310126304626,"max_confidence":0.2650676369667053},{"index":2,"count":5,"confidence":0.2731179535388947,"empirical_correctness":0.2,"signed_gap":0.07311795353889466,"absolute_gap":0.07311795353889466,"min_confidence":0.2675001919269562,"max_confidence":0.2789311110973358},{"index":3,"count":6,"confidence":0.2859063893556595,"empirical_correctness":0.3333333333333333,"signed_gap":-0.04742694397767383,"absolute_gap":0.04742694397767383,"min_confidence":0.2804229259490967,"max_confidence":0.28963974118232727},{"index":4,"count":6,"confidence":0.29467374583085376,"empirical_correctness":0.3333333333333333,"signed_gap":-0.03865958750247955,"absolute_gap":0.03865958750247955,"min_confidence":0.29070621728897095,"max_confidence":0.30042144656181335},{"index":5,"count":5,"confidence":0.3037588894367218,"empirical_correctness":0.0,"signed_gap":0.3037588894367218,"absolute_gap":0.3037588894367218,"min_confidence":0.30103394389152527,"max_confidence":0.30683887004852295},{"index":6,"count":6,"confidence":0.31409668922424316,"empirical_correctness":0.5,"signed_gap":-0.18590331077575684,"absolute_gap":0.18590331077575684,"min_confidence":0.30703410506248474,"max_confidence":0.32870879769325256},{"index":7,"count":5,"confidence":0.3401474118232727,"empirical_correctness":0.4,"signed_gap":-0.05985258817672734,"absolute_gap":0.05985258817672734,"min_confidence":0.33165907859802246,"max_confidence":0.34651196002960205},{"index":8,"count":6,"confidence":0.3664729942878087,"empirical_correctness":0.3333333333333333,"signed_gap":0.0331396609544754,"absolute_gap":0.0331396609544754,"min_confidence":0.35551708936691284,"max_confidence":0.37263622879981995},{"index":9,"count":6,"confidence":0.4248707592487335,"empirical_correctness":0.16666666666666666,"signed_gap":0.2582040925820669,"absolute_gap":0.2582040925820669,"min_confidence":0.39261671900749207,"max_confidence":0.4546016454696655}]}},"latency_ms":{"n":56,"median":25.784149998798966,"p95":26.469500153325498},"per_action":{"keep":{"n":8,"raw":{"n":8,"acceptable":0,"unsafe":0,"review":0,"automated_count":8,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.32440756488388023,"confidence_level":0.95}},"effective":{"n":8,"acceptable":0,"unsafe":0,"review":7,"automated_count":1,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":7,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":0.875,"automated_coverage":0.125,"conditional_correctness":0.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.875,"accuracy_wilson_95":{"lower":0.0,"upper":0.32440756488388023,"confidence_level":0.95}}},"drop":{"n":22,"raw":{"n":22,"acceptable":12,"unsafe":9,"review":0,"automated_count":22,"automated_acceptable":12,"automated_unsafe":9,"unnecessary_reviews":0,"accuracy":0.5454545454545454,"unsafe_rate":0.4090909090909091,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.5454545454545454,"conditional_unsafe":0.4090909090909091,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.3465979532073494,"upper":0.7307970581918668,"confidence_level":0.95}},"effective":{"n":22,"acceptable":0,"unsafe":0,"review":22,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":22,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.14865487460862087,"confidence_level":0.95}}},"mask":{"n":8,"raw":{"n":8,"acceptable":0,"unsafe":2,"review":0,"automated_count":8,"automated_acceptable":0,"automated_unsafe":2,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":0.25,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":0.25,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.32440756488388023,"confidence_level":0.95}},"effective":{"n":8,"acceptable":0,"unsafe":0,"review":8,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":8,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.32440756488388023,"confidence_level":0.95}}},"pseudonymize":{"n":8,"raw":{"n":8,"acceptable":1,"unsafe":4,"review":0,"automated_count":8,"automated_acceptable":1,"automated_unsafe":4,"unnecessary_reviews":0,"accuracy":0.125,"unsafe_rate":0.5,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.125,"conditional_unsafe":0.5,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.02241749145005667,"upper":0.4708881822128535,"confidence_level":0.95}},"effective":{"n":8,"acceptable":0,"unsafe":0,"review":8,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":8,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.32440756488388023,"confidence_level":0.95}}},"generalize":{"n":8,"raw":{"n":8,"acceptable":2,"unsafe":2,"review":0,"automated_count":8,"automated_acceptable":2,"automated_unsafe":2,"unnecessary_reviews":0,"accuracy":0.25,"unsafe_rate":0.25,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.25,"conditional_unsafe":0.25,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.071479212752109,"upper":0.5907245696898311,"confidence_level":0.95}},"effective":{"n":8,"acceptable":0,"unsafe":0,"review":8,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":8,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.32440756488388023,"confidence_level":0.95}}},"review":{"n":2,"raw":{"n":2,"acceptable":0,"unsafe":0,"review":0,"automated_count":2,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.6576197724933469,"confidence_level":0.95}},"effective":{"n":2,"acceptable":2,"unsafe":0,"review":2,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":1.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.34238022750665303,"upper":1.0,"confidence_level":0.95}}}}},"challenge56-compact_no_samples":{"n":56,"raw":{"n":56,"acceptable":7,"unsafe":47,"review":0,"automated_count":56,"automated_acceptable":7,"automated_unsafe":47,"unnecessary_reviews":0,"accuracy":0.125,"unsafe_rate":0.8392857142857143,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.125,"conditional_unsafe":0.8392857142857143,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.06189068695717122,"upper":0.2362547655819014,"confidence_level":0.95}},"effective":{"n":56,"acceptable":2,"unsafe":0,"review":56,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":54,"accuracy":0.03571428571428571,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":0.9642857142857143,"accuracy_wilson_95":{"lower":0.00984942514823639,"upper":0.12118780180490107,"confidence_level":0.95}},"confusion":{"raw":{"keep":{"keep":7,"drop":0,"mask":0,"pseudonymize":1,"generalize":0,"review":0},"drop":{"keep":22,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0},"mask":{"keep":6,"drop":1,"mask":0,"pseudonymize":1,"generalize":0,"review":0},"pseudonymize":{"keep":8,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0},"generalize":{"keep":8,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0},"review":{"keep":2,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0}},"effective":{"keep":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":8},"drop":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":22},"mask":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":8},"pseudonymize":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":8},"generalize":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":8},"review":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":2}}},"probability_metrics":{"singleton_n":56,"multiacceptable_n":0,"multiacceptable_coverage":0.0,"nll":1.9353268926333016,"nll_zero_mass_n":0,"brier":0.9178216081888992,"acceptable_set_n":56,"acceptable_set_nll":1.9353268926333016,"acceptable_set_zero_mass_n":0},"calibration_all_labels":{"equal_width":{"n":56,"bin_count":10,"ece":0.2931315414607525,"signed_mean_gap":0.2672533728182316,"absolute_mean_gap":0.2672533728182316,"bins":[{"index":0,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.0,"upper":0.1,"upper_inclusive":false},{"index":1,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.1,"upper":0.2,"upper_inclusive":false},{"index":2,"count":1,"confidence":0.2754112780094147,"empirical_correctness":1.0,"signed_gap":-0.7245887219905853,"absolute_gap":0.7245887219905853,"lower":0.2,"upper":0.3,"upper_inclusive":false},{"index":3,"count":31,"confidence":0.35021094833650895,"empirical_correctness":0.16129032258064516,"signed_gap":0.1889206257558638,"absolute_gap":0.1889206257558638,"lower":0.3,"upper":0.4,"upper_inclusive":false},{"index":4,"count":22,"confidence":0.446762278676033,"empirical_correctness":0.045454545454545456,"signed_gap":0.40130773322148755,"absolute_gap":0.40130773322148755,"lower":0.4,"upper":0.5,"upper_inclusive":false},{"index":5,"count":2,"confidence":0.5027340352535248,"empirical_correctness":0.0,"signed_gap":0.5027340352535248,"absolute_gap":0.5027340352535248,"lower":0.5,"upper":0.6,"upper_inclusive":false},{"index":6,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.6,"upper":0.7,"upper_inclusive":false},{"index":7,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.7,"upper":0.8,"upper_inclusive":false},{"index":8,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.8,"upper":0.9,"upper_inclusive":false},{"index":9,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.9,"upper":1.0,"upper_inclusive":true}]},"equal_mass":{"n":56,"bin_count":10,"ece":0.2840064630976745,"signed_mean_gap":0.2672533728182316,"absolute_mean_gap":0.2672533728182316,"bins":[{"index":0,"count":5,"confidence":0.30850919485092165,"empirical_correctness":0.4,"signed_gap":-0.09149080514907837,"absolute_gap":0.09149080514907837,"min_confidence":0.2754112780094147,"max_confidence":0.3228275179862976},{"index":1,"count":6,"confidence":0.33139458298683167,"empirical_correctness":0.3333333333333333,"signed_gap":-0.0019387503465016498,"absolute_gap":0.0019387503465016498,"min_confidence":0.3230094015598297,"max_confidence":0.3347153663635254},{"index":2,"count":5,"confidence":0.34116233587265016,"empirical_correctness":0.2,"signed_gap":0.14116233587265015,"absolute_gap":0.14116233587265015,"min_confidence":0.3372569680213928,"max_confidence":0.3433525860309601},{"index":3,"count":6,"confidence":0.35055748124917346,"empirical_correctness":0.16666666666666666,"signed_gap":0.1838908145825068,"absolute_gap":0.1838908145825068,"min_confidence":0.34561917185783386,"max_confidence":0.3537857234477997},{"index":4,"count":6,"confidence":0.36968547602494556,"empirical_correctness":0.0,"signed_gap":0.36968547602494556,"absolute_gap":0.36968547602494556,"min_confidence":0.356267511844635,"max_confidence":0.38373488187789917},{"index":5,"count":5,"confidence":0.39557700157165526,"empirical_correctness":0.0,"signed_gap":0.39557700157165526,"absolute_gap":0.39557700157165526,"min_confidence":0.3872387707233429,"max_confidence":0.404117226600647},{"index":6,"count":6,"confidence":0.41724814971288043,"empirical_correctness":0.0,"signed_gap":0.41724814971288043,"absolute_gap":0.41724814971288043,"min_confidence":0.4072137475013733,"max_confidence":0.42894527316093445},{"index":7,"count":5,"confidence":0.4430148839950562,"empirical_correctness":0.2,"signed_gap":0.24301488399505616,"absolute_gap":0.24301488399505616,"min_confidence":0.43722206354141235,"max_confidence":0.4498985707759857},{"index":8,"count":6,"confidence":0.4619824141263962,"empirical_correctness":0.0,"signed_gap":0.4619824141263962,"absolute_gap":0.4619824141263962,"min_confidence":0.45189976692199707,"max_confidence":0.4745405912399292},{"index":9,"count":6,"confidence":0.48994386196136475,"empirical_correctness":0.0,"signed_gap":0.48994386196136475,"absolute_gap":0.48994386196136475,"min_confidence":0.476746529340744,"max_confidence":0.5043742060661316}]}},"calibration_singleton_labels":{"equal_width":{"n":56,"bin_count":10,"ece":0.2931315414607525,"signed_mean_gap":0.2672533728182316,"absolute_mean_gap":0.2672533728182316,"bins":[{"index":0,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.0,"upper":0.1,"upper_inclusive":false},{"index":1,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.1,"upper":0.2,"upper_inclusive":false},{"index":2,"count":1,"confidence":0.2754112780094147,"empirical_correctness":1.0,"signed_gap":-0.7245887219905853,"absolute_gap":0.7245887219905853,"lower":0.2,"upper":0.3,"upper_inclusive":false},{"index":3,"count":31,"confidence":0.35021094833650895,"empirical_correctness":0.16129032258064516,"signed_gap":0.1889206257558638,"absolute_gap":0.1889206257558638,"lower":0.3,"upper":0.4,"upper_inclusive":false},{"index":4,"count":22,"confidence":0.446762278676033,"empirical_correctness":0.045454545454545456,"signed_gap":0.40130773322148755,"absolute_gap":0.40130773322148755,"lower":0.4,"upper":0.5,"upper_inclusive":false},{"index":5,"count":2,"confidence":0.5027340352535248,"empirical_correctness":0.0,"signed_gap":0.5027340352535248,"absolute_gap":0.5027340352535248,"lower":0.5,"upper":0.6,"upper_inclusive":false},{"index":6,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.6,"upper":0.7,"upper_inclusive":false},{"index":7,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.7,"upper":0.8,"upper_inclusive":false},{"index":8,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.8,"upper":0.9,"upper_inclusive":false},{"index":9,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.9,"upper":1.0,"upper_inclusive":true}]},"equal_mass":{"n":56,"bin_count":10,"ece":0.2840064630976745,"signed_mean_gap":0.2672533728182316,"absolute_mean_gap":0.2672533728182316,"bins":[{"index":0,"count":5,"confidence":0.30850919485092165,"empirical_correctness":0.4,"signed_gap":-0.09149080514907837,"absolute_gap":0.09149080514907837,"min_confidence":0.2754112780094147,"max_confidence":0.3228275179862976},{"index":1,"count":6,"confidence":0.33139458298683167,"empirical_correctness":0.3333333333333333,"signed_gap":-0.0019387503465016498,"absolute_gap":0.0019387503465016498,"min_confidence":0.3230094015598297,"max_confidence":0.3347153663635254},{"index":2,"count":5,"confidence":0.34116233587265016,"empirical_correctness":0.2,"signed_gap":0.14116233587265015,"absolute_gap":0.14116233587265015,"min_confidence":0.3372569680213928,"max_confidence":0.3433525860309601},{"index":3,"count":6,"confidence":0.35055748124917346,"empirical_correctness":0.16666666666666666,"signed_gap":0.1838908145825068,"absolute_gap":0.1838908145825068,"min_confidence":0.34561917185783386,"max_confidence":0.3537857234477997},{"index":4,"count":6,"confidence":0.36968547602494556,"empirical_correctness":0.0,"signed_gap":0.36968547602494556,"absolute_gap":0.36968547602494556,"min_confidence":0.356267511844635,"max_confidence":0.38373488187789917},{"index":5,"count":5,"confidence":0.39557700157165526,"empirical_correctness":0.0,"signed_gap":0.39557700157165526,"absolute_gap":0.39557700157165526,"min_confidence":0.3872387707233429,"max_confidence":0.404117226600647},{"index":6,"count":6,"confidence":0.41724814971288043,"empirical_correctness":0.0,"signed_gap":0.41724814971288043,"absolute_gap":0.41724814971288043,"min_confidence":0.4072137475013733,"max_confidence":0.42894527316093445},{"index":7,"count":5,"confidence":0.4430148839950562,"empirical_correctness":0.2,"signed_gap":0.24301488399505616,"absolute_gap":0.24301488399505616,"min_confidence":0.43722206354141235,"max_confidence":0.4498985707759857},{"index":8,"count":6,"confidence":0.4619824141263962,"empirical_correctness":0.0,"signed_gap":0.4619824141263962,"absolute_gap":0.4619824141263962,"min_confidence":0.45189976692199707,"max_confidence":0.4745405912399292},{"index":9,"count":6,"confidence":0.48994386196136475,"empirical_correctness":0.0,"signed_gap":0.48994386196136475,"absolute_gap":0.48994386196136475,"min_confidence":0.476746529340744,"max_confidence":0.5043742060661316}]}},"latency_ms":{"n":56,"median":29.992399970069528,"p95":31.286974903196096},"per_action":{"keep":{"n":8,"raw":{"n":8,"acceptable":7,"unsafe":0,"review":0,"automated_count":8,"automated_acceptable":7,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":0.875,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.875,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.5291118177871464,"upper":0.9775825085499433,"confidence_level":0.95}},"effective":{"n":8,"acceptable":0,"unsafe":0,"review":8,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":8,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.32440756488388023,"confidence_level":0.95}}},"drop":{"n":22,"raw":{"n":22,"acceptable":0,"unsafe":22,"review":0,"automated_count":22,"automated_acceptable":0,"automated_unsafe":22,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":1.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":1.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.14865487460862087,"confidence_level":0.95}},"effective":{"n":22,"acceptable":0,"unsafe":0,"review":22,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":22,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.14865487460862087,"confidence_level":0.95}}},"mask":{"n":8,"raw":{"n":8,"acceptable":0,"unsafe":7,"review":0,"automated_count":8,"automated_acceptable":0,"automated_unsafe":7,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":0.875,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":0.875,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.32440756488388023,"confidence_level":0.95}},"effective":{"n":8,"acceptable":0,"unsafe":0,"review":8,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":8,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.32440756488388023,"confidence_level":0.95}}},"pseudonymize":{"n":8,"raw":{"n":8,"acceptable":0,"unsafe":8,"review":0,"automated_count":8,"automated_acceptable":0,"automated_unsafe":8,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":1.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":1.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.32440756488388023,"confidence_level":0.95}},"effective":{"n":8,"acceptable":0,"unsafe":0,"review":8,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":8,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.32440756488388023,"confidence_level":0.95}}},"generalize":{"n":8,"raw":{"n":8,"acceptable":0,"unsafe":8,"review":0,"automated_count":8,"automated_acceptable":0,"automated_unsafe":8,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":1.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":1.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.32440756488388023,"confidence_level":0.95}},"effective":{"n":8,"acceptable":0,"unsafe":0,"review":8,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":8,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.32440756488388023,"confidence_level":0.95}}},"review":{"n":2,"raw":{"n":2,"acceptable":0,"unsafe":2,"review":0,"automated_count":2,"automated_acceptable":0,"automated_unsafe":2,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":1.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":1.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.6576197724933469,"confidence_level":0.95}},"effective":{"n":2,"acceptable":2,"unsafe":0,"review":2,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":1.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.34238022750665303,"upper":1.0,"confidence_level":0.95}}}}}},"composite":{"n":198,"scope":"descriptive sum across four correlated panels","raw":{"n":198,"acceptable":65,"unsafe":72,"review":0,"automated_count":198,"automated_acceptable":65,"automated_unsafe":72,"unnecessary_reviews":0,"accuracy":0.3282828282828283,"unsafe_rate":0.36363636363636365,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.3282828282828283,"conditional_unsafe":0.36363636363636365,"unnecessary_review_rate":0.0},"effective":{"n":198,"acceptable":19,"unsafe":0,"review":185,"automated_count":13,"automated_acceptable":5,"automated_unsafe":0,"unnecessary_reviews":171,"accuracy":0.09595959595959595,"unsafe_rate":0.0,"review_rate":0.9343434343434344,"automated_coverage":0.06565656565656566,"conditional_correctness":0.38461538461538464,"conditional_unsafe":0.0,"unnecessary_review_rate":0.8636363636363636}}},{"id":"ameerhmz5--laya-modernbert-decision-90pct-fp32","platform":"Local","model_id":"ameerhmz5/laya-modernbert-decision-90pct","revision":"c2caad4eebb0fef37f6d6011dc79bd539f484ca9","loading":"float32 / none","scoring_kind":"Native decision readout","settings":{"dtype":"float32","quantization":"none","max_tokens":512,"head_max_tokens":192,"temperature":1.9,"autocast":false,"truncation":false,"cpu_offload":false,"guard_threshold":0.45,"seed":0,"tf32":false},"scoring_metadata":{"backend":"laya_native_local_builtin","readout":"native_mask_slot_scorer_numpy_float32_softmax","probability_source":"native_choice_unrounded_published_temperature_not_task_calibrated","latency_scope":"synchronized serial complete encoder+head+act forward,input transfer and native readout; tokenization excluded","native_disclosures":["Stock installed torch attention in place of ONNX dynamic equivalent; no custom kernels or repo imports.","FP32/no-autocast explicit precision: convert every saved F16/BF16/F32 tensor toFP32; no quantization.","Unrounded native probability; guard uses winner p, not entropy-concentration confidence.","Act head fully loaded and evaluated; not used to alter six-action answer.","Native choice:6-10 bucket else choice scalar0, T=1.9. No task fitting, priors correction, prompt change or finetuning here."],"equal_mass_convention":"rank bins with floor boundaries at floor(i*n/k), k=min(10,n//2); minimum two rows"},"load_ms":null,"peak_allocated_bytes":1748760576,"peak_reserved_bytes":1870659584,"panels":{"diagnostic43-no_samples":{"n":43,"raw":{"n":43,"acceptable":21,"unsafe":4,"review":0,"automated_count":43,"automated_acceptable":21,"automated_unsafe":4,"unnecessary_reviews":0,"accuracy":0.4883720930232558,"unsafe_rate":0.09302325581395349,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.4883720930232558,"conditional_unsafe":0.09302325581395349,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.3461744754671859,"upper":0.632476915305507,"confidence_level":0.95}},"effective":{"n":43,"acceptable":5,"unsafe":0,"review":43,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":38,"accuracy":0.11627906976744186,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":0.8837209302325582,"accuracy_wilson_95":{"lower":0.05070427739804166,"upper":0.24479161810082128,"confidence_level":0.95}},"confusion":{"raw":{"keep":{"keep":1,"drop":11,"mask":0,"pseudonymize":0,"generalize":0,"review":0},"drop":{"keep":0,"drop":16,"mask":0,"pseudonymize":0,"generalize":0,"review":0},"mask":{"keep":0,"drop":2,"mask":0,"pseudonymize":0,"generalize":0,"review":0},"pseudonymize":{"keep":1,"drop":1,"mask":0,"pseudonymize":0,"generalize":0,"review":0},"generalize":{"keep":2,"drop":4,"mask":0,"pseudonymize":0,"generalize":0,"review":0},"review":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0}},"effective":{"keep":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":12},"drop":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":16},"mask":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":2},"pseudonymize":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":2},"generalize":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":6},"review":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0}}},"probability_metrics":{"singleton_n":38,"multiacceptable_n":5,"multiacceptable_coverage":0.11627906976744186,"nll":1.642225186317525,"nll_zero_mass_n":0,"brier":0.7716447341467453,"acceptable_set_n":43,"acceptable_set_nll":1.5179083795531991,"acceptable_set_zero_mass_n":0},"calibration_all_labels":{"equal_width":{"n":43,"bin_count":10,"ece":0.20547934327014658,"signed_mean_gap":-0.20547934327014658,"absolute_mean_gap":0.20547934327014658,"bins":[{"index":0,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.0,"upper":0.1,"upper_inclusive":false},{"index":1,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.1,"upper":0.2,"upper_inclusive":false},{"index":2,"count":27,"confidence":0.2546550488030469,"empirical_correctness":0.48148148148148145,"signed_gap":-0.22682643267843455,"absolute_gap":0.22682643267843455,"lower":0.2,"upper":0.3,"upper_inclusive":false},{"index":3,"count":16,"confidence":0.33054387010633945,"empirical_correctness":0.5,"signed_gap":-0.16945612989366055,"absolute_gap":0.16945612989366055,"lower":0.3,"upper":0.4,"upper_inclusive":false},{"index":4,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.4,"upper":0.5,"upper_inclusive":false},{"index":5,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.5,"upper":0.6,"upper_inclusive":false},{"index":6,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.6,"upper":0.7,"upper_inclusive":false},{"index":7,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.7,"upper":0.8,"upper_inclusive":false},{"index":8,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.8,"upper":0.9,"upper_inclusive":false},{"index":9,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.9,"upper":1.0,"upper_inclusive":true}]},"equal_mass":{"n":43,"bin_count":10,"ece":0.2455817128336707,"signed_mean_gap":-0.20547934327014658,"absolute_mean_gap":0.20547934327014658,"bins":[{"index":0,"count":4,"confidence":0.21735307201743126,"empirical_correctness":0.5,"signed_gap":-0.28264692798256874,"absolute_gap":0.28264692798256874,"min_confidence":0.21408158540725708,"max_confidence":0.2194606065750122},{"index":1,"count":4,"confidence":0.23001208901405334,"empirical_correctness":0.25,"signed_gap":-0.019987910985946655,"absolute_gap":0.019987910985946655,"min_confidence":0.2195301055908203,"max_confidence":0.23869642615318298},{"index":2,"count":4,"confidence":0.24738870188593864,"empirical_correctness":0.5,"signed_gap":-0.25261129811406136,"absolute_gap":0.25261129811406136,"min_confidence":0.24205271899700165,"max_confidence":0.25145354866981506},{"index":3,"count":5,"confidence":0.2577018737792969,"empirical_correctness":0.4,"signed_gap":-0.14229812622070315,"absolute_gap":0.14229812622070315,"min_confidence":0.2549774646759033,"max_confidence":0.26014867424964905},{"index":4,"count":4,"confidence":0.2640490159392357,"empirical_correctness":0.75,"signed_gap":-0.4859509840607643,"absolute_gap":0.4859509840607643,"min_confidence":0.26050594449043274,"max_confidence":0.26698073744773865},{"index":5,"count":4,"confidence":0.2885776534676552,"empirical_correctness":0.75,"signed_gap":-0.4614223465323448,"absolute_gap":0.4614223465323448,"min_confidence":0.2731287181377411,"max_confidence":0.2964984178543091},{"index":6,"count":5,"confidence":0.3056651532649994,"empirical_correctness":0.2,"signed_gap":0.10566515326499937,"absolute_gap":0.10566515326499937,"min_confidence":0.29875731468200684,"max_confidence":0.3141675889492035},{"index":7,"count":4,"confidence":0.3242048993706703,"empirical_correctness":0.75,"signed_gap":-0.4257951006293297,"absolute_gap":0.4257951006293297,"min_confidence":0.32123133540153503,"max_confidence":0.3269321620464325},{"index":8,"count":4,"confidence":0.33346879482269287,"empirical_correctness":0.25,"signed_gap":0.08346879482269287,"absolute_gap":0.08346879482269287,"min_confidence":0.329334557056427,"max_confidence":0.3362143039703369},{"index":9,"count":5,"confidence":0.3454672396183014,"empirical_correctness":0.6,"signed_gap":-0.2545327603816986,"absolute_gap":0.2545327603816986,"min_confidence":0.3368264138698578,"max_confidence":0.3524456322193146}]}},"calibration_singleton_labels":{"equal_width":{"n":38,"bin_count":10,"ece":0.16244705491944364,"signed_mean_gap":-0.16244705491944364,"absolute_mean_gap":0.16244705491944364,"bins":[{"index":0,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.0,"upper":0.1,"upper_inclusive":false},{"index":1,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.1,"upper":0.2,"upper_inclusive":false},{"index":2,"count":23,"confidence":0.2549190987711367,"empirical_correctness":0.43478260869565216,"signed_gap":-0.17986350992451544,"absolute_gap":0.17986350992451544,"lower":0.2,"upper":0.3,"upper_inclusive":false},{"index":3,"count":15,"confidence":0.3309248427549998,"empirical_correctness":0.4666666666666667,"signed_gap":-0.1357418239116669,"absolute_gap":0.1357418239116669,"lower":0.3,"upper":0.4,"upper_inclusive":false},{"index":4,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.4,"upper":0.5,"upper_inclusive":false},{"index":5,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.5,"upper":0.6,"upper_inclusive":false},{"index":6,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.6,"upper":0.7,"upper_inclusive":false},{"index":7,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.7,"upper":0.8,"upper_inclusive":false},{"index":8,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.8,"upper":0.9,"upper_inclusive":false},{"index":9,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.9,"upper":1.0,"upper_inclusive":true}]},"equal_mass":{"n":38,"bin_count":10,"ece":0.24691151082515717,"signed_mean_gap":-0.16244705491944364,"absolute_mean_gap":0.16244705491944364,"bins":[{"index":0,"count":3,"confidence":0.2166505604982376,"empirical_correctness":0.6666666666666666,"signed_gap":-0.450016106168429,"absolute_gap":0.450016106168429,"min_confidence":0.21408158540725708,"max_confidence":0.21892113983631134},{"index":1,"count":4,"confidence":0.22727232053875923,"empirical_correctness":0.25,"signed_gap":-0.02272767946124077,"absolute_gap":0.02272767946124077,"min_confidence":0.2194606065750122,"max_confidence":0.23869642615318298},{"index":2,"count":4,"confidence":0.24892057478427887,"empirical_correctness":0.25,"signed_gap":-0.0010794252157211304,"absolute_gap":0.0010794252157211304,"min_confidence":0.24205271899700165,"max_confidence":0.2549774646759033},{"index":3,"count":4,"confidence":0.2584722936153412,"empirical_correctness":0.25,"signed_gap":0.008472293615341187,"absolute_gap":0.008472293615341187,"min_confidence":0.2553330957889557,"max_confidence":0.26050594449043274},{"index":4,"count":4,"confidence":0.2716204449534416,"empirical_correctness":1.0,"signed_gap":-0.7283795550465584,"absolute_gap":0.7283795550465584,"min_confidence":0.2627260982990265,"max_confidence":0.29079166054725647},{"index":5,"count":3,"confidence":0.29638251662254333,"empirical_correctness":0.3333333333333333,"signed_gap":-0.03695081671078998,"absolute_gap":0.03695081671078998,"min_confidence":0.2938918173313141,"max_confidence":0.29875731468200684},{"index":6,"count":4,"confidence":0.30739211291074753,"empirical_correctness":0.25,"signed_gap":0.05739211291074753,"absolute_gap":0.05739211291074753,"min_confidence":0.29889750480651855,"max_confidence":0.3141675889492035},{"index":7,"count":4,"confidence":0.3253312185406685,"empirical_correctness":0.75,"signed_gap":-0.4246687814593315,"absolute_gap":0.4246687814593315,"min_confidence":0.32123133540153503,"max_confidence":0.329334557056427},{"index":8,"count":4,"confidence":0.33534175902605057,"empirical_correctness":0.0,"signed_gap":0.33534175902605057,"absolute_gap":0.33534175902605057,"min_confidence":0.33232632279396057,"max_confidence":0.3368264138698578},{"index":9,"count":4,"confidence":0.3476274460554123,"empirical_correctness":0.75,"signed_gap":-0.4023725539445877,"absolute_gap":0.4023725539445877,"min_confidence":0.3422323167324066,"max_confidence":0.3524456322193146}]}},"latency_ms":{"n":43,"median":18.050161015708,"p95":18.925724952714518},"per_action":{"keep":{"n":12,"raw":{"n":12,"acceptable":1,"unsafe":0,"review":0,"automated_count":12,"automated_acceptable":1,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":0.08333333333333333,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.08333333333333333,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.014865094404917095,"upper":0.35387991114111694,"confidence_level":0.95}},"effective":{"n":12,"acceptable":0,"unsafe":0,"review":12,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":12,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":2.7755575615628914e-17,"upper":0.24249400665524085,"confidence_level":0.95}}},"drop":{"n":16,"raw":{"n":16,"acceptable":16,"unsafe":0,"review":0,"automated_count":16,"automated_acceptable":16,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":1.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.8063923194655637,"upper":1.0,"confidence_level":0.95}},"effective":{"n":16,"acceptable":0,"unsafe":0,"review":16,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":16,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.1936076805344365,"confidence_level":0.95}}},"mask":{"n":2,"raw":{"n":2,"acceptable":0,"unsafe":0,"review":0,"automated_count":2,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.6576197724933469,"confidence_level":0.95}},"effective":{"n":2,"acceptable":0,"unsafe":0,"review":2,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":2,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.6576197724933469,"confidence_level":0.95}}},"pseudonymize":{"n":2,"raw":{"n":2,"acceptable":0,"unsafe":1,"review":0,"automated_count":2,"automated_acceptable":0,"automated_unsafe":1,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":0.5,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":0.5,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.6576197724933469,"confidence_level":0.95}},"effective":{"n":2,"acceptable":0,"unsafe":0,"review":2,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":2,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.6576197724933469,"confidence_level":0.95}}},"generalize":{"n":6,"raw":{"n":6,"acceptable":0,"unsafe":2,"review":0,"automated_count":6,"automated_acceptable":0,"automated_unsafe":2,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":0.3333333333333333,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":0.3333333333333333,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":2.7755575615628914e-17,"upper":0.3903342879021653,"confidence_level":0.95}},"effective":{"n":6,"acceptable":0,"unsafe":0,"review":6,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":6,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":2.7755575615628914e-17,"upper":0.3903342879021653,"confidence_level":0.95}}},"review":{"n":0,"raw":{"n":0,"acceptable":0,"unsafe":0,"review":0,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":null,"unsafe_rate":null,"review_rate":null,"automated_coverage":null,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":null},"effective":{"n":0,"acceptable":0,"unsafe":0,"review":0,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":null,"unsafe_rate":null,"review_rate":null,"automated_coverage":null,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":null}}}},"diagnostic43-compact_no_samples":{"n":43,"raw":{"n":43,"acceptable":23,"unsafe":1,"review":0,"automated_count":43,"automated_acceptable":23,"automated_unsafe":1,"unnecessary_reviews":0,"accuracy":0.5348837209302325,"unsafe_rate":0.023255813953488372,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.5348837209302325,"conditional_unsafe":0.023255813953488372,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.38915640472313967,"upper":0.6748894229587816,"confidence_level":0.95}},"effective":{"n":43,"acceptable":5,"unsafe":0,"review":43,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":38,"accuracy":0.11627906976744186,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":0.8837209302325582,"accuracy_wilson_95":{"lower":0.05070427739804166,"upper":0.24479161810082128,"confidence_level":0.95}},"confusion":{"raw":{"keep":{"keep":0,"drop":11,"mask":0,"pseudonymize":0,"generalize":1,"review":0},"drop":{"keep":0,"drop":16,"mask":0,"pseudonymize":0,"generalize":0,"review":0},"mask":{"keep":0,"drop":2,"mask":0,"pseudonymize":0,"generalize":0,"review":0},"pseudonymize":{"keep":0,"drop":2,"mask":0,"pseudonymize":0,"generalize":0,"review":0},"generalize":{"keep":1,"drop":3,"mask":0,"pseudonymize":0,"generalize":2,"review":0},"review":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0}},"effective":{"keep":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":12},"drop":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":16},"mask":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":2},"pseudonymize":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":2},"generalize":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":6},"review":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0}}},"probability_metrics":{"singleton_n":38,"multiacceptable_n":5,"multiacceptable_coverage":0.11627906976744186,"nll":1.535719659537519,"nll_zero_mass_n":0,"brier":0.7350390525755076,"acceptable_set_n":43,"acceptable_set_nll":1.411745369569043,"acceptable_set_zero_mass_n":0},"calibration_all_labels":{"equal_width":{"n":43,"bin_count":10,"ece":0.24583880845890488,"signed_mean_gap":-0.24583880845890488,"absolute_mean_gap":0.24583880845890488,"bins":[{"index":0,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.0,"upper":0.1,"upper_inclusive":false},{"index":1,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.1,"upper":0.2,"upper_inclusive":false},{"index":2,"count":26,"confidence":0.268553111415643,"empirical_correctness":0.46153846153846156,"signed_gap":-0.1929853501228186,"absolute_gap":0.1929853501228186,"lower":0.2,"upper":0.3,"upper_inclusive":false},{"index":3,"count":17,"confidence":0.3203853140859043,"empirical_correctness":0.6470588235294118,"signed_gap":-0.3266735094435075,"absolute_gap":0.3266735094435075,"lower":0.3,"upper":0.4,"upper_inclusive":false},{"index":4,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.4,"upper":0.5,"upper_inclusive":false},{"index":5,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.5,"upper":0.6,"upper_inclusive":false},{"index":6,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.6,"upper":0.7,"upper_inclusive":false},{"index":7,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.7,"upper":0.8,"upper_inclusive":false},{"index":8,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.8,"upper":0.9,"upper_inclusive":false},{"index":9,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.9,"upper":1.0,"upper_inclusive":true}]},"equal_mass":{"n":43,"bin_count":10,"ece":0.29046863871951434,"signed_mean_gap":-0.24583880845890488,"absolute_mean_gap":0.24583880845890488,"bins":[{"index":0,"count":4,"confidence":0.23179442435503006,"empirical_correctness":0.0,"signed_gap":0.23179442435503006,"absolute_gap":0.23179442435503006,"min_confidence":0.21229687333106995,"max_confidence":0.24606722593307495},{"index":1,"count":4,"confidence":0.25809091329574585,"empirical_correctness":0.25,"signed_gap":0.00809091329574585,"absolute_gap":0.00809091329574585,"min_confidence":0.2539146840572357,"max_confidence":0.2606549561023712},{"index":2,"count":4,"confidence":0.2651595026254654,"empirical_correctness":0.5,"signed_gap":-0.2348404973745346,"absolute_gap":0.2348404973745346,"min_confidence":0.26215967535972595,"max_confidence":0.26998066902160645},{"index":3,"count":5,"confidence":0.2731197953224182,"empirical_correctness":0.6,"signed_gap":-0.3268802046775818,"absolute_gap":0.3268802046775818,"min_confidence":0.27051153779029846,"max_confidence":0.2750050723552704},{"index":4,"count":4,"confidence":0.2803792506456375,"empirical_correctness":0.5,"signed_gap":-0.2196207493543625,"absolute_gap":0.2196207493543625,"min_confidence":0.2780092656612396,"max_confidence":0.28475525975227356},{"index":5,"count":4,"confidence":0.29433073103427887,"empirical_correctness":1.0,"signed_gap":-0.7056692689657211,"absolute_gap":0.7056692689657211,"min_confidence":0.2939377725124359,"max_confidence":0.2946389615535736},{"index":6,"count":5,"confidence":0.30223585963249205,"empirical_correctness":0.4,"signed_gap":-0.09776414036750797,"absolute_gap":0.09776414036750797,"min_confidence":0.2977626323699951,"max_confidence":0.3050328493118286},{"index":7,"count":4,"confidence":0.3091985881328583,"empirical_correctness":0.75,"signed_gap":-0.4408014118671417,"absolute_gap":0.4408014118671417,"min_confidence":0.3060273230075836,"max_confidence":0.3116224706172943},{"index":8,"count":4,"confidence":0.3216993063688278,"empirical_correctness":0.75,"signed_gap":-0.4283006936311722,"absolute_gap":0.4283006936311722,"min_confidence":0.31357869505882263,"max_confidence":0.3283587396144867},{"index":9,"count":5,"confidence":0.3419084191322327,"empirical_correctness":0.6,"signed_gap":-0.2580915808677673,"absolute_gap":0.2580915808677673,"min_confidence":0.32929861545562744,"max_confidence":0.3594549894332886}]}},"calibration_singleton_labels":{"equal_width":{"n":38,"bin_count":10,"ece":0.1849082190739481,"signed_mean_gap":-0.1849082190739481,"absolute_mean_gap":0.1849082190739481,"bins":[{"index":0,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.0,"upper":0.1,"upper_inclusive":false},{"index":1,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.1,"upper":0.2,"upper_inclusive":false},{"index":2,"count":23,"confidence":0.2677737655846969,"empirical_correctness":0.391304347826087,"signed_gap":-0.12353058224139007,"absolute_gap":0.12353058224139007,"lower":0.2,"upper":0.3,"upper_inclusive":false},{"index":3,"count":15,"confidence":0.3209794044494629,"empirical_correctness":0.6,"signed_gap":-0.2790205955505371,"absolute_gap":0.2790205955505371,"lower":0.3,"upper":0.4,"upper_inclusive":false},{"index":4,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.4,"upper":0.5,"upper_inclusive":false},{"index":5,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.5,"upper":0.6,"upper_inclusive":false},{"index":6,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.6,"upper":0.7,"upper_inclusive":false},{"index":7,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.7,"upper":0.8,"upper_inclusive":false},{"index":8,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.8,"upper":0.9,"upper_inclusive":false},{"index":9,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.9,"upper":1.0,"upper_inclusive":true}]},"equal_mass":{"n":38,"bin_count":10,"ece":0.23560117420397306,"signed_mean_gap":-0.1849082190739481,"absolute_mean_gap":0.1849082190739481,"bins":[{"index":0,"count":3,"confidence":0.2270368238290151,"empirical_correctness":0.0,"signed_gap":0.2270368238290151,"absolute_gap":0.2270368238290151,"min_confidence":0.21229687333106995,"max_confidence":0.23776163160800934},{"index":1,"count":4,"confidence":0.2544439807534218,"empirical_correctness":0.25,"signed_gap":0.0044439807534217834,"absolute_gap":0.0044439807534217834,"min_confidence":0.24606722593307495,"max_confidence":0.2595858573913574},{"index":2,"count":4,"confidence":0.2639152780175209,"empirical_correctness":0.25,"signed_gap":0.013915278017520905,"absolute_gap":0.013915278017520905,"min_confidence":0.2606549561023712,"max_confidence":0.26998066902160645},{"index":3,"count":4,"confidence":0.27310045063495636,"empirical_correctness":0.5,"signed_gap":-0.22689954936504364,"absolute_gap":0.22689954936504364,"min_confidence":0.27051153779029846,"max_confidence":0.2750050723552704},{"index":4,"count":4,"confidence":0.2826748788356781,"empirical_correctness":0.5,"signed_gap":-0.2173251211643219,"absolute_gap":0.2173251211643219,"min_confidence":0.2780092656612396,"max_confidence":0.2939377725124359},{"index":5,"count":3,"confidence":0.2944617172082265,"empirical_correctness":1.0,"signed_gap":-0.7055382827917736,"absolute_gap":0.7055382827917736,"min_confidence":0.29422831535339355,"max_confidence":0.2946389615535736},{"index":6,"count":4,"confidence":0.30215466022491455,"empirical_correctness":0.25,"signed_gap":0.05215466022491455,"absolute_gap":0.05215466022491455,"min_confidence":0.2977626323699951,"max_confidence":0.3050328493118286},{"index":7,"count":4,"confidence":0.3091985881328583,"empirical_correctness":0.75,"signed_gap":-0.4408014118671417,"absolute_gap":0.4408014118671417,"min_confidence":0.3060273230075836,"max_confidence":0.3116224706172943},{"index":8,"count":4,"confidence":0.3216993063688278,"empirical_correctness":0.75,"signed_gap":-0.4283006936311722,"absolute_gap":0.4283006936311722,"min_confidence":0.31357869505882263,"max_confidence":0.3283587396144867},{"index":9,"count":4,"confidence":0.34506087005138397,"empirical_correctness":0.5,"signed_gap":-0.15493912994861603,"absolute_gap":0.15493912994861603,"min_confidence":0.3331238031387329,"max_confidence":0.3594549894332886}]}},"latency_ms":{"n":43,"median":20.738496037665755,"p95":21.003482001833618},"per_action":{"keep":{"n":12,"raw":{"n":12,"acceptable":0,"unsafe":0,"review":0,"automated_count":12,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":2.7755575615628914e-17,"upper":0.24249400665524085,"confidence_level":0.95}},"effective":{"n":12,"acceptable":0,"unsafe":0,"review":12,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":12,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":2.7755575615628914e-17,"upper":0.24249400665524085,"confidence_level":0.95}}},"drop":{"n":16,"raw":{"n":16,"acceptable":16,"unsafe":0,"review":0,"automated_count":16,"automated_acceptable":16,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":1.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.8063923194655637,"upper":1.0,"confidence_level":0.95}},"effective":{"n":16,"acceptable":0,"unsafe":0,"review":16,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":16,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.1936076805344365,"confidence_level":0.95}}},"mask":{"n":2,"raw":{"n":2,"acceptable":0,"unsafe":0,"review":0,"automated_count":2,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.6576197724933469,"confidence_level":0.95}},"effective":{"n":2,"acceptable":0,"unsafe":0,"review":2,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":2,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.6576197724933469,"confidence_level":0.95}}},"pseudonymize":{"n":2,"raw":{"n":2,"acceptable":0,"unsafe":0,"review":0,"automated_count":2,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.6576197724933469,"confidence_level":0.95}},"effective":{"n":2,"acceptable":0,"unsafe":0,"review":2,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":2,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.6576197724933469,"confidence_level":0.95}}},"generalize":{"n":6,"raw":{"n":6,"acceptable":2,"unsafe":1,"review":0,"automated_count":6,"automated_acceptable":2,"automated_unsafe":1,"unnecessary_reviews":0,"accuracy":0.3333333333333333,"unsafe_rate":0.16666666666666666,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.3333333333333333,"conditional_unsafe":0.16666666666666666,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.09677141110578041,"upper":0.700006684861608,"confidence_level":0.95}},"effective":{"n":6,"acceptable":0,"unsafe":0,"review":6,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":6,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":2.7755575615628914e-17,"upper":0.3903342879021653,"confidence_level":0.95}}},"review":{"n":0,"raw":{"n":0,"acceptable":0,"unsafe":0,"review":0,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":null,"unsafe_rate":null,"review_rate":null,"automated_coverage":null,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":null},"effective":{"n":0,"acceptable":0,"unsafe":0,"review":0,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":null,"unsafe_rate":null,"review_rate":null,"automated_coverage":null,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":null}}}},"challenge56-no_samples":{"n":56,"raw":{"n":56,"acceptable":43,"unsafe":0,"review":0,"automated_count":56,"automated_acceptable":43,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":0.7678571428571429,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.7678571428571429,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.6423184678498916,"upper":0.8590062089079139,"confidence_level":0.95}},"effective":{"n":56,"acceptable":3,"unsafe":0,"review":55,"automated_count":1,"automated_acceptable":1,"automated_unsafe":0,"unnecessary_reviews":53,"accuracy":0.05357142857142857,"unsafe_rate":0.0,"review_rate":0.9821428571428571,"automated_coverage":0.017857142857142856,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.9464285714285714,"accuracy_wilson_95":{"lower":0.018385781382109126,"upper":0.14607309068821533,"confidence_level":0.95}},"confusion":{"raw":{"keep":{"keep":7,"drop":1,"mask":0,"pseudonymize":0,"generalize":0,"review":0},"drop":{"keep":0,"drop":20,"mask":1,"pseudonymize":0,"generalize":1,"review":0},"mask":{"keep":0,"drop":1,"mask":7,"pseudonymize":0,"generalize":0,"review":0},"pseudonymize":{"keep":0,"drop":0,"mask":6,"pseudonymize":1,"generalize":1,"review":0},"generalize":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":8,"review":0},"review":{"keep":0,"drop":0,"mask":2,"pseudonymize":0,"generalize":0,"review":0}},"effective":{"keep":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":8},"drop":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":22},"mask":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":8},"pseudonymize":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":8},"generalize":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":1,"review":7},"review":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":2}}},"probability_metrics":{"singleton_n":56,"multiacceptable_n":0,"multiacceptable_coverage":0.0,"nll":1.2647866313444809,"nll_zero_mass_n":0,"brier":0.6180303099602981,"acceptable_set_n":56,"acceptable_set_nll":1.2647866313444809,"acceptable_set_zero_mass_n":0},"calibration_all_labels":{"equal_width":{"n":56,"bin_count":10,"ece":0.4609747278903212,"signed_mean_gap":-0.4609747278903212,"absolute_mean_gap":0.4609747278903212,"bins":[{"index":0,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.0,"upper":0.1,"upper_inclusive":false},{"index":1,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.1,"upper":0.2,"upper_inclusive":false},{"index":2,"count":27,"confidence":0.2580291209397493,"empirical_correctness":0.5185185185185185,"signed_gap":-0.2604893975787692,"absolute_gap":0.2604893975787692,"lower":0.2,"upper":0.3,"upper_inclusive":false},{"index":3,"count":25,"confidence":0.33872578740119935,"empirical_correctness":1.0,"signed_gap":-0.6612742125988007,"absolute_gap":0.6612742125988007,"lower":0.3,"upper":0.4,"upper_inclusive":false},{"index":4,"count":4,"confidence":0.4376210719347,"empirical_correctness":1.0,"signed_gap":-0.5623789280653,"absolute_gap":0.5623789280653,"lower":0.4,"upper":0.5,"upper_inclusive":false},{"index":5,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.5,"upper":0.6,"upper_inclusive":false},{"index":6,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.6,"upper":0.7,"upper_inclusive":false},{"index":7,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.7,"upper":0.8,"upper_inclusive":false},{"index":8,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.8,"upper":0.9,"upper_inclusive":false},{"index":9,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.9,"upper":1.0,"upper_inclusive":true}]},"equal_mass":{"n":56,"bin_count":10,"ece":0.4609747278903212,"signed_mean_gap":-0.4609747278903212,"absolute_mean_gap":0.4609747278903212,"bins":[{"index":0,"count":5,"confidence":0.21582127809524537,"empirical_correctness":0.6,"signed_gap":-0.3841787219047546,"absolute_gap":0.3841787219047546,"min_confidence":0.20075547695159912,"max_confidence":0.22461426258087158},{"index":1,"count":6,"confidence":0.2396601065993309,"empirical_correctness":0.3333333333333333,"signed_gap":-0.09367322673400241,"absolute_gap":0.09367322673400241,"min_confidence":0.23640622198581696,"max_confidence":0.24642299115657806},{"index":2,"count":5,"confidence":0.260555037856102,"empirical_correctness":0.4,"signed_gap":-0.139444962143898,"absolute_gap":0.139444962143898,"min_confidence":0.24839378893375397,"max_confidence":0.27047818899154663},{"index":3,"count":6,"confidence":0.2783941626548767,"empirical_correctness":0.6666666666666666,"signed_gap":-0.3882725040117899,"absolute_gap":0.3882725040117899,"min_confidence":0.2724197804927826,"max_confidence":0.28462234139442444},{"index":4,"count":6,"confidence":0.29663697878519696,"empirical_correctness":0.6666666666666666,"signed_gap":-0.37002968788146967,"absolute_gap":0.37002968788146967,"min_confidence":0.2908439040184021,"max_confidence":0.3032428026199341},{"index":5,"count":5,"confidence":0.3082711100578308,"empirical_correctness":1.0,"signed_gap":-0.6917288899421692,"absolute_gap":0.6917288899421692,"min_confidence":0.3062991499900818,"max_confidence":0.3111061453819275},{"index":6,"count":6,"confidence":0.32140682140986127,"empirical_correctness":1.0,"signed_gap":-0.6785931785901387,"absolute_gap":0.6785931785901387,"min_confidence":0.31252601742744446,"max_confidence":0.33274129033088684},{"index":7,"count":5,"confidence":0.3442975401878357,"empirical_correctness":1.0,"signed_gap":-0.6557024598121644,"absolute_gap":0.6557024598121644,"min_confidence":0.33278849720954895,"max_confidence":0.35484248399734497},{"index":8,"count":6,"confidence":0.36601423223813373,"empirical_correctness":1.0,"signed_gap":-0.6339857677618663,"absolute_gap":0.6339857677618663,"min_confidence":0.35544002056121826,"max_confidence":0.3727021813392639},{"index":9,"count":6,"confidence":0.4213360995054245,"empirical_correctness":1.0,"signed_gap":-0.5786639004945755,"absolute_gap":0.5786639004945755,"min_confidence":0.38681361079216003,"max_confidence":0.4720378816127777}]}},"calibration_singleton_labels":{"equal_width":{"n":56,"bin_count":10,"ece":0.4609747278903212,"signed_mean_gap":-0.4609747278903212,"absolute_mean_gap":0.4609747278903212,"bins":[{"index":0,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.0,"upper":0.1,"upper_inclusive":false},{"index":1,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.1,"upper":0.2,"upper_inclusive":false},{"index":2,"count":27,"confidence":0.2580291209397493,"empirical_correctness":0.5185185185185185,"signed_gap":-0.2604893975787692,"absolute_gap":0.2604893975787692,"lower":0.2,"upper":0.3,"upper_inclusive":false},{"index":3,"count":25,"confidence":0.33872578740119935,"empirical_correctness":1.0,"signed_gap":-0.6612742125988007,"absolute_gap":0.6612742125988007,"lower":0.3,"upper":0.4,"upper_inclusive":false},{"index":4,"count":4,"confidence":0.4376210719347,"empirical_correctness":1.0,"signed_gap":-0.5623789280653,"absolute_gap":0.5623789280653,"lower":0.4,"upper":0.5,"upper_inclusive":false},{"index":5,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.5,"upper":0.6,"upper_inclusive":false},{"index":6,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.6,"upper":0.7,"upper_inclusive":false},{"index":7,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.7,"upper":0.8,"upper_inclusive":false},{"index":8,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.8,"upper":0.9,"upper_inclusive":false},{"index":9,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.9,"upper":1.0,"upper_inclusive":true}]},"equal_mass":{"n":56,"bin_count":10,"ece":0.4609747278903212,"signed_mean_gap":-0.4609747278903212,"absolute_mean_gap":0.4609747278903212,"bins":[{"index":0,"count":5,"confidence":0.21582127809524537,"empirical_correctness":0.6,"signed_gap":-0.3841787219047546,"absolute_gap":0.3841787219047546,"min_confidence":0.20075547695159912,"max_confidence":0.22461426258087158},{"index":1,"count":6,"confidence":0.2396601065993309,"empirical_correctness":0.3333333333333333,"signed_gap":-0.09367322673400241,"absolute_gap":0.09367322673400241,"min_confidence":0.23640622198581696,"max_confidence":0.24642299115657806},{"index":2,"count":5,"confidence":0.260555037856102,"empirical_correctness":0.4,"signed_gap":-0.139444962143898,"absolute_gap":0.139444962143898,"min_confidence":0.24839378893375397,"max_confidence":0.27047818899154663},{"index":3,"count":6,"confidence":0.2783941626548767,"empirical_correctness":0.6666666666666666,"signed_gap":-0.3882725040117899,"absolute_gap":0.3882725040117899,"min_confidence":0.2724197804927826,"max_confidence":0.28462234139442444},{"index":4,"count":6,"confidence":0.29663697878519696,"empirical_correctness":0.6666666666666666,"signed_gap":-0.37002968788146967,"absolute_gap":0.37002968788146967,"min_confidence":0.2908439040184021,"max_confidence":0.3032428026199341},{"index":5,"count":5,"confidence":0.3082711100578308,"empirical_correctness":1.0,"signed_gap":-0.6917288899421692,"absolute_gap":0.6917288899421692,"min_confidence":0.3062991499900818,"max_confidence":0.3111061453819275},{"index":6,"count":6,"confidence":0.32140682140986127,"empirical_correctness":1.0,"signed_gap":-0.6785931785901387,"absolute_gap":0.6785931785901387,"min_confidence":0.31252601742744446,"max_confidence":0.33274129033088684},{"index":7,"count":5,"confidence":0.3442975401878357,"empirical_correctness":1.0,"signed_gap":-0.6557024598121644,"absolute_gap":0.6557024598121644,"min_confidence":0.33278849720954895,"max_confidence":0.35484248399734497},{"index":8,"count":6,"confidence":0.36601423223813373,"empirical_correctness":1.0,"signed_gap":-0.6339857677618663,"absolute_gap":0.6339857677618663,"min_confidence":0.35544002056121826,"max_confidence":0.3727021813392639},{"index":9,"count":6,"confidence":0.4213360995054245,"empirical_correctness":1.0,"signed_gap":-0.5786639004945755,"absolute_gap":0.5786639004945755,"min_confidence":0.38681361079216003,"max_confidence":0.4720378816127777}]}},"latency_ms":{"n":56,"median":15.463685005670413,"p95":15.802814246853814},"per_action":{"keep":{"n":8,"raw":{"n":8,"acceptable":7,"unsafe":0,"review":0,"automated_count":8,"automated_acceptable":7,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":0.875,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.875,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.5291118177871464,"upper":0.9775825085499433,"confidence_level":0.95}},"effective":{"n":8,"acceptable":0,"unsafe":0,"review":8,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":8,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.32440756488388023,"confidence_level":0.95}}},"drop":{"n":22,"raw":{"n":22,"acceptable":20,"unsafe":0,"review":0,"automated_count":22,"automated_acceptable":20,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":0.9090909090909091,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.9090909090909091,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.7218505315839417,"upper":0.9747045710090046,"confidence_level":0.95}},"effective":{"n":22,"acceptable":0,"unsafe":0,"review":22,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":22,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.14865487460862087,"confidence_level":0.95}}},"mask":{"n":8,"raw":{"n":8,"acceptable":7,"unsafe":0,"review":0,"automated_count":8,"automated_acceptable":7,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":0.875,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.875,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.5291118177871464,"upper":0.9775825085499433,"confidence_level":0.95}},"effective":{"n":8,"acceptable":0,"unsafe":0,"review":8,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":8,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.32440756488388023,"confidence_level":0.95}}},"pseudonymize":{"n":8,"raw":{"n":8,"acceptable":1,"unsafe":0,"review":0,"automated_count":8,"automated_acceptable":1,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":0.125,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.125,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.02241749145005667,"upper":0.4708881822128535,"confidence_level":0.95}},"effective":{"n":8,"acceptable":0,"unsafe":0,"review":8,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":8,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.32440756488388023,"confidence_level":0.95}}},"generalize":{"n":8,"raw":{"n":8,"acceptable":8,"unsafe":0,"review":0,"automated_count":8,"automated_acceptable":8,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":1.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.6755924351161198,"upper":1.0,"confidence_level":0.95}},"effective":{"n":8,"acceptable":1,"unsafe":0,"review":7,"automated_count":1,"automated_acceptable":1,"automated_unsafe":0,"unnecessary_reviews":7,"accuracy":0.125,"unsafe_rate":0.0,"review_rate":0.875,"automated_coverage":0.125,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.875,"accuracy_wilson_95":{"lower":0.02241749145005667,"upper":0.4708881822128535,"confidence_level":0.95}}},"review":{"n":2,"raw":{"n":2,"acceptable":0,"unsafe":0,"review":0,"automated_count":2,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.6576197724933469,"confidence_level":0.95}},"effective":{"n":2,"acceptable":2,"unsafe":0,"review":2,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":1.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.34238022750665303,"upper":1.0,"confidence_level":0.95}}}}},"challenge56-compact_no_samples":{"n":56,"raw":{"n":56,"acceptable":43,"unsafe":1,"review":0,"automated_count":56,"automated_acceptable":43,"automated_unsafe":1,"unnecessary_reviews":0,"accuracy":0.7678571428571429,"unsafe_rate":0.017857142857142856,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.7678571428571429,"conditional_unsafe":0.017857142857142856,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.6423184678498916,"upper":0.8590062089079139,"confidence_level":0.95}},"effective":{"n":56,"acceptable":4,"unsafe":0,"review":54,"automated_count":2,"automated_acceptable":2,"automated_unsafe":0,"unnecessary_reviews":52,"accuracy":0.07142857142857142,"unsafe_rate":0.0,"review_rate":0.9642857142857143,"automated_coverage":0.03571428571428571,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.9285714285714286,"accuracy_wilson_95":{"lower":0.028126036578876953,"upper":0.16975448060863452,"confidence_level":0.95}},"confusion":{"raw":{"keep":{"keep":6,"drop":1,"mask":0,"pseudonymize":0,"generalize":1,"review":0},"drop":{"keep":0,"drop":22,"mask":0,"pseudonymize":0,"generalize":0,"review":0},"mask":{"keep":0,"drop":2,"mask":6,"pseudonymize":0,"generalize":0,"review":0},"pseudonymize":{"keep":0,"drop":2,"mask":4,"pseudonymize":2,"generalize":0,"review":0},"generalize":{"keep":1,"drop":0,"mask":0,"pseudonymize":0,"generalize":7,"review":0},"review":{"keep":0,"drop":0,"mask":2,"pseudonymize":0,"generalize":0,"review":0}},"effective":{"keep":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":8},"drop":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":22},"mask":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":8},"pseudonymize":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":8},"generalize":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":2,"review":6},"review":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":2}}},"probability_metrics":{"singleton_n":56,"multiacceptable_n":0,"multiacceptable_coverage":0.0,"nll":1.2904384737487913,"nll_zero_mass_n":0,"brier":0.6278078318962453,"acceptable_set_n":56,"acceptable_set_nll":1.2904384737487913,"acceptable_set_zero_mass_n":0},"calibration_all_labels":{"equal_width":{"n":56,"bin_count":10,"ece":0.47439716357205597,"signed_mean_gap":-0.47439716357205597,"absolute_mean_gap":0.47439716357205597,"bins":[{"index":0,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.0,"upper":0.1,"upper_inclusive":false},{"index":1,"count":2,"confidence":0.19732321053743362,"empirical_correctness":0.5,"signed_gap":-0.3026767894625664,"absolute_gap":0.3026767894625664,"lower":0.1,"upper":0.2,"upper_inclusive":false},{"index":2,"count":34,"confidence":0.24638032299630783,"empirical_correctness":0.6470588235294118,"signed_gap":-0.40067850053310394,"absolute_gap":0.40067850053310394,"lower":0.2,"upper":0.3,"upper_inclusive":false},{"index":3,"count":15,"confidence":0.36612948576609294,"empirical_correctness":1.0,"signed_gap":-0.6338705142339071,"absolute_gap":0.6338705142339071,"lower":0.3,"upper":0.4,"upper_inclusive":false},{"index":4,"count":5,"confidence":0.4340478301048279,"empirical_correctness":1.0,"signed_gap":-0.5659521698951722,"absolute_gap":0.5659521698951722,"lower":0.4,"upper":0.5,"upper_inclusive":false},{"index":5,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.5,"upper":0.6,"upper_inclusive":false},{"index":6,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.6,"upper":0.7,"upper_inclusive":false},{"index":7,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.7,"upper":0.8,"upper_inclusive":false},{"index":8,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.8,"upper":0.9,"upper_inclusive":false},{"index":9,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.9,"upper":1.0,"upper_inclusive":true}]},"equal_mass":{"n":56,"bin_count":10,"ece":0.48590342035251005,"signed_mean_gap":-0.47439716357205597,"absolute_mean_gap":0.47439716357205597,"bins":[{"index":0,"count":5,"confidence":0.20233293771743774,"empirical_correctness":0.4,"signed_gap":-0.19766706228256228,"absolute_gap":0.19766706228256228,"min_confidence":0.19719842076301575,"max_confidence":0.2096993625164032},{"index":1,"count":6,"confidence":0.22036253164211908,"empirical_correctness":0.16666666666666666,"signed_gap":0.05369586497545242,"absolute_gap":0.05369586497545242,"min_confidence":0.2120005190372467,"max_confidence":0.22741134464740753},{"index":2,"count":5,"confidence":0.23134163916110992,"empirical_correctness":0.8,"signed_gap":-0.5686583608388901,"absolute_gap":0.5686583608388901,"min_confidence":0.22871628403663635,"max_confidence":0.2344994992017746},{"index":3,"count":6,"confidence":0.24266398698091507,"empirical_correctness":0.6666666666666666,"signed_gap":-0.42400267968575156,"absolute_gap":0.42400267968575156,"min_confidence":0.23690302670001984,"max_confidence":0.24880653619766235},{"index":4,"count":6,"confidence":0.25963208576043445,"empirical_correctness":0.8333333333333334,"signed_gap":-0.5737012475728989,"absolute_gap":0.5737012475728989,"min_confidence":0.2515515089035034,"max_confidence":0.274541437625885},{"index":5,"count":5,"confidence":0.27761725783348085,"empirical_correctness":0.8,"signed_gap":-0.5223827421665193,"absolute_gap":0.5223827421665193,"min_confidence":0.27524226903915405,"max_confidence":0.2810872793197632},{"index":6,"count":6,"confidence":0.31142565111319226,"empirical_correctness":1.0,"signed_gap":-0.6885743488868077,"absolute_gap":0.6885743488868077,"min_confidence":0.2896358072757721,"max_confidence":0.3429791331291199},{"index":7,"count":5,"confidence":0.3598709762096405,"empirical_correctness":1.0,"signed_gap":-0.6401290237903595,"absolute_gap":0.6401290237903595,"min_confidence":0.3478395342826843,"max_confidence":0.37467172741889954},{"index":8,"count":6,"confidence":0.3847547521193822,"empirical_correctness":1.0,"signed_gap":-0.6152452478806179,"absolute_gap":0.6152452478806179,"min_confidence":0.37553149461746216,"max_confidence":0.3932461738586426},{"index":9,"count":6,"confidence":0.4274851232767105,"empirical_correctness":1.0,"signed_gap":-0.5725148767232895,"absolute_gap":0.5725148767232895,"min_confidence":0.39467158913612366,"max_confidence":0.4837418496608734}]}},"calibration_singleton_labels":{"equal_width":{"n":56,"bin_count":10,"ece":0.47439716357205597,"signed_mean_gap":-0.47439716357205597,"absolute_mean_gap":0.47439716357205597,"bins":[{"index":0,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.0,"upper":0.1,"upper_inclusive":false},{"index":1,"count":2,"confidence":0.19732321053743362,"empirical_correctness":0.5,"signed_gap":-0.3026767894625664,"absolute_gap":0.3026767894625664,"lower":0.1,"upper":0.2,"upper_inclusive":false},{"index":2,"count":34,"confidence":0.24638032299630783,"empirical_correctness":0.6470588235294118,"signed_gap":-0.40067850053310394,"absolute_gap":0.40067850053310394,"lower":0.2,"upper":0.3,"upper_inclusive":false},{"index":3,"count":15,"confidence":0.36612948576609294,"empirical_correctness":1.0,"signed_gap":-0.6338705142339071,"absolute_gap":0.6338705142339071,"lower":0.3,"upper":0.4,"upper_inclusive":false},{"index":4,"count":5,"confidence":0.4340478301048279,"empirical_correctness":1.0,"signed_gap":-0.5659521698951722,"absolute_gap":0.5659521698951722,"lower":0.4,"upper":0.5,"upper_inclusive":false},{"index":5,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.5,"upper":0.6,"upper_inclusive":false},{"index":6,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.6,"upper":0.7,"upper_inclusive":false},{"index":7,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.7,"upper":0.8,"upper_inclusive":false},{"index":8,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.8,"upper":0.9,"upper_inclusive":false},{"index":9,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.9,"upper":1.0,"upper_inclusive":true}]},"equal_mass":{"n":56,"bin_count":10,"ece":0.48590342035251005,"signed_mean_gap":-0.47439716357205597,"absolute_mean_gap":0.47439716357205597,"bins":[{"index":0,"count":5,"confidence":0.20233293771743774,"empirical_correctness":0.4,"signed_gap":-0.19766706228256228,"absolute_gap":0.19766706228256228,"min_confidence":0.19719842076301575,"max_confidence":0.2096993625164032},{"index":1,"count":6,"confidence":0.22036253164211908,"empirical_correctness":0.16666666666666666,"signed_gap":0.05369586497545242,"absolute_gap":0.05369586497545242,"min_confidence":0.2120005190372467,"max_confidence":0.22741134464740753},{"index":2,"count":5,"confidence":0.23134163916110992,"empirical_correctness":0.8,"signed_gap":-0.5686583608388901,"absolute_gap":0.5686583608388901,"min_confidence":0.22871628403663635,"max_confidence":0.2344994992017746},{"index":3,"count":6,"confidence":0.24266398698091507,"empirical_correctness":0.6666666666666666,"signed_gap":-0.42400267968575156,"absolute_gap":0.42400267968575156,"min_confidence":0.23690302670001984,"max_confidence":0.24880653619766235},{"index":4,"count":6,"confidence":0.25963208576043445,"empirical_correctness":0.8333333333333334,"signed_gap":-0.5737012475728989,"absolute_gap":0.5737012475728989,"min_confidence":0.2515515089035034,"max_confidence":0.274541437625885},{"index":5,"count":5,"confidence":0.27761725783348085,"empirical_correctness":0.8,"signed_gap":-0.5223827421665193,"absolute_gap":0.5223827421665193,"min_confidence":0.27524226903915405,"max_confidence":0.2810872793197632},{"index":6,"count":6,"confidence":0.31142565111319226,"empirical_correctness":1.0,"signed_gap":-0.6885743488868077,"absolute_gap":0.6885743488868077,"min_confidence":0.2896358072757721,"max_confidence":0.3429791331291199},{"index":7,"count":5,"confidence":0.3598709762096405,"empirical_correctness":1.0,"signed_gap":-0.6401290237903595,"absolute_gap":0.6401290237903595,"min_confidence":0.3478395342826843,"max_confidence":0.37467172741889954},{"index":8,"count":6,"confidence":0.3847547521193822,"empirical_correctness":1.0,"signed_gap":-0.6152452478806179,"absolute_gap":0.6152452478806179,"min_confidence":0.37553149461746216,"max_confidence":0.3932461738586426},{"index":9,"count":6,"confidence":0.4274851232767105,"empirical_correctness":1.0,"signed_gap":-0.5725148767232895,"absolute_gap":0.5725148767232895,"min_confidence":0.39467158913612366,"max_confidence":0.4837418496608734}]}},"latency_ms":{"n":56,"median":18.01739097572863,"p95":18.355175212491304},"per_action":{"keep":{"n":8,"raw":{"n":8,"acceptable":6,"unsafe":0,"review":0,"automated_count":8,"automated_acceptable":6,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":0.75,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.75,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.40927543031016883,"upper":0.9285207872478909,"confidence_level":0.95}},"effective":{"n":8,"acceptable":0,"unsafe":0,"review":8,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":8,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.32440756488388023,"confidence_level":0.95}}},"drop":{"n":22,"raw":{"n":22,"acceptable":22,"unsafe":0,"review":0,"automated_count":22,"automated_acceptable":22,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":1.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.8513451253913791,"upper":1.0,"confidence_level":0.95}},"effective":{"n":22,"acceptable":0,"unsafe":0,"review":22,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":22,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.14865487460862087,"confidence_level":0.95}}},"mask":{"n":8,"raw":{"n":8,"acceptable":6,"unsafe":0,"review":0,"automated_count":8,"automated_acceptable":6,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":0.75,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.75,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.40927543031016883,"upper":0.9285207872478909,"confidence_level":0.95}},"effective":{"n":8,"acceptable":0,"unsafe":0,"review":8,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":8,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.32440756488388023,"confidence_level":0.95}}},"pseudonymize":{"n":8,"raw":{"n":8,"acceptable":2,"unsafe":0,"review":0,"automated_count":8,"automated_acceptable":2,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":0.25,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.25,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.071479212752109,"upper":0.5907245696898311,"confidence_level":0.95}},"effective":{"n":8,"acceptable":0,"unsafe":0,"review":8,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":8,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.32440756488388023,"confidence_level":0.95}}},"generalize":{"n":8,"raw":{"n":8,"acceptable":7,"unsafe":1,"review":0,"automated_count":8,"automated_acceptable":7,"automated_unsafe":1,"unnecessary_reviews":0,"accuracy":0.875,"unsafe_rate":0.125,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.875,"conditional_unsafe":0.125,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.5291118177871464,"upper":0.9775825085499433,"confidence_level":0.95}},"effective":{"n":8,"acceptable":2,"unsafe":0,"review":6,"automated_count":2,"automated_acceptable":2,"automated_unsafe":0,"unnecessary_reviews":6,"accuracy":0.25,"unsafe_rate":0.0,"review_rate":0.75,"automated_coverage":0.25,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.75,"accuracy_wilson_95":{"lower":0.071479212752109,"upper":0.5907245696898311,"confidence_level":0.95}}},"review":{"n":2,"raw":{"n":2,"acceptable":0,"unsafe":0,"review":0,"automated_count":2,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.6576197724933469,"confidence_level":0.95}},"effective":{"n":2,"acceptable":2,"unsafe":0,"review":2,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":1.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.34238022750665303,"upper":1.0,"confidence_level":0.95}}}}}},"composite":{"n":198,"scope":"descriptive sum across four correlated panels","raw":{"n":198,"acceptable":130,"unsafe":6,"review":0,"automated_count":198,"automated_acceptable":130,"automated_unsafe":6,"unnecessary_reviews":0,"accuracy":0.6565656565656566,"unsafe_rate":0.030303030303030304,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.6565656565656566,"conditional_unsafe":0.030303030303030304,"unnecessary_review_rate":0.0},"effective":{"n":198,"acceptable":17,"unsafe":0,"review":195,"automated_count":3,"automated_acceptable":3,"automated_unsafe":0,"unnecessary_reviews":181,"accuracy":0.08585858585858586,"unsafe_rate":0.0,"review_rate":0.9848484848484849,"automated_coverage":0.015151515151515152,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.9141414141414141}}},{"id":"bofenghuang--docto-decision-modernbert-fr-base-v0.1-fp32","platform":"Local","model_id":"bofenghuang/docto-decision-modernbert-fr-base-v0.1","revision":"003beea2c0026cd54020541106b825e7efb7221e","loading":"float32 / none","scoring_kind":"Native decision readout","settings":{"dtype":"float32","quantization":"none","max_tokens":8192,"temperature":1.0,"autocast":false,"truncation":false,"cpu_offload":false,"guard_threshold":0.45,"seed":0,"tf32":false},"scoring_metadata":{"backend":"docto_native_local_builtin","readout":"per_slot_saved_MLM_logit_1_minus_logit_0_then_softmax","probability_source":"native_per_slot_saved_MLM_logit_1_minus_0_softmax_T1_not_task_calibrated","latency_scope":"synchronized serial complete builtin MLM forward and input transfer/native readout; tokenization excluded","native_disclosures":["French-trained model, unchanged English frozen inputs.","Publisher BF16 example differs from explicit exactFP32/noautocast screening condition.","Full saved MLM head loaded, tied decoder uses saved trained input embedding; no new head.","Question name action is fixed adapter mapping, not a source job field.","Native direct softmax T1; no temperature fitting or translation."],"equal_mass_convention":"rank bins with floor boundaries at floor(i*n/k), k=min(10,n//2); minimum two rows"},"load_ms":null,"peak_allocated_bytes":773829632,"peak_reserved_bytes":1054867456,"panels":{"diagnostic43-no_samples":{"n":43,"raw":{"n":43,"acceptable":18,"unsafe":0,"review":5,"automated_count":38,"automated_acceptable":18,"automated_unsafe":0,"unnecessary_reviews":5,"accuracy":0.4186046511627907,"unsafe_rate":0.0,"review_rate":0.11627906976744186,"automated_coverage":0.8837209302325582,"conditional_correctness":0.47368421052631576,"conditional_unsafe":0.0,"unnecessary_review_rate":0.11627906976744186,"accuracy_wilson_95":{"lower":0.28384550927849195,"upper":0.5667142261303578,"confidence_level":0.95}},"effective":{"n":43,"acceptable":9,"unsafe":0,"review":35,"automated_count":8,"automated_acceptable":5,"automated_unsafe":0,"unnecessary_reviews":31,"accuracy":0.20930232558139536,"unsafe_rate":0.0,"review_rate":0.813953488372093,"automated_coverage":0.18604651162790697,"conditional_correctness":0.625,"conditional_unsafe":0.0,"unnecessary_review_rate":0.7209302325581395,"accuracy_wilson_95":{"lower":0.11422834220995602,"upper":0.3520564271073644,"confidence_level":0.95}},"confusion":{"raw":{"keep":{"keep":0,"drop":11,"mask":0,"pseudonymize":0,"generalize":0,"review":1},"drop":{"keep":0,"drop":13,"mask":0,"pseudonymize":0,"generalize":0,"review":3},"mask":{"keep":0,"drop":1,"mask":0,"pseudonymize":0,"generalize":0,"review":1},"pseudonymize":{"keep":0,"drop":2,"mask":0,"pseudonymize":0,"generalize":0,"review":0},"generalize":{"keep":0,"drop":6,"mask":0,"pseudonymize":0,"generalize":0,"review":0},"review":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0}},"effective":{"keep":{"keep":0,"drop":2,"mask":0,"pseudonymize":0,"generalize":0,"review":10},"drop":{"keep":0,"drop":4,"mask":0,"pseudonymize":0,"generalize":0,"review":12},"mask":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":2},"pseudonymize":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":2},"generalize":{"keep":0,"drop":1,"mask":0,"pseudonymize":0,"generalize":0,"review":5},"review":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0}}},"probability_metrics":{"singleton_n":38,"multiacceptable_n":5,"multiacceptable_coverage":0.11627906976744186,"nll":1.8960954353416635,"nll_zero_mass_n":0,"brier":0.8298862019718337,"acceptable_set_n":43,"acceptable_set_nll":1.7038919543788482,"acceptable_set_zero_mass_n":0},"calibration_all_labels":{"equal_width":{"n":43,"bin_count":10,"ece":0.07336842043455258,"signed_mean_gap":-0.022150139475977666,"absolute_mean_gap":0.022150139475977666,"bins":[{"index":0,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.0,"upper":0.1,"upper_inclusive":false},{"index":1,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.1,"upper":0.2,"upper_inclusive":false},{"index":2,"count":7,"confidence":0.2677707714693887,"empirical_correctness":0.42857142857142855,"signed_gap":-0.16080065710203983,"absolute_gap":0.16080065710203983,"lower":0.2,"upper":0.3,"upper_inclusive":false},{"index":3,"count":19,"confidence":0.37374700213733475,"empirical_correctness":0.3157894736842105,"signed_gap":0.05795752845312424,"absolute_gap":0.05795752845312424,"lower":0.3,"upper":0.4,"upper_inclusive":false},{"index":4,"count":9,"confidence":0.416867991288503,"empirical_correctness":0.4444444444444444,"signed_gap":-0.027576453155941427,"absolute_gap":0.027576453155941427,"lower":0.4,"upper":0.5,"upper_inclusive":false},{"index":5,"count":8,"confidence":0.5400179550051689,"empirical_correctness":0.625,"signed_gap":-0.08498204499483109,"absolute_gap":0.08498204499483109,"lower":0.5,"upper":0.6,"upper_inclusive":false},{"index":6,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.6,"upper":0.7,"upper_inclusive":false},{"index":7,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.7,"upper":0.8,"upper_inclusive":false},{"index":8,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.8,"upper":0.9,"upper_inclusive":false},{"index":9,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.9,"upper":1.0,"upper_inclusive":true}]},"equal_mass":{"n":43,"bin_count":10,"ece":0.1556309420008992,"signed_mean_gap":-0.0221501394759777,"absolute_mean_gap":0.0221501394759777,"bins":[{"index":0,"count":4,"confidence":0.26252933591604233,"empirical_correctness":0.5,"signed_gap":-0.23747066408395767,"absolute_gap":0.23747066408395767,"min_confidence":0.255749374628067,"max_confidence":0.2696058750152588},{"index":1,"count":4,"confidence":0.2914260923862457,"empirical_correctness":0.5,"signed_gap":-0.20857390761375427,"absolute_gap":0.20857390761375427,"min_confidence":0.27264732122421265,"max_confidence":0.3414263129234314},{"index":2,"count":4,"confidence":0.3484843745827675,"empirical_correctness":0.25,"signed_gap":0.09848437458276749,"absolute_gap":0.09848437458276749,"min_confidence":0.34259235858917236,"max_confidence":0.3529776930809021},{"index":3,"count":5,"confidence":0.3717449188232422,"empirical_correctness":0.2,"signed_gap":0.17174491882324217,"absolute_gap":0.17174491882324217,"min_confidence":0.3557615876197815,"max_confidence":0.3813125193119049},{"index":4,"count":4,"confidence":0.3863347917795181,"empirical_correctness":0.5,"signed_gap":-0.11366520822048187,"absolute_gap":0.11366520822048187,"min_confidence":0.38386663794517517,"max_confidence":0.3892935514450073},{"index":5,"count":4,"confidence":0.39130450785160065,"empirical_correctness":0.25,"signed_gap":0.14130450785160065,"absolute_gap":0.14130450785160065,"min_confidence":0.3897477686405182,"max_confidence":0.3944871723651886},{"index":6,"count":5,"confidence":0.40730844140052797,"empirical_correctness":0.2,"signed_gap":0.20730844140052795,"absolute_gap":0.20730844140052795,"min_confidence":0.39654743671417236,"max_confidence":0.41759636998176575},{"index":7,"count":4,"confidence":0.4207228869199753,"empirical_correctness":0.5,"signed_gap":-0.07927711308002472,"absolute_gap":0.07927711308002472,"min_confidence":0.4183422029018402,"max_confidence":0.4234970808029175},{"index":8,"count":4,"confidence":0.5038537308573723,"empirical_correctness":0.5,"signed_gap":0.003853730857372284,"absolute_gap":0.003853730857372284,"min_confidence":0.42892560362815857,"max_confidence":0.5331385135650635},{"index":9,"count":5,"confidence":0.5467308640480042,"empirical_correctness":0.8,"signed_gap":-0.25326913595199585,"absolute_gap":0.25326913595199585,"min_confidence":0.5394099354743958,"max_confidence":0.5562885403633118}]}},"calibration_singleton_labels":{"equal_width":{"n":38,"bin_count":10,"ece":0.10052809825069026,"signed_mean_gap":0.05546277996740843,"absolute_mean_gap":0.05546277996740843,"bins":[{"index":0,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.0,"upper":0.1,"upper_inclusive":false},{"index":1,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.1,"upper":0.2,"upper_inclusive":false},{"index":2,"count":6,"confidence":0.26977433760960895,"empirical_correctness":0.3333333333333333,"signed_gap":-0.06355899572372437,"absolute_gap":0.06355899572372437,"lower":0.2,"upper":0.3,"upper_inclusive":false},{"index":3,"count":16,"confidence":0.3727391678839922,"empirical_correctness":0.1875,"signed_gap":0.1852391678839922,"absolute_gap":0.1852391678839922,"lower":0.3,"upper":0.4,"upper_inclusive":false},{"index":4,"count":9,"confidence":0.416867991288503,"empirical_correctness":0.4444444444444444,"signed_gap":-0.027576453155941427,"absolute_gap":0.027576453155941427,"lower":0.4,"upper":0.5,"upper_inclusive":false},{"index":5,"count":7,"confidence":0.5390430007662091,"empirical_correctness":0.5714285714285714,"signed_gap":-0.03238557066236225,"absolute_gap":0.03238557066236225,"lower":0.5,"upper":0.6,"upper_inclusive":false},{"index":6,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.6,"upper":0.7,"upper_inclusive":false},{"index":7,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.7,"upper":0.8,"upper_inclusive":false},{"index":8,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.8,"upper":0.9,"upper_inclusive":false},{"index":9,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.9,"upper":1.0,"upper_inclusive":true}]},"equal_mass":{"n":38,"bin_count":10,"ece":0.16968511829250738,"signed_mean_gap":0.05546277996740843,"absolute_mean_gap":0.05546277996740843,"bins":[{"index":0,"count":3,"confidence":0.2647893230120341,"empirical_correctness":0.3333333333333333,"signed_gap":-0.0685440103212992,"absolute_gap":0.0685440103212992,"min_confidence":0.25769057869911194,"max_confidence":0.2696058750152588},{"index":1,"count":4,"confidence":0.2914260923862457,"empirical_correctness":0.5,"signed_gap":-0.20857390761375427,"absolute_gap":0.20857390761375427,"min_confidence":0.27264732122421265,"max_confidence":0.3414263129234314},{"index":2,"count":4,"confidence":0.3484843745827675,"empirical_correctness":0.25,"signed_gap":0.09848437458276749,"absolute_gap":0.09848437458276749,"min_confidence":0.34259235858917236,"max_confidence":0.3529776930809021},{"index":3,"count":4,"confidence":0.37422014772892,"empirical_correctness":0.0,"signed_gap":0.37422014772892,"absolute_gap":0.37422014772892,"min_confidence":0.3557615876197815,"max_confidence":0.3813125193119049},{"index":4,"count":4,"confidence":0.38733096420764923,"empirical_correctness":0.25,"signed_gap":0.13733096420764923,"absolute_gap":0.13733096420764923,"min_confidence":0.38386663794517517,"max_confidence":0.3897477686405182},{"index":5,"count":3,"confidence":0.3940861423810323,"empirical_correctness":0.0,"signed_gap":0.3940861423810323,"absolute_gap":0.3940861423810323,"min_confidence":0.39122381806373596,"max_confidence":0.39654743671417236},{"index":6,"count":4,"confidence":0.40999869257211685,"empirical_correctness":0.25,"signed_gap":0.15999869257211685,"absolute_gap":0.15999869257211685,"min_confidence":0.4015539288520813,"max_confidence":0.41759636998176575},{"index":7,"count":4,"confidence":0.4207228869199753,"empirical_correctness":0.5,"signed_gap":-0.07927711308002472,"absolute_gap":0.07927711308002472,"min_confidence":0.4183422029018402,"max_confidence":0.4234970808029175},{"index":8,"count":4,"confidence":0.5038537308573723,"empirical_correctness":0.5,"signed_gap":0.003853730857372284,"absolute_gap":0.003853730857372284,"min_confidence":0.42892560362815857,"max_confidence":0.5331385135650635},{"index":9,"count":4,"confidence":0.5467029213905334,"empirical_correctness":0.75,"signed_gap":-0.20329707860946655,"absolute_gap":0.20329707860946655,"min_confidence":0.5394099354743958,"max_confidence":0.5562885403633118}]}},"latency_ms":{"n":43,"median":12.758370023220778,"p95":13.024371973006055},"per_action":{"keep":{"n":12,"raw":{"n":12,"acceptable":0,"unsafe":0,"review":1,"automated_count":11,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":1,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":0.08333333333333333,"automated_coverage":0.9166666666666666,"conditional_correctness":0.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.08333333333333333,"accuracy_wilson_95":{"lower":2.7755575615628914e-17,"upper":0.24249400665524085,"confidence_level":0.95}},"effective":{"n":12,"acceptable":0,"unsafe":0,"review":10,"automated_count":2,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":10,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":0.8333333333333334,"automated_coverage":0.16666666666666666,"conditional_correctness":0.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.8333333333333334,"accuracy_wilson_95":{"lower":2.7755575615628914e-17,"upper":0.24249400665524085,"confidence_level":0.95}}},"drop":{"n":16,"raw":{"n":16,"acceptable":13,"unsafe":0,"review":3,"automated_count":13,"automated_acceptable":13,"automated_unsafe":0,"unnecessary_reviews":3,"accuracy":0.8125,"unsafe_rate":0.0,"review_rate":0.1875,"automated_coverage":0.8125,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.1875,"accuracy_wilson_95":{"lower":0.5699111903802586,"upper":0.9340840092857187,"confidence_level":0.95}},"effective":{"n":16,"acceptable":4,"unsafe":0,"review":12,"automated_count":4,"automated_acceptable":4,"automated_unsafe":0,"unnecessary_reviews":12,"accuracy":0.25,"unsafe_rate":0.0,"review_rate":0.75,"automated_coverage":0.25,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.75,"accuracy_wilson_95":{"lower":0.10182067491213048,"upper":0.49498316535508774,"confidence_level":0.95}}},"mask":{"n":2,"raw":{"n":2,"acceptable":0,"unsafe":0,"review":1,"automated_count":1,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":1,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":0.5,"automated_coverage":0.5,"conditional_correctness":0.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.5,"accuracy_wilson_95":{"lower":0.0,"upper":0.6576197724933469,"confidence_level":0.95}},"effective":{"n":2,"acceptable":0,"unsafe":0,"review":2,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":2,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.6576197724933469,"confidence_level":0.95}}},"pseudonymize":{"n":2,"raw":{"n":2,"acceptable":0,"unsafe":0,"review":0,"automated_count":2,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.6576197724933469,"confidence_level":0.95}},"effective":{"n":2,"acceptable":0,"unsafe":0,"review":2,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":2,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.6576197724933469,"confidence_level":0.95}}},"generalize":{"n":6,"raw":{"n":6,"acceptable":0,"unsafe":0,"review":0,"automated_count":6,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":2.7755575615628914e-17,"upper":0.3903342879021653,"confidence_level":0.95}},"effective":{"n":6,"acceptable":0,"unsafe":0,"review":5,"automated_count":1,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":5,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":0.8333333333333334,"automated_coverage":0.16666666666666666,"conditional_correctness":0.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.8333333333333334,"accuracy_wilson_95":{"lower":2.7755575615628914e-17,"upper":0.3903342879021653,"confidence_level":0.95}}},"review":{"n":0,"raw":{"n":0,"acceptable":0,"unsafe":0,"review":0,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":null,"unsafe_rate":null,"review_rate":null,"automated_coverage":null,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":null},"effective":{"n":0,"acceptable":0,"unsafe":0,"review":0,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":null,"unsafe_rate":null,"review_rate":null,"automated_coverage":null,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":null}}}},"diagnostic43-compact_no_samples":{"n":43,"raw":{"n":43,"acceptable":18,"unsafe":2,"review":9,"automated_count":34,"automated_acceptable":16,"automated_unsafe":2,"unnecessary_reviews":7,"accuracy":0.4186046511627907,"unsafe_rate":0.046511627906976744,"review_rate":0.20930232558139536,"automated_coverage":0.7906976744186046,"conditional_correctness":0.47058823529411764,"conditional_unsafe":0.058823529411764705,"unnecessary_review_rate":0.16279069767441862,"accuracy_wilson_95":{"lower":0.28384550927849195,"upper":0.5667142261303578,"confidence_level":0.95}},"effective":{"n":43,"acceptable":10,"unsafe":0,"review":35,"automated_count":8,"automated_acceptable":5,"automated_unsafe":0,"unnecessary_reviews":30,"accuracy":0.23255813953488372,"unsafe_rate":0.0,"review_rate":0.813953488372093,"automated_coverage":0.18604651162790697,"conditional_correctness":0.625,"conditional_unsafe":0.0,"unnecessary_review_rate":0.6976744186046512,"accuracy_wilson_95":{"lower":0.1315367847938448,"upper":0.37744520297809003,"confidence_level":0.95}},"confusion":{"raw":{"keep":{"keep":0,"drop":9,"mask":0,"pseudonymize":0,"generalize":0,"review":3},"drop":{"keep":0,"drop":12,"mask":0,"pseudonymize":1,"generalize":0,"review":3},"mask":{"keep":0,"drop":1,"mask":0,"pseudonymize":1,"generalize":0,"review":0},"pseudonymize":{"keep":0,"drop":1,"mask":0,"pseudonymize":1,"generalize":0,"review":0},"generalize":{"keep":0,"drop":5,"mask":0,"pseudonymize":0,"generalize":0,"review":1},"review":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0}},"effective":{"keep":{"keep":0,"drop":3,"mask":0,"pseudonymize":0,"generalize":0,"review":9},"drop":{"keep":0,"drop":5,"mask":0,"pseudonymize":0,"generalize":0,"review":11},"mask":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":2},"pseudonymize":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":2},"generalize":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":6},"review":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":0}}},"probability_metrics":{"singleton_n":38,"multiacceptable_n":5,"multiacceptable_coverage":0.11627906976744186,"nll":1.9303709664913542,"nll_zero_mass_n":0,"brier":0.8414202088132919,"acceptable_set_n":43,"acceptable_set_nll":1.7420177427026007,"acceptable_set_zero_mass_n":0},"calibration_all_labels":{"equal_width":{"n":43,"bin_count":10,"ece":0.030570030905479623,"signed_mean_gap":-0.025116851856542173,"absolute_mean_gap":0.025116851856542173,"bins":[{"index":0,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.0,"upper":0.1,"upper_inclusive":false},{"index":1,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.1,"upper":0.2,"upper_inclusive":false},{"index":2,"count":5,"confidence":0.2760542809963226,"empirical_correctness":0.4,"signed_gap":-0.1239457190036774,"absolute_gap":0.1239457190036774,"lower":0.2,"upper":0.3,"upper_inclusive":false},{"index":3,"count":14,"confidence":0.3472608838762556,"empirical_correctness":0.35714285714285715,"signed_gap":-0.009881973266601562,"absolute_gap":0.009881973266601562,"lower":0.3,"upper":0.4,"upper_inclusive":false},{"index":4,"count":21,"confidence":0.43415444521676927,"empirical_correctness":0.42857142857142855,"signed_gap":0.0055830166453407215,"absolute_gap":0.0055830166453407215,"lower":0.4,"upper":0.5,"upper_inclusive":false},{"index":5,"count":3,"confidence":0.5202694137891134,"empirical_correctness":0.6666666666666666,"signed_gap":-0.14639725287755323,"absolute_gap":0.14639725287755323,"lower":0.5,"upper":0.6,"upper_inclusive":false},{"index":6,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.6,"upper":0.7,"upper_inclusive":false},{"index":7,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.7,"upper":0.8,"upper_inclusive":false},{"index":8,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.8,"upper":0.9,"upper_inclusive":false},{"index":9,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.9,"upper":1.0,"upper_inclusive":true}]},"equal_mass":{"n":43,"bin_count":10,"ece":0.1536019742488861,"signed_mean_gap":-0.025116851856542197,"absolute_mean_gap":0.025116851856542197,"bins":[{"index":0,"count":4,"confidence":0.27353227883577347,"empirical_correctness":0.25,"signed_gap":0.023532278835773468,"absolute_gap":0.023532278835773468,"min_confidence":0.2635732889175415,"max_confidence":0.28349626064300537},{"index":1,"count":4,"confidence":0.30378593504428864,"empirical_correctness":0.75,"signed_gap":-0.44621406495571136,"absolute_gap":0.44621406495571136,"min_confidence":0.2861422896385193,"max_confidence":0.3187706172466278},{"index":2,"count":4,"confidence":0.34138452261686325,"empirical_correctness":0.25,"signed_gap":0.09138452261686325,"absolute_gap":0.09138452261686325,"min_confidence":0.32947564125061035,"max_confidence":0.3481139540672302},{"index":3,"count":5,"confidence":0.3613950490951538,"empirical_correctness":0.4,"signed_gap":-0.038604950904846225,"absolute_gap":0.038604950904846225,"min_confidence":0.3494824469089508,"max_confidence":0.37118855118751526},{"index":4,"count":4,"confidence":0.39081696420907974,"empirical_correctness":0.25,"signed_gap":0.14081696420907974,"absolute_gap":0.14081696420907974,"min_confidence":0.3716535270214081,"max_confidence":0.4024937152862549},{"index":5,"count":4,"confidence":0.40960656851530075,"empirical_correctness":0.0,"signed_gap":0.40960656851530075,"absolute_gap":0.40960656851530075,"min_confidence":0.4031168222427368,"max_confidence":0.41476890444755554},{"index":6,"count":5,"confidence":0.4202137589454651,"empirical_correctness":0.4,"signed_gap":0.020213758945465066,"absolute_gap":0.020213758945465066,"min_confidence":0.41488131880760193,"max_confidence":0.4253949820995331},{"index":7,"count":4,"confidence":0.4378933534026146,"empirical_correctness":0.75,"signed_gap":-0.3121066465973854,"absolute_gap":0.3121066465973854,"min_confidence":0.4331076741218567,"max_confidence":0.4435271918773651},{"index":8,"count":4,"confidence":0.45871030539274216,"empirical_correctness":0.5,"signed_gap":-0.04128969460725784,"absolute_gap":0.04128969460725784,"min_confidence":0.44885873794555664,"max_confidence":0.4696139991283417},{"index":9,"count":5,"confidence":0.5098023235797882,"empirical_correctness":0.6,"signed_gap":-0.09019767642021181,"absolute_gap":0.09019767642021181,"min_confidence":0.4937288463115692,"max_confidence":0.5266442894935608}]}},"calibration_singleton_labels":{"equal_width":{"n":38,"bin_count":10,"ece":0.07546394671264448,"signed_mean_gap":0.052348590995136064,"absolute_mean_gap":0.052348590995136064,"bins":[{"index":0,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.0,"upper":0.1,"upper_inclusive":false},{"index":1,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.1,"upper":0.2,"upper_inclusive":false},{"index":2,"count":4,"confidence":0.27353227883577347,"empirical_correctness":0.25,"signed_gap":0.023532278835773468,"absolute_gap":0.023532278835773468,"lower":0.2,"upper":0.3,"upper_inclusive":false},{"index":3,"count":13,"confidence":0.3454202940830818,"empirical_correctness":0.3076923076923077,"signed_gap":0.03772798639077407,"absolute_gap":0.03772798639077407,"lower":0.3,"upper":0.4,"upper_inclusive":false},{"index":4,"count":18,"confidence":0.43576918211248183,"empirical_correctness":0.3333333333333333,"signed_gap":0.10243584877914852,"absolute_gap":0.10243584877914852,"lower":0.4,"upper":0.5,"upper_inclusive":false},{"index":5,"count":3,"confidence":0.5202694137891134,"empirical_correctness":0.6666666666666666,"signed_gap":-0.14639725287755323,"absolute_gap":0.14639725287755323,"lower":0.5,"upper":0.6,"upper_inclusive":false},{"index":6,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.6,"upper":0.7,"upper_inclusive":false},{"index":7,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.7,"upper":0.8,"upper_inclusive":false},{"index":8,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.8,"upper":0.9,"upper_inclusive":false},{"index":9,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.9,"upper":1.0,"upper_inclusive":true}]},"equal_mass":{"n":38,"bin_count":10,"ece":0.1752355561444634,"signed_mean_gap":0.05234859099513606,"absolute_mean_gap":0.05234859099513606,"bins":[{"index":0,"count":3,"confidence":0.27021095156669617,"empirical_correctness":0.3333333333333333,"signed_gap":-0.06312238176663715,"absolute_gap":0.06312238176663715,"min_confidence":0.2635732889175415,"max_confidence":0.2800637483596802},{"index":1,"count":4,"confidence":0.30312442779541016,"empirical_correctness":0.5,"signed_gap":-0.19687557220458984,"absolute_gap":0.19687557220458984,"min_confidence":0.28349626064300537,"max_confidence":0.3187706172466278},{"index":2,"count":4,"confidence":0.34138452261686325,"empirical_correctness":0.25,"signed_gap":0.09138452261686325,"absolute_gap":0.09138452261686325,"min_confidence":0.32947564125061035,"max_confidence":0.3481139540672302},{"index":3,"count":4,"confidence":0.35894667357206345,"empirical_correctness":0.25,"signed_gap":0.10894667357206345,"absolute_gap":0.10894667357206345,"min_confidence":0.3494824469089508,"max_confidence":0.3639580309391022},{"index":4,"count":4,"confidence":0.39081696420907974,"empirical_correctness":0.25,"signed_gap":0.14081696420907974,"absolute_gap":0.14081696420907974,"min_confidence":0.3716535270214081,"max_confidence":0.4024937152862549},{"index":5,"count":3,"confidence":0.4078857898712158,"empirical_correctness":0.0,"signed_gap":0.4078857898712158,"absolute_gap":0.4078857898712158,"min_confidence":0.4031168222427368,"max_confidence":0.4116915762424469},{"index":6,"count":4,"confidence":0.4201415032148361,"empirical_correctness":0.0,"signed_gap":0.4201415032148361,"absolute_gap":0.4201415032148361,"min_confidence":0.41476890444755554,"max_confidence":0.4253949820995331},{"index":7,"count":4,"confidence":0.4405764415860176,"empirical_correctness":0.5,"signed_gap":-0.05942355841398239,"absolute_gap":0.05942355841398239,"min_confidence":0.4331076741218567,"max_confidence":0.44885873794555664},{"index":8,"count":4,"confidence":0.4699278324842453,"empirical_correctness":0.75,"signed_gap":-0.2800721675157547,"absolute_gap":0.2800721675157547,"min_confidence":0.4537966847419739,"max_confidence":0.4937288463115692},{"index":9,"count":4,"confidence":0.513820692896843,"empirical_correctness":0.5,"signed_gap":0.013820692896842957,"absolute_gap":0.013820692896842957,"min_confidence":0.49447453022003174,"max_confidence":0.5266442894935608}]}},"latency_ms":{"n":43,"median":12.973800010513514,"p95":13.699054013704881},"per_action":{"keep":{"n":12,"raw":{"n":12,"acceptable":0,"unsafe":0,"review":3,"automated_count":9,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":3,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":0.25,"automated_coverage":0.75,"conditional_correctness":0.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.25,"accuracy_wilson_95":{"lower":2.7755575615628914e-17,"upper":0.24249400665524085,"confidence_level":0.95}},"effective":{"n":12,"acceptable":0,"unsafe":0,"review":9,"automated_count":3,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":9,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":0.75,"automated_coverage":0.25,"conditional_correctness":0.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.75,"accuracy_wilson_95":{"lower":2.7755575615628914e-17,"upper":0.24249400665524085,"confidence_level":0.95}}},"drop":{"n":16,"raw":{"n":16,"acceptable":12,"unsafe":1,"review":3,"automated_count":13,"automated_acceptable":12,"automated_unsafe":1,"unnecessary_reviews":3,"accuracy":0.75,"unsafe_rate":0.0625,"review_rate":0.1875,"automated_coverage":0.8125,"conditional_correctness":0.9230769230769231,"conditional_unsafe":0.07692307692307693,"unnecessary_review_rate":0.1875,"accuracy_wilson_95":{"lower":0.5050168346449122,"upper":0.8981793250878696,"confidence_level":0.95}},"effective":{"n":16,"acceptable":5,"unsafe":0,"review":11,"automated_count":5,"automated_acceptable":5,"automated_unsafe":0,"unnecessary_reviews":11,"accuracy":0.3125,"unsafe_rate":0.0,"review_rate":0.6875,"automated_coverage":0.3125,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.6875,"accuracy_wilson_95":{"lower":0.14164643854782039,"upper":0.5559564416525933,"confidence_level":0.95}}},"mask":{"n":2,"raw":{"n":2,"acceptable":0,"unsafe":1,"review":0,"automated_count":2,"automated_acceptable":0,"automated_unsafe":1,"unnecessary_reviews":0,"accuracy":0.0,"unsafe_rate":0.5,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.0,"conditional_unsafe":0.5,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.6576197724933469,"confidence_level":0.95}},"effective":{"n":2,"acceptable":0,"unsafe":0,"review":2,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":2,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.6576197724933469,"confidence_level":0.95}}},"pseudonymize":{"n":2,"raw":{"n":2,"acceptable":1,"unsafe":0,"review":0,"automated_count":2,"automated_acceptable":1,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":0.5,"unsafe_rate":0.0,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.5,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.09453120573423074,"upper":0.9054687942657693,"confidence_level":0.95}},"effective":{"n":2,"acceptable":0,"unsafe":0,"review":2,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":2,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.6576197724933469,"confidence_level":0.95}}},"generalize":{"n":6,"raw":{"n":6,"acceptable":0,"unsafe":0,"review":1,"automated_count":5,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":1,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":0.16666666666666666,"automated_coverage":0.8333333333333334,"conditional_correctness":0.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.16666666666666666,"accuracy_wilson_95":{"lower":2.7755575615628914e-17,"upper":0.3903342879021653,"confidence_level":0.95}},"effective":{"n":6,"acceptable":0,"unsafe":0,"review":6,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":6,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":2.7755575615628914e-17,"upper":0.3903342879021653,"confidence_level":0.95}}},"review":{"n":0,"raw":{"n":0,"acceptable":0,"unsafe":0,"review":0,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":null,"unsafe_rate":null,"review_rate":null,"automated_coverage":null,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":null},"effective":{"n":0,"acceptable":0,"unsafe":0,"review":0,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":null,"unsafe_rate":null,"review_rate":null,"automated_coverage":null,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":null}}}},"challenge56-no_samples":{"n":56,"raw":{"n":56,"acceptable":9,"unsafe":0,"review":49,"automated_count":7,"automated_acceptable":7,"automated_unsafe":0,"unnecessary_reviews":47,"accuracy":0.16071428571428573,"unsafe_rate":0.0,"review_rate":0.875,"automated_coverage":0.125,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.8392857142857143,"accuracy_wilson_95":{"lower":0.08692667607595139,"upper":0.2780620666974952,"confidence_level":0.95}},"effective":{"n":56,"acceptable":3,"unsafe":0,"review":55,"automated_count":1,"automated_acceptable":1,"automated_unsafe":0,"unnecessary_reviews":53,"accuracy":0.05357142857142857,"unsafe_rate":0.0,"review_rate":0.9821428571428571,"automated_coverage":0.017857142857142856,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.9464285714285714,"accuracy_wilson_95":{"lower":0.018385781382109126,"upper":0.14607309068821533,"confidence_level":0.95}},"confusion":{"raw":{"keep":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":8},"drop":{"keep":0,"drop":7,"mask":0,"pseudonymize":0,"generalize":0,"review":15},"mask":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":8},"pseudonymize":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":8},"generalize":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":8},"review":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":2}},"effective":{"keep":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":8},"drop":{"keep":0,"drop":1,"mask":0,"pseudonymize":0,"generalize":0,"review":21},"mask":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":8},"pseudonymize":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":8},"generalize":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":8},"review":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":2}}},"probability_metrics":{"singleton_n":56,"multiacceptable_n":0,"multiacceptable_coverage":0.0,"nll":1.690553682886682,"nll_zero_mass_n":0,"brier":0.8586584725882701,"acceptable_set_n":56,"acceptable_set_nll":1.690553682886682,"acceptable_set_zero_mass_n":0},"calibration_all_labels":{"equal_width":{"n":56,"bin_count":10,"ece":0.2802708862083299,"signed_mean_gap":0.2802708862083299,"absolute_mean_gap":0.2802708862083299,"bins":[{"index":0,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.0,"upper":0.1,"upper_inclusive":false},{"index":1,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.1,"upper":0.2,"upper_inclusive":false},{"index":2,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.2,"upper":0.3,"upper_inclusive":false},{"index":3,"count":17,"confidence":0.375629475888084,"empirical_correctness":0.17647058823529413,"signed_gap":0.19915888765278986,"absolute_gap":0.19915888765278986,"lower":0.3,"upper":0.4,"upper_inclusive":false},{"index":4,"count":29,"confidence":0.427796909521366,"empirical_correctness":0.20689655172413793,"signed_gap":0.22090035779722808,"absolute_gap":0.22090035779722808,"lower":0.4,"upper":0.5,"upper_inclusive":false},{"index":5,"count":4,"confidence":0.518148884177208,"empirical_correctness":0.0,"signed_gap":0.518148884177208,"absolute_gap":0.518148884177208,"lower":0.5,"upper":0.6,"upper_inclusive":false},{"index":6,"count":6,"confidence":0.6384604374567667,"empirical_correctness":0.0,"signed_gap":0.6384604374567667,"absolute_gap":0.6384604374567667,"lower":0.6,"upper":0.7,"upper_inclusive":false},{"index":7,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.7,"upper":0.8,"upper_inclusive":false},{"index":8,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.8,"upper":0.9,"upper_inclusive":false},{"index":9,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.9,"upper":1.0,"upper_inclusive":true}]},"equal_mass":{"n":56,"bin_count":10,"ece":0.2887108623981476,"signed_mean_gap":0.2802708862083299,"absolute_mean_gap":0.2802708862083299,"bins":[{"index":0,"count":5,"confidence":0.35273613333702086,"empirical_correctness":0.4,"signed_gap":-0.04726386666297916,"absolute_gap":0.04726386666297916,"min_confidence":0.3392011821269989,"max_confidence":0.3635598421096802},{"index":1,"count":6,"confidence":0.37950973212718964,"empirical_correctness":0.16666666666666666,"signed_gap":0.21284306546052298,"absolute_gap":0.21284306546052298,"min_confidence":0.3639676868915558,"max_confidence":0.3848356306552887},{"index":2,"count":5,"confidence":0.3898014724254608,"empirical_correctness":0.0,"signed_gap":0.3898014724254608,"absolute_gap":0.3898014724254608,"min_confidence":0.3849027752876282,"max_confidence":0.39299675822257996},{"index":3,"count":6,"confidence":0.4022624045610428,"empirical_correctness":0.16666666666666666,"signed_gap":0.23559573789437613,"absolute_gap":0.23559573789437613,"min_confidence":0.3959546685218811,"max_confidence":0.40515589714050293},{"index":4,"count":6,"confidence":0.40931429465611774,"empirical_correctness":0.16666666666666666,"signed_gap":0.24264762798945108,"absolute_gap":0.24264762798945108,"min_confidence":0.40593674778938293,"max_confidence":0.41242194175720215},{"index":5,"count":5,"confidence":0.4174203217029572,"empirical_correctness":0.4,"signed_gap":0.017420321702957153,"absolute_gap":0.017420321702957153,"min_confidence":0.415353000164032,"max_confidence":0.41984400153160095},{"index":6,"count":6,"confidence":0.435155729452769,"empirical_correctness":0.0,"signed_gap":0.435155729452769,"absolute_gap":0.435155729452769,"min_confidence":0.4241294264793396,"max_confidence":0.44667160511016846},{"index":7,"count":5,"confidence":0.45865869522094727,"empirical_correctness":0.0,"signed_gap":0.45865869522094727,"absolute_gap":0.45865869522094727,"min_confidence":0.45260724425315857,"max_confidence":0.46676379442214966},{"index":8,"count":6,"confidence":0.5023118207852045,"empirical_correctness":0.3333333333333333,"signed_gap":0.16897848745187122,"absolute_gap":0.16897848745187122,"min_confidence":0.4680881202220917,"max_confidence":0.5335965752601624},{"index":9,"count":6,"confidence":0.6384604374567667,"empirical_correctness":0.0,"signed_gap":0.6384604374567667,"absolute_gap":0.6384604374567667,"min_confidence":0.6141742467880249,"max_confidence":0.6643553972244263}]}},"calibration_singleton_labels":{"equal_width":{"n":56,"bin_count":10,"ece":0.2802708862083299,"signed_mean_gap":0.2802708862083299,"absolute_mean_gap":0.2802708862083299,"bins":[{"index":0,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.0,"upper":0.1,"upper_inclusive":false},{"index":1,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.1,"upper":0.2,"upper_inclusive":false},{"index":2,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.2,"upper":0.3,"upper_inclusive":false},{"index":3,"count":17,"confidence":0.375629475888084,"empirical_correctness":0.17647058823529413,"signed_gap":0.19915888765278986,"absolute_gap":0.19915888765278986,"lower":0.3,"upper":0.4,"upper_inclusive":false},{"index":4,"count":29,"confidence":0.427796909521366,"empirical_correctness":0.20689655172413793,"signed_gap":0.22090035779722808,"absolute_gap":0.22090035779722808,"lower":0.4,"upper":0.5,"upper_inclusive":false},{"index":5,"count":4,"confidence":0.518148884177208,"empirical_correctness":0.0,"signed_gap":0.518148884177208,"absolute_gap":0.518148884177208,"lower":0.5,"upper":0.6,"upper_inclusive":false},{"index":6,"count":6,"confidence":0.6384604374567667,"empirical_correctness":0.0,"signed_gap":0.6384604374567667,"absolute_gap":0.6384604374567667,"lower":0.6,"upper":0.7,"upper_inclusive":false},{"index":7,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.7,"upper":0.8,"upper_inclusive":false},{"index":8,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.8,"upper":0.9,"upper_inclusive":false},{"index":9,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.9,"upper":1.0,"upper_inclusive":true}]},"equal_mass":{"n":56,"bin_count":10,"ece":0.2887108623981476,"signed_mean_gap":0.2802708862083299,"absolute_mean_gap":0.2802708862083299,"bins":[{"index":0,"count":5,"confidence":0.35273613333702086,"empirical_correctness":0.4,"signed_gap":-0.04726386666297916,"absolute_gap":0.04726386666297916,"min_confidence":0.3392011821269989,"max_confidence":0.3635598421096802},{"index":1,"count":6,"confidence":0.37950973212718964,"empirical_correctness":0.16666666666666666,"signed_gap":0.21284306546052298,"absolute_gap":0.21284306546052298,"min_confidence":0.3639676868915558,"max_confidence":0.3848356306552887},{"index":2,"count":5,"confidence":0.3898014724254608,"empirical_correctness":0.0,"signed_gap":0.3898014724254608,"absolute_gap":0.3898014724254608,"min_confidence":0.3849027752876282,"max_confidence":0.39299675822257996},{"index":3,"count":6,"confidence":0.4022624045610428,"empirical_correctness":0.16666666666666666,"signed_gap":0.23559573789437613,"absolute_gap":0.23559573789437613,"min_confidence":0.3959546685218811,"max_confidence":0.40515589714050293},{"index":4,"count":6,"confidence":0.40931429465611774,"empirical_correctness":0.16666666666666666,"signed_gap":0.24264762798945108,"absolute_gap":0.24264762798945108,"min_confidence":0.40593674778938293,"max_confidence":0.41242194175720215},{"index":5,"count":5,"confidence":0.4174203217029572,"empirical_correctness":0.4,"signed_gap":0.017420321702957153,"absolute_gap":0.017420321702957153,"min_confidence":0.415353000164032,"max_confidence":0.41984400153160095},{"index":6,"count":6,"confidence":0.435155729452769,"empirical_correctness":0.0,"signed_gap":0.435155729452769,"absolute_gap":0.435155729452769,"min_confidence":0.4241294264793396,"max_confidence":0.44667160511016846},{"index":7,"count":5,"confidence":0.45865869522094727,"empirical_correctness":0.0,"signed_gap":0.45865869522094727,"absolute_gap":0.45865869522094727,"min_confidence":0.45260724425315857,"max_confidence":0.46676379442214966},{"index":8,"count":6,"confidence":0.5023118207852045,"empirical_correctness":0.3333333333333333,"signed_gap":0.16897848745187122,"absolute_gap":0.16897848745187122,"min_confidence":0.4680881202220917,"max_confidence":0.5335965752601624},{"index":9,"count":6,"confidence":0.6384604374567667,"empirical_correctness":0.0,"signed_gap":0.6384604374567667,"absolute_gap":0.6384604374567667,"min_confidence":0.6141742467880249,"max_confidence":0.6643553972244263}]}},"latency_ms":{"n":56,"median":11.459742003353313,"p95":11.569784503080882},"per_action":{"keep":{"n":8,"raw":{"n":8,"acceptable":0,"unsafe":0,"review":8,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":8,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.32440756488388023,"confidence_level":0.95}},"effective":{"n":8,"acceptable":0,"unsafe":0,"review":8,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":8,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.32440756488388023,"confidence_level":0.95}}},"drop":{"n":22,"raw":{"n":22,"acceptable":7,"unsafe":0,"review":15,"automated_count":7,"automated_acceptable":7,"automated_unsafe":0,"unnecessary_reviews":15,"accuracy":0.3181818181818182,"unsafe_rate":0.0,"review_rate":0.6818181818181818,"automated_coverage":0.3181818181818182,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.6818181818181818,"accuracy_wilson_95":{"lower":0.1636059318135624,"upper":0.5268140225895723,"confidence_level":0.95}},"effective":{"n":22,"acceptable":1,"unsafe":0,"review":21,"automated_count":1,"automated_acceptable":1,"automated_unsafe":0,"unnecessary_reviews":21,"accuracy":0.045454545454545456,"unsafe_rate":0.0,"review_rate":0.9545454545454546,"automated_coverage":0.045454545454545456,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.9545454545454546,"accuracy_wilson_95":{"lower":0.00806942762126385,"upper":0.21798045838657332,"confidence_level":0.95}}},"mask":{"n":8,"raw":{"n":8,"acceptable":0,"unsafe":0,"review":8,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":8,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.32440756488388023,"confidence_level":0.95}},"effective":{"n":8,"acceptable":0,"unsafe":0,"review":8,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":8,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.32440756488388023,"confidence_level":0.95}}},"pseudonymize":{"n":8,"raw":{"n":8,"acceptable":0,"unsafe":0,"review":8,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":8,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.32440756488388023,"confidence_level":0.95}},"effective":{"n":8,"acceptable":0,"unsafe":0,"review":8,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":8,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.32440756488388023,"confidence_level":0.95}}},"generalize":{"n":8,"raw":{"n":8,"acceptable":0,"unsafe":0,"review":8,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":8,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.32440756488388023,"confidence_level":0.95}},"effective":{"n":8,"acceptable":0,"unsafe":0,"review":8,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":8,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.32440756488388023,"confidence_level":0.95}}},"review":{"n":2,"raw":{"n":2,"acceptable":2,"unsafe":0,"review":2,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":1.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.34238022750665303,"upper":1.0,"confidence_level":0.95}},"effective":{"n":2,"acceptable":2,"unsafe":0,"review":2,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":1.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.34238022750665303,"upper":1.0,"confidence_level":0.95}}}}},"challenge56-compact_no_samples":{"n":56,"raw":{"n":56,"acceptable":20,"unsafe":6,"review":15,"automated_count":41,"automated_acceptable":19,"automated_unsafe":6,"unnecessary_reviews":14,"accuracy":0.35714285714285715,"unsafe_rate":0.10714285714285714,"review_rate":0.26785714285714285,"automated_coverage":0.7321428571428571,"conditional_correctness":0.4634146341463415,"conditional_unsafe":0.14634146341463414,"unnecessary_review_rate":0.25,"accuracy_wilson_95":{"lower":0.24456571707649905,"upper":0.4880611219860049,"confidence_level":0.95}},"effective":{"n":56,"acceptable":5,"unsafe":0,"review":53,"automated_count":3,"automated_acceptable":3,"automated_unsafe":0,"unnecessary_reviews":51,"accuracy":0.08928571428571429,"unsafe_rate":0.0,"review_rate":0.9464285714285714,"automated_coverage":0.05357142857142857,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.9107142857142857,"accuracy_wilson_95":{"lower":0.03874214844958693,"upper":0.19256001385511162,"confidence_level":0.95}},"confusion":{"raw":{"keep":{"keep":0,"drop":4,"mask":0,"pseudonymize":0,"generalize":0,"review":4},"drop":{"keep":0,"drop":17,"mask":0,"pseudonymize":5,"generalize":0,"review":0},"mask":{"keep":0,"drop":6,"mask":0,"pseudonymize":1,"generalize":0,"review":1},"pseudonymize":{"keep":0,"drop":1,"mask":4,"pseudonymize":2,"generalize":0,"review":1},"generalize":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":8},"review":{"keep":0,"drop":1,"mask":0,"pseudonymize":0,"generalize":0,"review":1}},"effective":{"keep":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":8},"drop":{"keep":0,"drop":3,"mask":0,"pseudonymize":0,"generalize":0,"review":19},"mask":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":8},"pseudonymize":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":8},"generalize":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":8},"review":{"keep":0,"drop":0,"mask":0,"pseudonymize":0,"generalize":0,"review":2}}},"probability_metrics":{"singleton_n":56,"multiacceptable_n":0,"multiacceptable_coverage":0.0,"nll":1.473299000003372,"nll_zero_mass_n":0,"brier":0.7304449664600072,"acceptable_set_n":56,"acceptable_set_nll":1.473299000003372,"acceptable_set_zero_mass_n":0},"calibration_all_labels":{"equal_width":{"n":56,"bin_count":10,"ece":0.1288681346923113,"signed_mean_gap":-0.011939053290656627,"absolute_mean_gap":0.011939053290656627,"bins":[{"index":0,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.0,"upper":0.1,"upper_inclusive":false},{"index":1,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.1,"upper":0.2,"upper_inclusive":false},{"index":2,"count":25,"confidence":0.26672881543636323,"empirical_correctness":0.16,"signed_gap":0.10672881543636323,"absolute_gap":0.10672881543636323,"lower":0.2,"upper":0.3,"upper_inclusive":false},{"index":3,"count":14,"confidence":0.3423634724957602,"empirical_correctness":0.42857142857142855,"signed_gap":-0.08620795607566833,"absolute_gap":0.08620795607566833,"lower":0.3,"upper":0.4,"upper_inclusive":false},{"index":4,"count":12,"confidence":0.43869251012802124,"empirical_correctness":0.6666666666666666,"signed_gap":-0.2279741565386454,"absolute_gap":0.2279741565386454,"lower":0.4,"upper":0.5,"upper_inclusive":false},{"index":5,"count":5,"confidence":0.52115877866745,"empirical_correctness":0.4,"signed_gap":0.12115877866744995,"absolute_gap":0.12115877866744995,"lower":0.5,"upper":0.6,"upper_inclusive":false},{"index":6,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.6,"upper":0.7,"upper_inclusive":false},{"index":7,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.7,"upper":0.8,"upper_inclusive":false},{"index":8,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.8,"upper":0.9,"upper_inclusive":false},{"index":9,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.9,"upper":1.0,"upper_inclusive":true}]},"equal_mass":{"n":56,"bin_count":10,"ece":0.1783755770219224,"signed_mean_gap":-0.01193905329065662,"absolute_mean_gap":0.01193905329065662,"bins":[{"index":0,"count":5,"confidence":0.24462451040744781,"empirical_correctness":0.0,"signed_gap":0.24462451040744781,"absolute_gap":0.24462451040744781,"min_confidence":0.23361195623874664,"max_confidence":0.25257453322410583},{"index":1,"count":6,"confidence":0.2592856337626775,"empirical_correctness":0.0,"signed_gap":0.2592856337626775,"absolute_gap":0.2592856337626775,"min_confidence":0.2571769058704376,"max_confidence":0.2625490427017212},{"index":2,"count":5,"confidence":0.2664622187614441,"empirical_correctness":0.4,"signed_gap":-0.13353778123855592,"absolute_gap":0.13353778123855592,"min_confidence":0.26390859484672546,"max_confidence":0.2687568664550781},{"index":3,"count":6,"confidence":0.2815895080566406,"empirical_correctness":0.3333333333333333,"signed_gap":-0.05174382527669269,"absolute_gap":0.05174382527669269,"min_confidence":0.2765207588672638,"max_confidence":0.28614458441734314},{"index":4,"count":6,"confidence":0.2975590576728185,"empirical_correctness":0.16666666666666666,"signed_gap":0.13089239100615183,"absolute_gap":0.13089239100615183,"min_confidence":0.28666406869888306,"max_confidence":0.31616073846817017},{"index":5,"count":5,"confidence":0.33268338441848755,"empirical_correctness":0.4,"signed_gap":-0.06731661558151247,"absolute_gap":0.06731661558151247,"min_confidence":0.3216593861579895,"max_confidence":0.345784068107605},{"index":6,"count":6,"confidence":0.36864220599333447,"empirical_correctness":0.5,"signed_gap":-0.13135779400666553,"absolute_gap":0.13135779400666553,"min_confidence":0.3469696342945099,"max_confidence":0.39782440662384033},{"index":7,"count":5,"confidence":0.41524580121040344,"empirical_correctness":0.8,"signed_gap":-0.3847541987895966,"absolute_gap":0.3847541987895966,"min_confidence":0.40680986642837524,"max_confidence":0.4241659939289093},{"index":8,"count":6,"confidence":0.44964050749937695,"empirical_correctness":0.6666666666666666,"signed_gap":-0.21702615916728968,"absolute_gap":0.21702615916728968,"min_confidence":0.4251093864440918,"max_confidence":0.48064175248146057},{"index":9,"count":6,"confidence":0.5160053273042043,"empirical_correctness":0.3333333333333333,"signed_gap":0.18267199397087103,"absolute_gap":0.18267199397087103,"min_confidence":0.4902380704879761,"max_confidence":0.5489709973335266}]}},"calibration_singleton_labels":{"equal_width":{"n":56,"bin_count":10,"ece":0.1288681346923113,"signed_mean_gap":-0.011939053290656627,"absolute_mean_gap":0.011939053290656627,"bins":[{"index":0,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.0,"upper":0.1,"upper_inclusive":false},{"index":1,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.1,"upper":0.2,"upper_inclusive":false},{"index":2,"count":25,"confidence":0.26672881543636323,"empirical_correctness":0.16,"signed_gap":0.10672881543636323,"absolute_gap":0.10672881543636323,"lower":0.2,"upper":0.3,"upper_inclusive":false},{"index":3,"count":14,"confidence":0.3423634724957602,"empirical_correctness":0.42857142857142855,"signed_gap":-0.08620795607566833,"absolute_gap":0.08620795607566833,"lower":0.3,"upper":0.4,"upper_inclusive":false},{"index":4,"count":12,"confidence":0.43869251012802124,"empirical_correctness":0.6666666666666666,"signed_gap":-0.2279741565386454,"absolute_gap":0.2279741565386454,"lower":0.4,"upper":0.5,"upper_inclusive":false},{"index":5,"count":5,"confidence":0.52115877866745,"empirical_correctness":0.4,"signed_gap":0.12115877866744995,"absolute_gap":0.12115877866744995,"lower":0.5,"upper":0.6,"upper_inclusive":false},{"index":6,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.6,"upper":0.7,"upper_inclusive":false},{"index":7,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.7,"upper":0.8,"upper_inclusive":false},{"index":8,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.8,"upper":0.9,"upper_inclusive":false},{"index":9,"count":0,"confidence":null,"empirical_correctness":null,"signed_gap":null,"absolute_gap":null,"lower":0.9,"upper":1.0,"upper_inclusive":true}]},"equal_mass":{"n":56,"bin_count":10,"ece":0.1783755770219224,"signed_mean_gap":-0.01193905329065662,"absolute_mean_gap":0.01193905329065662,"bins":[{"index":0,"count":5,"confidence":0.24462451040744781,"empirical_correctness":0.0,"signed_gap":0.24462451040744781,"absolute_gap":0.24462451040744781,"min_confidence":0.23361195623874664,"max_confidence":0.25257453322410583},{"index":1,"count":6,"confidence":0.2592856337626775,"empirical_correctness":0.0,"signed_gap":0.2592856337626775,"absolute_gap":0.2592856337626775,"min_confidence":0.2571769058704376,"max_confidence":0.2625490427017212},{"index":2,"count":5,"confidence":0.2664622187614441,"empirical_correctness":0.4,"signed_gap":-0.13353778123855592,"absolute_gap":0.13353778123855592,"min_confidence":0.26390859484672546,"max_confidence":0.2687568664550781},{"index":3,"count":6,"confidence":0.2815895080566406,"empirical_correctness":0.3333333333333333,"signed_gap":-0.05174382527669269,"absolute_gap":0.05174382527669269,"min_confidence":0.2765207588672638,"max_confidence":0.28614458441734314},{"index":4,"count":6,"confidence":0.2975590576728185,"empirical_correctness":0.16666666666666666,"signed_gap":0.13089239100615183,"absolute_gap":0.13089239100615183,"min_confidence":0.28666406869888306,"max_confidence":0.31616073846817017},{"index":5,"count":5,"confidence":0.33268338441848755,"empirical_correctness":0.4,"signed_gap":-0.06731661558151247,"absolute_gap":0.06731661558151247,"min_confidence":0.3216593861579895,"max_confidence":0.345784068107605},{"index":6,"count":6,"confidence":0.36864220599333447,"empirical_correctness":0.5,"signed_gap":-0.13135779400666553,"absolute_gap":0.13135779400666553,"min_confidence":0.3469696342945099,"max_confidence":0.39782440662384033},{"index":7,"count":5,"confidence":0.41524580121040344,"empirical_correctness":0.8,"signed_gap":-0.3847541987895966,"absolute_gap":0.3847541987895966,"min_confidence":0.40680986642837524,"max_confidence":0.4241659939289093},{"index":8,"count":6,"confidence":0.44964050749937695,"empirical_correctness":0.6666666666666666,"signed_gap":-0.21702615916728968,"absolute_gap":0.21702615916728968,"min_confidence":0.4251093864440918,"max_confidence":0.48064175248146057},{"index":9,"count":6,"confidence":0.5160053273042043,"empirical_correctness":0.3333333333333333,"signed_gap":0.18267199397087103,"absolute_gap":0.18267199397087103,"min_confidence":0.4902380704879761,"max_confidence":0.5489709973335266}]}},"latency_ms":{"n":56,"median":11.733671999536455,"p95":12.831166037358344},"per_action":{"keep":{"n":8,"raw":{"n":8,"acceptable":0,"unsafe":0,"review":4,"automated_count":4,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":4,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":0.5,"automated_coverage":0.5,"conditional_correctness":0.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.5,"accuracy_wilson_95":{"lower":0.0,"upper":0.32440756488388023,"confidence_level":0.95}},"effective":{"n":8,"acceptable":0,"unsafe":0,"review":8,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":8,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.32440756488388023,"confidence_level":0.95}}},"drop":{"n":22,"raw":{"n":22,"acceptable":17,"unsafe":5,"review":0,"automated_count":22,"automated_acceptable":17,"automated_unsafe":5,"unnecessary_reviews":0,"accuracy":0.7727272727272727,"unsafe_rate":0.22727272727272727,"review_rate":0.0,"automated_coverage":1.0,"conditional_correctness":0.7727272727272727,"conditional_unsafe":0.22727272727272727,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.5656004682477829,"upper":0.8987696001475147,"confidence_level":0.95}},"effective":{"n":22,"acceptable":3,"unsafe":0,"review":19,"automated_count":3,"automated_acceptable":3,"automated_unsafe":0,"unnecessary_reviews":19,"accuracy":0.13636363636363635,"unsafe_rate":0.0,"review_rate":0.8636363636363636,"automated_coverage":0.13636363636363635,"conditional_correctness":1.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.8636363636363636,"accuracy_wilson_95":{"lower":0.04749003714055253,"upper":0.33334987166571717,"confidence_level":0.95}}},"mask":{"n":8,"raw":{"n":8,"acceptable":0,"unsafe":1,"review":1,"automated_count":7,"automated_acceptable":0,"automated_unsafe":1,"unnecessary_reviews":1,"accuracy":0.0,"unsafe_rate":0.125,"review_rate":0.125,"automated_coverage":0.875,"conditional_correctness":0.0,"conditional_unsafe":0.14285714285714285,"unnecessary_review_rate":0.125,"accuracy_wilson_95":{"lower":0.0,"upper":0.32440756488388023,"confidence_level":0.95}},"effective":{"n":8,"acceptable":0,"unsafe":0,"review":8,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":8,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.32440756488388023,"confidence_level":0.95}}},"pseudonymize":{"n":8,"raw":{"n":8,"acceptable":2,"unsafe":0,"review":1,"automated_count":7,"automated_acceptable":2,"automated_unsafe":0,"unnecessary_reviews":1,"accuracy":0.25,"unsafe_rate":0.0,"review_rate":0.125,"automated_coverage":0.875,"conditional_correctness":0.2857142857142857,"conditional_unsafe":0.0,"unnecessary_review_rate":0.125,"accuracy_wilson_95":{"lower":0.071479212752109,"upper":0.5907245696898311,"confidence_level":0.95}},"effective":{"n":8,"acceptable":0,"unsafe":0,"review":8,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":8,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.32440756488388023,"confidence_level":0.95}}},"generalize":{"n":8,"raw":{"n":8,"acceptable":0,"unsafe":0,"review":8,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":8,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.32440756488388023,"confidence_level":0.95}},"effective":{"n":8,"acceptable":0,"unsafe":0,"review":8,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":8,"accuracy":0.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":1.0,"accuracy_wilson_95":{"lower":0.0,"upper":0.32440756488388023,"confidence_level":0.95}}},"review":{"n":2,"raw":{"n":2,"acceptable":1,"unsafe":0,"review":1,"automated_count":1,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":0.5,"unsafe_rate":0.0,"review_rate":0.5,"automated_coverage":0.5,"conditional_correctness":0.0,"conditional_unsafe":0.0,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.09453120573423074,"upper":0.9054687942657693,"confidence_level":0.95}},"effective":{"n":2,"acceptable":2,"unsafe":0,"review":2,"automated_count":0,"automated_acceptable":0,"automated_unsafe":0,"unnecessary_reviews":0,"accuracy":1.0,"unsafe_rate":0.0,"review_rate":1.0,"automated_coverage":0.0,"conditional_correctness":null,"conditional_unsafe":null,"unnecessary_review_rate":0.0,"accuracy_wilson_95":{"lower":0.34238022750665303,"upper":1.0,"confidence_level":0.95}}}}}},"composite":{"n":198,"scope":"descriptive sum across four correlated panels","raw":{"n":198,"acceptable":65,"unsafe":8,"review":78,"automated_count":120,"automated_acceptable":60,"automated_unsafe":8,"unnecessary_reviews":73,"accuracy":0.3282828282828283,"unsafe_rate":0.04040404040404041,"review_rate":0.3939393939393939,"automated_coverage":0.6060606060606061,"conditional_correctness":0.5,"conditional_unsafe":0.06666666666666667,"unnecessary_review_rate":0.3686868686868687},"effective":{"n":198,"acceptable":27,"unsafe":0,"review":178,"automated_count":20,"automated_acceptable":14,"automated_unsafe":0,"unnecessary_reviews":165,"accuracy":0.13636363636363635,"unsafe_rate":0.0,"review_rate":0.898989898989899,"automated_coverage":0.10101010101010101,"conditional_correctness":0.7,"conditional_unsafe":0.0,"unnecessary_review_rate":0.8333333333333334}}},{"id":"openrouter-mercury","platform":"OpenRouter","model_id":"inception/mercury-decide:free","display_name":"Mercury Decide","revision":"inception/mercury-decide-20260930","loading":"Provider undisclosed","scoring_kind":"Native hosted Choice","settings":{"guard_threshold":0.45,"interface":"native_choice","guard_signal":"selected Choice probability","provider":"Inception","weights_revision":null,"dtype":null,"quantization":null},"scoring_metadata":{"probability_source":"returned categorical Choice probabilities","latency_scope":"client wall-clock network-inclusive request roundtrip; provider compute time unavailable","equal_mass_convention":"balanced larger-first chunks; minimum two rows"},"panels":{"diagnostic43-no_samples":{"n":43,"raw":{"n":43,"acceptable":43,"accuracy":1.0,"unsafe":0,"unsafe_rate":0.0,"review":0,"review_rate":0.0,"automated_count":43,"automated_acceptable":43,"automated_coverage":1.0,"conditional_correctness":1,"unnecessary_reviews":0,"unnecessary_review_rate":0.0},"effective":{"n":43,"acceptable":42,"accuracy":0.9767441860465116,"unsafe":0,"unsafe_rate":0.0,"review":1,"review_rate":0.023255813953488372,"automated_count":42,"automated_acceptable":42,"automated_coverage":0.9767441860465116,"conditional_correctness":1,"unnecessary_reviews":1,"unnecessary_review_rate":0.023255813953488372},"latency_ms":{"n":43,"median":311.259963986231,"p90_nearest_rank":378.51097699604},"probability_metrics":{"singleton_n":38,"multiacceptable_n":5,"nll":0.00020657017560691047,"brier":1.5929022794584373e-07},"calibration_singleton_labels":{"equal_width":{"n":38,"ece":0.00020652220375361185},"equal_mass":{"n":38,"ece":0.00020652220375357678}},"hosted_aggregate_diagnostics":{"n":43,"unique_label_n":38,"multi_acceptable_n":5,"raw":{"n":43,"acceptable":43,"unsafe":0,"incorrect_unlabeled_unsafe":0,"reviews":0,"unnecessary_reviews":0,"automatic":43,"coverage":1.0,"automatic_acceptable":43,"automatic_conditional_accuracy":1},"guarded":{"n":43,"acceptable":42,"unsafe":0,"incorrect_unlabeled_unsafe":1,"reviews":1,"unnecessary_reviews":1,"automatic":42,"coverage":0.9767441860465116,"automatic_acceptable":42,"automatic_conditional_accuracy":1},"network_roundtrip":{"n":43,"minimum_ms":243.4718880103901,"median_ms":311.259963986231,"p90_nearest_rank_ms":378.51097699604,"maximum_ms":672.030770976562},"categorical_probability_metrics":{"unique_nll":0.00020657017560691047,"unique_brier_sum":1.5929022794584373e-07,"multi_acceptable_mass":0.9993966639404267,"multi_acceptable_event_logloss":0.0006036674273035885,"mean_unsafe_probability_mass":8.46031348234961e-05},"categorical_calibration":{"ece_width10":0.00020652220375361185,"ece_mass10_min2":0.00020652220375357678,"signed_gap":-0.00020652220375358264,"absolute_gap":0.00020652220375358264,"width_bins":[{"n":0,"mean_confidence":null,"accuracy":null,"min":null,"max":null},{"n":0,"mean_confidence":null,"accuracy":null,"min":null,"max":null},{"n":0,"mean_confidence":null,"accuracy":null,"min":null,"max":null},{"n":0,"mean_confidence":null,"accuracy":null,"min":null,"max":null},{"n":0,"mean_confidence":null,"accuracy":null,"min":null,"max":null},{"n":0,"mean_confidence":null,"accuracy":null,"min":null,"max":null},{"n":0,"mean_confidence":null,"accuracy":null,"min":null,"max":null},{"n":0,"mean_confidence":null,"accuracy":null,"min":null,"max":null},{"n":0,"mean_confidence":null,"accuracy":null,"min":null,"max":null},{"n":38,"mean_confidence":0.9997934777962464,"accuracy":1,"min":0.9988889570023414,"max":0.9999853315952676}],"mass_bins":[{"n":4,"mean_confidence":0.999246494757517,"accuracy":1,"min":0.9988889570023414,"max":0.999382606884378},{"n":4,"mean_confidence":0.9996196810783172,"accuracy":1,"min":0.999455881081185,"max":0.9997380278591247},{"n":4,"mean_confidence":0.9997558954132494,"accuracy":1,"min":0.9997406501580721,"max":0.9997743458716069},{"n":4,"mean_confidence":0.9997947025936413,"accuracy":1,"min":0.9997792954232694,"max":0.9998095254177564},{"n":4,"mean_confidence":0.999839548331809,"accuracy":1,"min":0.9998204574086055,"max":0.9998672409156898},{"n":4,"mean_confidence":0.9999129367286097,"accuracy":1,"min":0.9999015961990471,"max":0.9999218613707235},{"n":4,"mean_confidence":0.999937968436245,"accuracy":1,"min":0.9999294997229851,"max":0.9999426975290119},{"n":4,"mean_confidence":0.9999638154934463,"accuracy":1,"min":0.999960188829643,"max":0.999968180037468},{"n":3,"mean_confidence":0.9999720860475564,"accuracy":1,"min":0.999969306122563,"max":0.9999748694373245},{"n":3,"mean_confidence":0.999983908927785,"accuracy":1,"min":0.9999819615457723,"max":0.9999853315952676}]},"independent_suitability_score_diagnostics":null,"probability_status":"Native categorical distribution preserved exactly.","nll_zero_gold_mass_n":0}},"diagnostic43-compact_no_samples":{"n":43,"raw":{"n":43,"acceptable":43,"accuracy":1.0,"unsafe":0,"unsafe_rate":0.0,"review":0,"review_rate":0.0,"automated_count":43,"automated_acceptable":43,"automated_coverage":1.0,"conditional_correctness":1,"unnecessary_reviews":0,"unnecessary_review_rate":0.0},"effective":{"n":43,"acceptable":42,"accuracy":0.9767441860465116,"unsafe":0,"unsafe_rate":0.0,"review":1,"review_rate":0.023255813953488372,"automated_count":42,"automated_acceptable":42,"automated_coverage":0.9767441860465116,"conditional_correctness":1,"unnecessary_reviews":1,"unnecessary_review_rate":0.023255813953488372},"latency_ms":{"n":43,"median":298.8322570163291,"p90_nearest_rank":410.6341800070368},"probability_metrics":{"singleton_n":38,"multiacceptable_n":5,"nll":0.00012462231489759169,"brier":5.71833888786833e-08},"calibration_singleton_labels":{"equal_width":{"n":38,"ece":0.0001246058229644742},"equal_mass":{"n":38,"ece":0.00012460582296444498}},"hosted_aggregate_diagnostics":{"n":43,"unique_label_n":38,"multi_acceptable_n":5,"raw":{"n":43,"acceptable":43,"unsafe":0,"incorrect_unlabeled_unsafe":0,"reviews":0,"unnecessary_reviews":0,"automatic":43,"coverage":1.0,"automatic_acceptable":43,"automatic_conditional_accuracy":1},"guarded":{"n":43,"acceptable":42,"unsafe":0,"incorrect_unlabeled_unsafe":1,"reviews":1,"unnecessary_reviews":1,"automatic":42,"coverage":0.9767441860465116,"automatic_acceptable":42,"automatic_conditional_accuracy":1},"network_roundtrip":{"n":43,"minimum_ms":236.59758298890665,"median_ms":298.8322570163291,"p90_nearest_rank_ms":410.6341800070368,"maximum_ms":1340.0275970052462},"categorical_probability_metrics":{"unique_nll":0.00012462231489759169,"unique_brier_sum":5.71833888786833e-08,"multi_acceptable_mass":0.9960352521960374,"multi_acceptable_event_logloss":0.003981363659332707,"mean_unsafe_probability_mass":0.0004703304573065857},"categorical_calibration":{"ece_width10":0.0001246058229644742,"ece_mass10_min2":0.00012460582296444498,"signed_gap":-0.00012460582296442454,"absolute_gap":0.00012460582296442454,"width_bins":[{"n":0,"mean_confidence":null,"accuracy":null,"min":null,"max":null},{"n":0,"mean_confidence":null,"accuracy":null,"min":null,"max":null},{"n":0,"mean_confidence":null,"accuracy":null,"min":null,"max":null},{"n":0,"mean_confidence":null,"accuracy":null,"min":null,"max":null},{"n":0,"mean_confidence":null,"accuracy":null,"min":null,"max":null},{"n":0,"mean_confidence":null,"accuracy":null,"min":null,"max":null},{"n":0,"mean_confidence":null,"accuracy":null,"min":null,"max":null},{"n":0,"mean_confidence":null,"accuracy":null,"min":null,"max":null},{"n":0,"mean_confidence":null,"accuracy":null,"min":null,"max":null},{"n":38,"mean_confidence":0.9998753941770355,"accuracy":1,"min":0.9992510907446928,"max":0.9999824771899825}],"mass_bins":[{"n":4,"mean_confidence":0.9995977794220615,"accuracy":1,"min":0.9992510907446928,"max":0.9997562042070879},{"n":4,"mean_confidence":0.9997741339385899,"accuracy":1,"min":0.9997593637170046,"max":0.9998152373039083},{"n":4,"mean_confidence":0.9998326936447689,"accuracy":1,"min":0.9998158642026278,"max":0.9998474819386791},{"n":4,"mean_confidence":0.9998823048821972,"accuracy":1,"min":0.9998647133456481,"max":0.9998951536972883},{"n":4,"mean_confidence":0.9999088208052677,"accuracy":1,"min":0.9998968638671385,"max":0.9999230155818574},{"n":4,"mean_confidence":0.9999340506496321,"accuracy":1,"min":0.999925576776588,"max":0.9999459963747679},{"n":4,"mean_confidence":0.999952433002876,"accuracy":1,"min":0.9999480328098198,"max":0.9999580117199349},{"n":4,"mean_confidence":0.9999674358350754,"accuracy":1,"min":0.9999647675934168,"max":0.9999694887085346},{"n":3,"mean_confidence":0.9999744823105022,"accuracy":1,"min":0.9999715011962915,"max":0.9999766755979957},{"n":3,"mean_confidence":0.9999809743579898,"accuracy":1,"min":0.9999784787350454,"max":0.9999824771899825}]},"independent_suitability_score_diagnostics":null,"probability_status":"Native categorical distribution preserved exactly.","nll_zero_gold_mass_n":0}},"challenge56-no_samples":{"n":56,"raw":{"n":56,"acceptable":56,"accuracy":1.0,"unsafe":0,"unsafe_rate":0.0,"review":2,"review_rate":0.03571428571428571,"automated_count":54,"automated_acceptable":54,"automated_coverage":0.9642857142857143,"conditional_correctness":1,"unnecessary_reviews":0,"unnecessary_review_rate":0.0},"effective":{"n":56,"acceptable":46,"accuracy":0.8214285714285714,"unsafe":0,"unsafe_rate":0.0,"review":12,"review_rate":0.21428571428571427,"automated_count":44,"automated_acceptable":44,"automated_coverage":0.7857142857142857,"conditional_correctness":1,"unnecessary_reviews":10,"unnecessary_review_rate":0.17857142857142858},"latency_ms":{"n":56,"median":299.0292265021708,"p90_nearest_rank":412.3686930106487},"probability_metrics":{"singleton_n":56,"multiacceptable_n":0,"nll":0.003772319806548354,"brier":0.00019940256984896074},"calibration_singleton_labels":{"equal_width":{"n":56,"ece":0.0037141858274645223},"equal_mass":{"n":56,"ece":0.003714185827464552}},"hosted_aggregate_diagnostics":{"n":56,"unique_label_n":56,"multi_acceptable_n":0,"raw":{"n":56,"acceptable":56,"unsafe":0,"incorrect_unlabeled_unsafe":0,"reviews":2,"unnecessary_reviews":0,"automatic":54,"coverage":0.9642857142857143,"automatic_acceptable":54,"automatic_conditional_accuracy":1},"guarded":{"n":56,"acceptable":46,"unsafe":0,"incorrect_unlabeled_unsafe":10,"reviews":12,"unnecessary_reviews":10,"automatic":44,"coverage":0.7857142857142857,"automatic_acceptable":44,"automatic_conditional_accuracy":1},"network_roundtrip":{"n":56,"minimum_ms":230.60417000669986,"median_ms":299.0292265021708,"p90_nearest_rank_ms":412.3686930106487,"maximum_ms":648.9052230026573},"categorical_probability_metrics":{"unique_nll":0.003772319806548354,"unique_brier_sum":0.00019940256984896074,"multi_acceptable_mass":null,"multi_acceptable_event_logloss":null,"mean_unsafe_probability_mass":0.00012690049326342776},"categorical_calibration":{"ece_width10":0.0037141858274645223,"ece_mass10_min2":0.003714185827464552,"signed_gap":-0.003714185827464534,"absolute_gap":0.003714185827464534,"width_bins":[{"n":0,"mean_confidence":null,"accuracy":null,"min":null,"max":null},{"n":0,"mean_confidence":null,"accuracy":null,"min":null,"max":null},{"n":0,"mean_confidence":null,"accuracy":null,"min":null,"max":null},{"n":0,"mean_confidence":null,"accuracy":null,"min":null,"max":null},{"n":0,"mean_confidence":null,"accuracy":null,"min":null,"max":null},{"n":0,"mean_confidence":null,"accuracy":null,"min":null,"max":null},{"n":0,"mean_confidence":null,"accuracy":null,"min":null,"max":null},{"n":0,"mean_confidence":null,"accuracy":null,"min":null,"max":null},{"n":0,"mean_confidence":null,"accuracy":null,"min":null,"max":null},{"n":56,"mean_confidence":0.9962858141725355,"accuracy":1,"min":0.9515352597489799,"max":0.9999880552436667}],"mass_bins":[{"n":6,"mean_confidence":0.9705460226095848,"accuracy":1,"min":0.9515352597489799,"max":0.9874329601596649},{"n":6,"mean_confidence":0.9962432482052038,"accuracy":1,"min":0.9897549736372504,"max":0.9990100833595782},{"n":6,"mean_confidence":0.9993514954598713,"accuracy":1,"min":0.9992334073145959,"max":0.9995492187425724},{"n":6,"mean_confidence":0.999669243571902,"accuracy":1,"min":0.9995787634675145,"max":0.9997201328420561},{"n":6,"mean_confidence":0.9997749826114593,"accuracy":1,"min":0.9997477399429975,"max":0.9998004176002923},{"n":6,"mean_confidence":0.9998651482405142,"accuracy":1,"min":0.9998079222310667,"max":0.9999234793348526},{"n":5,"mean_confidence":0.9999424197723925,"accuracy":1,"min":0.9999253149277326,"max":0.9999568574511933},{"n":5,"mean_confidence":0.9999636286967601,"accuracy":1,"min":0.9999602141702743,"max":0.9999675549019973},{"n":5,"mean_confidence":0.9999719860773206,"accuracy":1,"min":0.9999676047786267,"max":0.9999799374758671},{"n":5,"mean_confidence":0.9999829153476812,"accuracy":1,"min":0.9999802378464115,"max":0.9999880552436667}]},"independent_suitability_score_diagnostics":null,"probability_status":"Native categorical distribution preserved exactly.","nll_zero_gold_mass_n":0}},"challenge56-compact_no_samples":{"n":56,"raw":{"n":56,"acceptable":56,"accuracy":1.0,"unsafe":0,"unsafe_rate":0.0,"review":2,"review_rate":0.03571428571428571,"automated_count":54,"automated_acceptable":54,"automated_coverage":0.9642857142857143,"conditional_correctness":1,"unnecessary_reviews":0,"unnecessary_review_rate":0.0},"effective":{"n":56,"acceptable":46,"accuracy":0.8214285714285714,"unsafe":0,"unsafe_rate":0.0,"review":12,"review_rate":0.21428571428571427,"automated_count":44,"automated_acceptable":44,"automated_coverage":0.7857142857142857,"conditional_correctness":1,"unnecessary_reviews":10,"unnecessary_review_rate":0.17857142857142858},"latency_ms":{"n":56,"median":297.0818050089292,"p90_nearest_rank":355.80978699726984},"probability_metrics":{"singleton_n":56,"multiacceptable_n":0,"nll":0.0015026429951824874,"brier":2.5010470048193e-05},"calibration_singleton_labels":{"equal_width":{"n":56,"ece":0.0014962270666993849},"equal_mass":{"n":56,"ece":0.001496227066699383}},"hosted_aggregate_diagnostics":{"n":56,"unique_label_n":56,"multi_acceptable_n":0,"raw":{"n":56,"acceptable":56,"unsafe":0,"incorrect_unlabeled_unsafe":0,"reviews":2,"unnecessary_reviews":0,"automatic":54,"coverage":0.9642857142857143,"automatic_acceptable":54,"automatic_conditional_accuracy":1},"guarded":{"n":56,"acceptable":46,"unsafe":0,"incorrect_unlabeled_unsafe":10,"reviews":12,"unnecessary_reviews":10,"automatic":44,"coverage":0.7857142857142857,"automatic_acceptable":44,"automatic_conditional_accuracy":1},"network_roundtrip":{"n":56,"minimum_ms":235.2669920073822,"median_ms":297.0818050089292,"p90_nearest_rank_ms":355.80978699726984,"maximum_ms":834.9446080101188},"categorical_probability_metrics":{"unique_nll":0.0015026429951824874,"unique_brier_sum":2.5010470048193e-05,"multi_acceptable_mass":null,"multi_acceptable_event_logloss":null,"mean_unsafe_probability_mass":3.962457514489283e-05},"categorical_calibration":{"ece_width10":0.0014962270666993849,"ece_mass10_min2":0.001496227066699383,"signed_gap":-0.0014962270666993769,"absolute_gap":0.0014962270666993769,"width_bins":[{"n":0,"mean_confidence":null,"accuracy":null,"min":null,"max":null},{"n":0,"mean_confidence":null,"accuracy":null,"min":null,"max":null},{"n":0,"mean_confidence":null,"accuracy":null,"min":null,"max":null},{"n":0,"mean_confidence":null,"accuracy":null,"min":null,"max":null},{"n":0,"mean_confidence":null,"accuracy":null,"min":null,"max":null},{"n":0,"mean_confidence":null,"accuracy":null,"min":null,"max":null},{"n":0,"mean_confidence":null,"accuracy":null,"min":null,"max":null},{"n":0,"mean_confidence":null,"accuracy":null,"min":null,"max":null},{"n":0,"mean_confidence":null,"accuracy":null,"min":null,"max":null},{"n":56,"mean_confidence":0.9985037729333006,"accuracy":1,"min":0.9873516845062414,"max":0.9999844745233636}],"mass_bins":[{"n":6,"mean_confidence":0.9900026197049208,"accuracy":1,"min":0.9873516845062414,"max":0.9912709470269161},{"n":6,"mean_confidence":0.9966984542494163,"accuracy":1,"min":0.9923144418190144,"max":0.9986183001790835},{"n":6,"mean_confidence":0.9996409872001626,"accuracy":1,"min":0.9991180997243383,"max":0.9998858711041776},{"n":6,"mean_confidence":0.9999074414115674,"accuracy":1,"min":0.9998869384207155,"max":0.9999260311270707},{"n":6,"mean_confidence":0.9999376093155081,"accuracy":1,"min":0.9999310369013297,"max":0.9999477027744262},{"n":6,"mean_confidence":0.999953015573293,"accuracy":1,"min":0.9999499099627982,"max":0.9999563569250517},{"n":5,"mean_confidence":0.999957897977966,"accuracy":1,"min":0.9999565724799507,"max":0.9999588974339981},{"n":5,"mean_confidence":0.9999632991237444,"accuracy":1,"min":0.9999592791318843,"max":0.9999670261367176},{"n":5,"mean_confidence":0.9999713271236925,"accuracy":1,"min":0.9999693384486024,"max":0.9999740175011447},{"n":5,"mean_confidence":0.9999815796817223,"accuracy":1,"min":0.9999770496283064,"max":0.9999844745233636}]},"independent_suitability_score_diagnostics":null,"probability_status":"Native categorical distribution preserved exactly.","nll_zero_gold_mass_n":0}}},"composite":{"n":198,"scope":"descriptive sum across four correlated panels","raw":{"n":198,"acceptable":198,"accuracy":1.0,"unsafe":0,"unsafe_rate":0.0,"review":4,"review_rate":0.020202020202020204,"automated_count":194,"automated_acceptable":194,"automated_coverage":0.9797979797979798,"conditional_correctness":1,"unnecessary_reviews":0,"unnecessary_review_rate":0.0},"effective":{"n":198,"acceptable":176,"accuracy":0.8888888888888888,"unsafe":0,"unsafe_rate":0.0,"review":26,"review_rate":0.13131313131313133,"automated_count":172,"automated_acceptable":172,"automated_coverage":0.8686868686868687,"conditional_correctness":1,"unnecessary_reviews":22,"unnecessary_review_rate":0.1111111111111111}},"recorded_whole_run_latency_ms":{"n":198,"median":299.66990450338926,"p90_nearest_rank":405.0728230213281},"benchmark_reported_cost_usd":0,"hosted_run_metadata":{"key":"mercury","name":"Mercury Decide","requested_model_id":"inception/mercury-decide:free","returned_model_id":"inception/mercury-decide-20260930","provider":"Inception","endpoint":"https://openrouter.ai/api/v1/systemone","interface":"native_choice","prediction_count":198,"benchmark_http_attempts":198,"benchmark_retry_count":0,"benchmark_usage_complete":true,"benchmark_input_tokens":65754,"benchmark_output_tokens":588,"benchmark_reported_cost_usd":0,"observed_input_price_usd_per_million":0,"observed_output_price_usd_per_million":0,"known_success_probe_cost_usd":0,"failed_interface_probes_without_usage":0,"probe_billing_note":"Mercury preliminary interface discovery is not included in benchmark journals; complete historical discovery expenditure cannot be reconstructed from this export.","network_roundtrip":{"n":198,"minimum_ms":230.60417000669986,"median_ms":299.66990450338926,"p90_nearest_rank_ms":405.0728230213281,"maximum_ms":1340.0275970052462},"per_request_roundtrip_available":true,"provider_compute_timing_ms":null,"provider_compute_timing_status":"not returned","per_job_walltime_available":false,"load_warmup_status":"Hosted service; no controllable model load/warmup measurement. D1/Solar have one recorded interface probe before benchmark; full Span has one successful Noul probe after a Choice rejection.","recorded_start_utc":null,"recorded_end_utc":null,"source_prediction_journal_sha256":"ee6aecdc16503750e5020b7383c932cff58ebf98093fab9504780c4198666754","job_walltime_including_retries":null}},{"id":"openrouter-jev","platform":"OpenRouter","model_id":"typesafe/jev-1.13","display_name":"Jev 1.13","revision":"typesafe/jev-1.13-20260917","loading":"Provider undisclosed","scoring_kind":"Native hosted Choice","settings":{"guard_threshold":0.45,"interface":"native_choice","guard_signal":"selected Choice probability","provider":"TypeSafe","weights_revision":null,"dtype":null,"quantization":null},"scoring_metadata":{"probability_source":"returned categorical Choice probabilities","latency_scope":"client wall-clock network-inclusive request roundtrip; provider compute time unavailable","equal_mass_convention":"balanced larger-first chunks; minimum two rows"},"panels":{"diagnostic43-no_samples":{"n":43,"raw":{"n":43,"acceptable":43,"accuracy":1.0,"unsafe":0,"unsafe_rate":0.0,"review":3,"review_rate":0.06976744186046512,"automated_count":40,"automated_acceptable":40,"automated_coverage":0.9302325581395349,"conditional_correctness":1,"unnecessary_reviews":0,"unnecessary_review_rate":0.0},"effective":{"n":43,"acceptable":42,"accuracy":0.9767441860465116,"unsafe":0,"unsafe_rate":0.0,"review":4,"review_rate":0.09302325581395349,"automated_count":39,"automated_acceptable":39,"automated_coverage":0.9069767441860465,"conditional_correctness":1,"unnecessary_reviews":1,"unnecessary_review_rate":0.023255813953488372},"latency_ms":{"n":43,"median":236.05009203311056,"p90_nearest_rank":383.23856401257217},"probability_metrics":{"singleton_n":38,"multiacceptable_n":5,"nll":0.028673014399510033,"brier":0.011263157894736843},"calibration_singleton_labels":{"equal_width":{"n":38,"ece":0.02342105263157895},"equal_mass":{"n":38,"ece":0.02342105263157896}},"hosted_aggregate_diagnostics":{"n":43,"unique_label_n":38,"multi_acceptable_n":5,"raw":{"n":43,"acceptable":43,"unsafe":0,"incorrect_unlabeled_unsafe":0,"reviews":3,"unnecessary_reviews":0,"automatic":40,"coverage":0.9302325581395349,"automatic_acceptable":40,"automatic_conditional_accuracy":1},"guarded":{"n":43,"acceptable":42,"unsafe":0,"incorrect_unlabeled_unsafe":1,"reviews":4,"unnecessary_reviews":1,"automatic":39,"coverage":0.9069767441860465,"automatic_acceptable":39,"automatic_conditional_accuracy":1},"network_roundtrip":{"n":43,"minimum_ms":181.89432000508532,"median_ms":236.05009203311056,"p90_nearest_rank_ms":383.23856401257217,"maximum_ms":1496.8284289934672},"categorical_probability_metrics":{"unique_nll":0.028673014399510033,"unique_brier_sum":0.011263157894736843,"multi_acceptable_mass":0.998,"multi_acceptable_event_logloss":0.0020100671707003344,"mean_unsafe_probability_mass":0.0034883720930232558},"categorical_calibration":{"ece_width10":0.02342105263157895,"ece_mass10_min2":0.02342105263157896,"signed_gap":-0.023421052631578957,"absolute_gap":0.023421052631578957,"width_bins":[{"n":0,"mean_confidence":null,"accuracy":null,"min":null,"max":null},{"n":0,"mean_confidence":null,"accuracy":null,"min":null,"max":null},{"n":0,"mean_confidence":null,"accuracy":null,"min":null,"max":null},{"n":0,"mean_confidence":null,"accuracy":null,"min":null,"max":null},{"n":0,"mean_confidence":null,"accuracy":null,"min":null,"max":null},{"n":1,"mean_confidence":0.5,"accuracy":1,"min":0.5,"max":0.5},{"n":0,"mean_confidence":null,"accuracy":null,"min":null,"max":null},{"n":0,"mean_confidence":null,"accuracy":null,"min":null,"max":null},{"n":0,"mean_confidence":null,"accuracy":null,"min":null,"max":null},{"n":37,"mean_confidence":0.9894594594594595,"accuracy":1,"min":0.94,"max":1}],"mass_bins":[{"n":4,"mean_confidence":0.8374999999999999,"accuracy":1,"min":0.5,"max":0.96},{"n":4,"mean_confidence":0.975,"accuracy":1,"min":0.97,"max":0.98},{"n":4,"mean_confidence":0.98,"accuracy":1,"min":0.98,"max":0.98},{"n":4,"mean_confidence":0.99,"accuracy":1,"min":0.99,"max":0.99},{"n":4,"mean_confidence":0.995,"accuracy":1,"min":0.99,"max":1},{"n":4,"mean_confidence":1,"accuracy":1,"min":1,"max":1},{"n":4,"mean_confidence":1,"accuracy":1,"min":1,"max":1},{"n":4,"mean_confidence":1,"accuracy":1,"min":1,"max":1},{"n":3,"mean_confidence":1,"accuracy":1,"min":1,"max":1},{"n":3,"mean_confidence":1,"accuracy":1,"min":1,"max":1}]},"independent_suitability_score_diagnostics":null,"probability_status":"Native categorical distribution preserved exactly.","nll_zero_gold_mass_n":0}},"diagnostic43-compact_no_samples":{"n":43,"raw":{"n":43,"acceptable":43,"accuracy":1.0,"unsafe":0,"unsafe_rate":0.0,"review":1,"review_rate":0.023255813953488372,"automated_count":42,"automated_acceptable":42,"automated_coverage":0.9767441860465116,"conditional_correctness":1,"unnecessary_reviews":0,"unnecessary_review_rate":0.0},"effective":{"n":43,"acceptable":42,"accuracy":0.9767441860465116,"unsafe":0,"unsafe_rate":0.0,"review":2,"review_rate":0.046511627906976744,"automated_count":41,"automated_acceptable":41,"automated_coverage":0.9534883720930233,"conditional_correctness":1,"unnecessary_reviews":1,"unnecessary_review_rate":0.023255813953488372},"latency_ms":{"n":43,"median":230.299673974514,"p90_nearest_rank":287.8080310183577},"probability_metrics":{"singleton_n":38,"multiacceptable_n":5,"nll":0.047577140841098216,"brier":0.018263157894736842},"calibration_singleton_labels":{"equal_width":{"n":38,"ece":0.04078947368421057},"equal_mass":{"n":38,"ece":0.04078947368421053}},"hosted_aggregate_diagnostics":{"n":43,"unique_label_n":38,"multi_acceptable_n":5,"raw":{"n":43,"acceptable":43,"unsafe":0,"incorrect_unlabeled_unsafe":0,"reviews":1,"unnecessary_reviews":0,"automatic":42,"coverage":0.9767441860465116,"automatic_acceptable":42,"automatic_conditional_accuracy":1},"guarded":{"n":43,"acceptable":42,"unsafe":0,"incorrect_unlabeled_unsafe":1,"reviews":2,"unnecessary_reviews":1,"automatic":41,"coverage":0.9534883720930233,"automatic_acceptable":41,"automatic_conditional_accuracy":1},"network_roundtrip":{"n":43,"minimum_ms":187.58676096331328,"median_ms":230.299673974514,"p90_nearest_rank_ms":287.8080310183577,"maximum_ms":1021.7997659929097},"categorical_probability_metrics":{"unique_nll":0.047577140841098216,"unique_brier_sum":0.018263157894736842,"multi_acceptable_mass":1.0,"multi_acceptable_event_logloss":0.0,"mean_unsafe_probability_mass":0.0016279069767441861},"categorical_calibration":{"ece_width10":0.04078947368421057,"ece_mass10_min2":0.04078947368421053,"signed_gap":-0.04078947368421053,"absolute_gap":0.04078947368421053,"width_bins":[{"n":0,"mean_confidence":null,"accuracy":null,"min":null,"max":null},{"n":0,"mean_confidence":null,"accuracy":null,"min":null,"max":null},{"n":0,"mean_confidence":null,"accuracy":null,"min":null,"max":null},{"n":0,"mean_confidence":null,"accuracy":null,"min":null,"max":null},{"n":0,"mean_confidence":null,"accuracy":null,"min":null,"max":null},{"n":0,"mean_confidence":null,"accuracy":null,"min":null,"max":null},{"n":2,"mean_confidence":0.645,"accuracy":1,"min":0.62,"max":0.67},{"n":2,"mean_confidence":0.745,"accuracy":1,"min":0.7,"max":0.79},{"n":0,"mean_confidence":null,"accuracy":null,"min":null,"max":null},{"n":34,"mean_confidence":0.9902941176470588,"accuracy":1,"min":0.92,"max":1}],"mass_bins":[{"n":4,"mean_confidence":0.6950000000000001,"accuracy":1,"min":0.62,"max":0.79},{"n":4,"mean_confidence":0.95,"accuracy":1,"min":0.92,"max":0.97},{"n":4,"mean_confidence":0.9824999999999999,"accuracy":1,"min":0.97,"max":0.99},{"n":4,"mean_confidence":0.99,"accuracy":1,"min":0.99,"max":0.99},{"n":4,"mean_confidence":0.995,"accuracy":1,"min":0.99,"max":1},{"n":4,"mean_confidence":1,"accuracy":1,"min":1,"max":1},{"n":4,"mean_confidence":1,"accuracy":1,"min":1,"max":1},{"n":4,"mean_confidence":1,"accuracy":1,"min":1,"max":1},{"n":3,"mean_confidence":1,"accuracy":1,"min":1,"max":1},{"n":3,"mean_confidence":1,"accuracy":1,"min":1,"max":1}]},"independent_suitability_score_diagnostics":null,"probability_status":"Native categorical distribution preserved exactly.","nll_zero_gold_mass_n":0}},"challenge56-no_samples":{"n":56,"raw":{"n":56,"acceptable":56,"accuracy":1.0,"unsafe":0,"unsafe_rate":0.0,"review":2,"review_rate":0.03571428571428571,"automated_count":54,"automated_acceptable":54,"automated_coverage":0.9642857142857143,"conditional_correctness":1,"unnecessary_reviews":0,"unnecessary_review_rate":0.0},"effective":{"n":56,"acceptable":46,"accuracy":0.8214285714285714,"unsafe":0,"unsafe_rate":0.0,"review":12,"review_rate":0.21428571428571427,"automated_count":44,"automated_acceptable":44,"automated_coverage":0.7857142857142857,"conditional_correctness":1,"unnecessary_reviews":10,"unnecessary_review_rate":0.17857142857142858},"latency_ms":{"n":56,"median":221.78294698824175,"p90_nearest_rank":275.0757810426876},"probability_metrics":{"singleton_n":56,"multiacceptable_n":0,"nll":0.012975768614108249,"brier":0.000953571428571429},"calibration_singleton_labels":{"equal_width":{"n":56,"ece":0.012678571428571428},"equal_mass":{"n":56,"ece":0.012678571428571435}},"hosted_aggregate_diagnostics":{"n":56,"unique_label_n":56,"multi_acceptable_n":0,"raw":{"n":56,"acceptable":56,"unsafe":0,"incorrect_unlabeled_unsafe":0,"reviews":2,"unnecessary_reviews":0,"automatic":54,"coverage":0.9642857142857143,"automatic_acceptable":54,"automatic_conditional_accuracy":1},"guarded":{"n":56,"acceptable":46,"unsafe":0,"incorrect_unlabeled_unsafe":10,"reviews":12,"unnecessary_reviews":10,"automatic":44,"coverage":0.7857142857142857,"automatic_acceptable":44,"automatic_conditional_accuracy":1},"network_roundtrip":{"n":56,"minimum_ms":185.63630996504799,"median_ms":221.78294698824175,"p90_nearest_rank_ms":275.0757810426876,"maximum_ms":1437.876631040126},"categorical_probability_metrics":{"unique_nll":0.012975768614108249,"unique_brier_sum":0.000953571428571429,"multi_acceptable_mass":null,"multi_acceptable_event_logloss":null,"mean_unsafe_probability_mass":0.0008928571428571428},"categorical_calibration":{"ece_width10":0.012678571428571428,"ece_mass10_min2":0.012678571428571435,"signed_gap":-0.012678571428571437,"absolute_gap":0.012678571428571437,"width_bins":[{"n":0,"mean_confidence":null,"accuracy":null,"min":null,"max":null},{"n":0,"mean_confidence":null,"accuracy":null,"min":null,"max":null},{"n":0,"mean_confidence":null,"accuracy":null,"min":null,"max":null},{"n":0,"mean_confidence":null,"accuracy":null,"min":null,"max":null},{"n":0,"mean_confidence":null,"accuracy":null,"min":null,"max":null},{"n":0,"mean_confidence":null,"accuracy":null,"min":null,"max":null},{"n":0,"mean_confidence":null,"accuracy":null,"min":null,"max":null},{"n":0,"mean_confidence":null,"accuracy":null,"min":null,"max":null},{"n":0,"mean_confidence":null,"accuracy":null,"min":null,"max":null},{"n":56,"mean_confidence":0.9873214285714286,"accuracy":1,"min":0.91,"max":1}],"mass_bins":[{"n":6,"mean_confidence":0.9383333333333334,"accuracy":1,"min":0.91,"max":0.95},{"n":6,"mean_confidence":0.9716666666666667,"accuracy":1,"min":0.95,"max":0.98},{"n":6,"mean_confidence":0.9883333333333333,"accuracy":1,"min":0.98,"max":0.99},{"n":6,"mean_confidence":0.99,"accuracy":1,"min":0.99,"max":0.99},{"n":6,"mean_confidence":0.9933333333333333,"accuracy":1,"min":0.99,"max":1},{"n":6,"mean_confidence":1,"accuracy":1,"min":1,"max":1},{"n":5,"mean_confidence":1,"accuracy":1,"min":1,"max":1},{"n":5,"mean_confidence":1,"accuracy":1,"min":1,"max":1},{"n":5,"mean_confidence":1,"accuracy":1,"min":1,"max":1},{"n":5,"mean_confidence":1,"accuracy":1,"min":1,"max":1}]},"independent_suitability_score_diagnostics":null,"probability_status":"Native categorical distribution preserved exactly.","nll_zero_gold_mass_n":0}},"challenge56-compact_no_samples":{"n":56,"raw":{"n":56,"acceptable":56,"accuracy":1.0,"unsafe":0,"unsafe_rate":0.0,"review":2,"review_rate":0.03571428571428571,"automated_count":54,"automated_acceptable":54,"automated_coverage":0.9642857142857143,"conditional_correctness":1,"unnecessary_reviews":0,"unnecessary_review_rate":0.0},"effective":{"n":56,"acceptable":46,"accuracy":0.8214285714285714,"unsafe":0,"unsafe_rate":0.0,"review":12,"review_rate":0.21428571428571427,"automated_count":44,"automated_acceptable":44,"automated_coverage":0.7857142857142857,"conditional_correctness":1,"unnecessary_reviews":10,"unnecessary_review_rate":0.17857142857142858},"latency_ms":{"n":56,"median":224.97721752733923,"p90_nearest_rank":361.18473001988605},"probability_metrics":{"singleton_n":56,"multiacceptable_n":0,"nll":0.010837733821888929,"brier":0.0011107142857142856},"calibration_singleton_labels":{"equal_width":{"n":56,"ece":0.010535714285714252},"equal_mass":{"n":56,"ece":0.010535714285714303}},"hosted_aggregate_diagnostics":{"n":56,"unique_label_n":56,"multi_acceptable_n":0,"raw":{"n":56,"acceptable":56,"unsafe":0,"incorrect_unlabeled_unsafe":0,"reviews":2,"unnecessary_reviews":0,"automatic":54,"coverage":0.9642857142857143,"automatic_acceptable":54,"automatic_conditional_accuracy":1},"guarded":{"n":56,"acceptable":46,"unsafe":0,"incorrect_unlabeled_unsafe":10,"reviews":12,"unnecessary_reviews":10,"automatic":44,"coverage":0.7857142857142857,"automatic_acceptable":44,"automatic_conditional_accuracy":1},"network_roundtrip":{"n":56,"minimum_ms":179.4649160001427,"median_ms":224.97721752733923,"p90_nearest_rank_ms":361.18473001988605,"maximum_ms":891.9883799972013},"categorical_probability_metrics":{"unique_nll":0.010837733821888929,"unique_brier_sum":0.0011107142857142856,"multi_acceptable_mass":null,"multi_acceptable_event_logloss":null,"mean_unsafe_probability_mass":0},"categorical_calibration":{"ece_width10":0.010535714285714252,"ece_mass10_min2":0.010535714285714303,"signed_gap":-0.010535714285714289,"absolute_gap":0.010535714285714289,"width_bins":[{"n":0,"mean_confidence":null,"accuracy":null,"min":null,"max":null},{"n":0,"mean_confidence":null,"accuracy":null,"min":null,"max":null},{"n":0,"mean_confidence":null,"accuracy":null,"min":null,"max":null},{"n":0,"mean_confidence":null,"accuracy":null,"min":null,"max":null},{"n":0,"mean_confidence":null,"accuracy":null,"min":null,"max":null},{"n":0,"mean_confidence":null,"accuracy":null,"min":null,"max":null},{"n":0,"mean_confidence":null,"accuracy":null,"min":null,"max":null},{"n":0,"mean_confidence":null,"accuracy":null,"min":null,"max":null},{"n":1,"mean_confidence":0.89,"accuracy":1,"min":0.89,"max":0.89},{"n":55,"mean_confidence":0.9912727272727273,"accuracy":1,"min":0.91,"max":1}],"mass_bins":[{"n":6,"mean_confidence":0.94,"accuracy":1,"min":0.89,"max":0.98},{"n":6,"mean_confidence":0.9833333333333333,"accuracy":1,"min":0.98,"max":0.99},{"n":6,"mean_confidence":0.99,"accuracy":1,"min":0.99,"max":0.99},{"n":6,"mean_confidence":0.99,"accuracy":1,"min":0.99,"max":0.99},{"n":6,"mean_confidence":0.9983333333333333,"accuracy":1,"min":0.99,"max":1},{"n":6,"mean_confidence":1,"accuracy":1,"min":1,"max":1},{"n":5,"mean_confidence":1,"accuracy":1,"min":1,"max":1},{"n":5,"mean_confidence":1,"accuracy":1,"min":1,"max":1},{"n":5,"mean_confidence":1,"accuracy":1,"min":1,"max":1},{"n":5,"mean_confidence":1,"accuracy":1,"min":1,"max":1}]},"independent_suitability_score_diagnostics":null,"probability_status":"Native categorical distribution preserved exactly.","nll_zero_gold_mass_n":0}}},"composite":{"n":198,"scope":"descriptive sum across four correlated panels","raw":{"n":198,"acceptable":198,"accuracy":1.0,"unsafe":0,"unsafe_rate":0.0,"review":8,"review_rate":0.04040404040404041,"automated_count":190,"automated_acceptable":190,"automated_coverage":0.9595959595959596,"conditional_correctness":1,"unnecessary_reviews":0,"unnecessary_review_rate":0.0},"effective":{"n":198,"acceptable":176,"accuracy":0.8888888888888888,"unsafe":0,"unsafe_rate":0.0,"review":30,"review_rate":0.15151515151515152,"automated_count":168,"automated_acceptable":168,"automated_coverage":0.8484848484848485,"conditional_correctness":1,"unnecessary_reviews":22,"unnecessary_review_rate":0.1111111111111111}},"recorded_whole_run_latency_ms":{"n":198,"median":228.5437495156657,"p90_nearest_rank":345.2474070363678},"benchmark_reported_cost_usd":0.005274234,"hosted_run_metadata":{"key":"jev","name":"Jev 1.13","requested_model_id":"typesafe/jev-1.13","returned_model_id":"typesafe/jev-1.13-20260917","provider":"TypeSafe","endpoint":"https://openrouter.ai/api/alpha/decisions","interface":"native_choice","prediction_count":198,"benchmark_http_attempts":198,"benchmark_retry_count":0,"benchmark_usage_complete":true,"benchmark_input_tokens":125577,"benchmark_output_tokens":13006,"benchmark_reported_cost_usd":0.005274234,"observed_input_price_usd_per_million":0.042,"observed_output_price_usd_per_million":0,"known_success_probe_cost_usd":0,"failed_interface_probes_without_usage":0,"probe_billing_note":"No unreported errors in this recorded run.","network_roundtrip":{"n":198,"minimum_ms":179.4649160001427,"median_ms":228.5437495156657,"p90_nearest_rank_ms":345.2474070363678,"maximum_ms":1496.8284289934672},"per_request_roundtrip_available":true,"provider_compute_timing_ms":null,"provider_compute_timing_status":"not returned","per_job_walltime_available":false,"load_warmup_status":"Hosted service; no controllable model load/warmup measurement. D1/Solar have one recorded interface probe before benchmark; full Span has one successful Noul probe after a Choice rejection.","recorded_start_utc":"2026-10-02T15:00:55.938697+00:00","recorded_end_utc":"2026-10-02T15:11:06.936206+00:00","source_prediction_journal_sha256":"7ce746d6c966ea5dbc087e977a9e9c69d25069bf0021e380636311452d497aa2","job_walltime_including_retries":null}},{"id":"openrouter-span_full","platform":"OpenRouter","model_id":"respan/span-01","display_name":"Span-01","revision":"respan/span-01-20260925","loading":"Provider undisclosed","scoring_kind":"Adapted independent suitability","settings":{"guard_threshold":0.45,"interface":"six_independent_noul_suitability_questions","guard_signal":"maximum independent yes score","provider":"Respan","weights_revision":null,"dtype":null,"quantization":null},"scoring_metadata":{"probability_source":"six independent Bernoulli yes scores; not normalized","latency_scope":"client wall-clock network-inclusive request roundtrip; provider compute time unavailable","equal_mass_convention":"balanced larger-first chunks; minimum two rows"},"panels":{"diagnostic43-no_samples":{"n":43,"raw":{"n":43,"acceptable":38,"accuracy":0.8837209302325582,"unsafe":1,"unsafe_rate":0.023255813953488372,"review":4,"review_rate":0.09302325581395349,"automated_count":39,"automated_acceptable":36,"automated_coverage":0.9069767441860465,"conditional_correctness":0.9230769230769231,"unnecessary_reviews":2,"unnecessary_review_rate":0.046511627906976744},"effective":{"n":43,"acceptable":35,"accuracy":0.813953488372093,"unsafe":1,"unsafe_rate":0.023255813953488372,"review":9,"review_rate":0.20930232558139536,"automated_count":34,"automated_acceptable":33,"automated_coverage":0.7906976744186046,"conditional_correctness":0.9705882352941176,"unnecessary_reviews":7,"unnecessary_review_rate":0.16279069767441862},"latency_ms":{"n":43,"median":392.9521780228242,"p90_nearest_rank":482.4054899509065},"probability_metrics":{"singleton_n":38,"multiacceptable_n":5,"nll":null,"brier":null},"calibration_singleton_labels":{},"hosted_aggregate_diagnostics":{"n":43,"unique_label_n":38,"multi_acceptable_n":5,"raw":{"n":43,"acceptable":38,"unsafe":1,"incorrect_unlabeled_unsafe":4,"reviews":4,"unnecessary_reviews":2,"automatic":39,"coverage":0.9069767441860465,"automatic_acceptable":36,"conditional_automatic_accuracy":0.9230769230769231},"guarded":{"n":43,"acceptable":35,"unsafe":1,"incorrect_unlabeled_unsafe":7,"reviews":9,"unnecessary_reviews":7,"automatic":34,"coverage":0.7906976744186046,"automatic_acceptable":33,"conditional_automatic_accuracy":0.9705882352941176},"network_roundtrip":{"n":43,"minimum_ms":316.0409639822319,"median_ms":392.9521780228242,"p90_nearest_rank_ms":482.4054899509065,"maximum_ms":1192.144847009331},"categorical_probability_metrics":null,"categorical_calibration":null,"independent_suitability_score_diagnostics":{"yes_count_at_0_5":{"2":15,"3":8,"1":14,"0":6},"independent_score_sum_range":[1.257473822,3.14654735],"selected_action_yes_score_reliability_unique":{"n":38,"ece_width10":0.1918656107894737,"ece_mass10_min2":0.19236136236842105,"signed_gap":-0.14880101236842105,"absolute_gap":0.14880101236842105,"width_bins":[{"n":0,"mean_signal":null,"action_acceptability":null},{"n":0,"mean_signal":null,"action_acceptability":null},{"n":0,"mean_signal":null,"action_acceptability":null},{"n":3,"mean_signal":0.37571595333333335,"action_acceptability":0.3333333333333333},{"n":3,"mean_signal":0.47178959666666664,"action_acceptability":0.6666666666666666},{"n":5,"mean_signal":0.547462296,"action_acceptability":0.8},{"n":1,"mean_signal":0.67890024,"action_acceptability":1},{"n":11,"mean_signal":0.7710617727272727,"action_acceptability":1},{"n":11,"mean_signal":0.8376431045454545,"action_acceptability":1},{"n":4,"mean_signal":0.9227698775,"action_acceptability":0.75}],"mass_bins":[{"n":4,"mean_signal":0.39352099,"action_acceptability":0.5},{"n":4,"mean_signal":0.509171035,"action_acceptability":0.5},{"n":4,"mean_signal":0.5869900674999999,"action_acceptability":1},{"n":4,"mean_signal":0.7471055424999999,"action_acceptability":1},{"n":4,"mean_signal":0.77665344,"action_acceptability":1},{"n":4,"mean_signal":0.7967256175,"action_acceptability":1},{"n":4,"mean_signal":0.81663545,"action_acceptability":1},{"n":4,"mean_signal":0.8442794,"action_acceptability":1},{"n":3,"mean_signal":0.8900909499999999,"action_acceptability":1},{"n":3,"mean_signal":0.9303208366666667,"action_acceptability":0.6666666666666666}]},"maximum_yes_score_heuristic_reliability_whole":{"n":43,"ece_width10":0.20062963325581396,"ece_mass10_min2":0.19718678023255815,"signed_gap":-0.16257254627906975,"absolute_gap":0.16257254627906975,"width_bins":[{"n":0,"mean_signal":null,"action_acceptability":null},{"n":0,"mean_signal":null,"action_acceptability":null},{"n":0,"mean_signal":null,"action_acceptability":null},{"n":3,"mean_signal":0.37571595333333335,"action_acceptability":0.3333333333333333},{"n":3,"mean_signal":0.47178959666666664,"action_acceptability":0.6666666666666666},{"n":5,"mean_signal":0.547462296,"action_acceptability":0.8},{"n":3,"mean_signal":0.6697873933333334,"action_acceptability":1},{"n":13,"mean_signal":0.7687460692307693,"action_acceptability":1},{"n":12,"mean_signal":0.8362843158333333,"action_acceptability":1},{"n":4,"mean_signal":0.9227698775,"action_acceptability":0.75}],"mass_bins":[{"n":5,"mean_signal":0.410625304,"action_acceptability":0.4},{"n":5,"mean_signal":0.532073862,"action_acceptability":0.8},{"n":5,"mean_signal":0.658125826,"action_acceptability":1},{"n":4,"mean_signal":0.753819605,"action_acceptability":1},{"n":4,"mean_signal":0.7735631775,"action_acceptability":1},{"n":4,"mean_signal":0.79230828,"action_acceptability":1},{"n":4,"mean_signal":0.8105054,"action_acceptability":1},{"n":4,"mean_signal":0.83051921,"action_acceptability":1},{"n":4,"mean_signal":0.8678283375,"action_acceptability":1},{"n":4,"mean_signal":0.9227698775,"action_acceptability":0.75}]}},"probability_status":"Independent Noul probabilities, not categorical; no normalization. Reliability diagnostic differs from native Choice confidence."}},"diagnostic43-compact_no_samples":{"n":43,"raw":{"n":43,"acceptable":32,"accuracy":0.7441860465116279,"unsafe":3,"unsafe_rate":0.06976744186046512,"review":0,"review_rate":0.0,"automated_count":43,"automated_acceptable":32,"automated_coverage":1.0,"conditional_correctness":0.7441860465116279,"unnecessary_reviews":0,"unnecessary_review_rate":0.0},"effective":{"n":43,"acceptable":33,"accuracy":0.7674418604651163,"unsafe":2,"unsafe_rate":0.046511627906976744,"review":2,"review_rate":0.046511627906976744,"automated_count":41,"automated_acceptable":32,"automated_coverage":0.9534883720930233,"conditional_correctness":0.7804878048780488,"unnecessary_reviews":1,"unnecessary_review_rate":0.023255813953488372},"latency_ms":{"n":43,"median":394.9190839775838,"p90_nearest_rank":548.5773680265993},"probability_metrics":{"singleton_n":38,"multiacceptable_n":5,"nll":null,"brier":null},"calibration_singleton_labels":{},"hosted_aggregate_diagnostics":{"n":43,"unique_label_n":38,"multi_acceptable_n":5,"raw":{"n":43,"acceptable":32,"unsafe":3,"incorrect_unlabeled_unsafe":8,"reviews":0,"unnecessary_reviews":0,"automatic":43,"coverage":1.0,"automatic_acceptable":32,"conditional_automatic_accuracy":0.7441860465116279},"guarded":{"n":43,"acceptable":33,"unsafe":2,"incorrect_unlabeled_unsafe":8,"reviews":2,"unnecessary_reviews":1,"automatic":41,"coverage":0.9534883720930233,"automatic_acceptable":32,"conditional_automatic_accuracy":0.7804878048780488},"network_roundtrip":{"n":43,"minimum_ms":311.14201596938074,"median_ms":394.9190839775838,"p90_nearest_rank_ms":548.5773680265993,"maximum_ms":1078.3993090153672},"categorical_probability_metrics":null,"categorical_calibration":null,"independent_suitability_score_diagnostics":{"yes_count_at_0_5":{"1":21,"2":18,"3":3,"4":1},"independent_score_sum_range":[1.77606345,2.86623553],"selected_action_yes_score_reliability_unique":{"n":38,"ece_width10":0.1845928221052631,"ece_mass10_min2":0.21808203736842102,"signed_gap":0.017024072631578947,"absolute_gap":0.017024072631578947,"width_bins":[{"n":0,"mean_signal":null,"action_acceptability":null},{"n":0,"mean_signal":null,"action_acceptability":null},{"n":0,"mean_signal":null,"action_acceptability":null},{"n":0,"mean_signal":null,"action_acceptability":null},{"n":0,"mean_signal":null,"action_acceptability":null},{"n":2,"mean_signal":0.539821655,"action_acceptability":0},{"n":10,"mean_signal":0.652695506,"action_acceptability":0.7},{"n":9,"mean_signal":0.7501197433333333,"action_acceptability":0.4444444444444444},{"n":16,"mean_signal":0.8364768031250001,"action_acceptability":1},{"n":1,"mean_signal":0.90560985,"action_acceptability":1}],"mass_bins":[{"n":4,"mean_signal":0.5797035525,"action_acceptability":0.25},{"n":4,"mean_signal":0.64424441,"action_acceptability":0.75},{"n":4,"mean_signal":0.67770163,"action_acceptability":0.75},{"n":4,"mean_signal":0.73096109,"action_acceptability":0},{"n":4,"mean_signal":0.757373515,"action_acceptability":1},{"n":4,"mean_signal":0.80608938,"action_acceptability":0.75},{"n":4,"mean_signal":0.8205668925,"action_acceptability":1},{"n":4,"mean_signal":0.8358711625,"action_acceptability":1},{"n":3,"mean_signal":0.8514279933333333,"action_acceptability":1},{"n":3,"mean_signal":0.89419475,"action_acceptability":1}]},"maximum_yes_score_heuristic_reliability_whole":{"n":43,"ece_width10":0.14677454325581396,"ece_mass10_min2":0.16806045488372093,"signed_gap":0.007280200465116279,"absolute_gap":0.007280200465116279,"width_bins":[{"n":0,"mean_signal":null,"action_acceptability":null},{"n":0,"mean_signal":null,"action_acceptability":null},{"n":0,"mean_signal":null,"action_acceptability":null},{"n":0,"mean_signal":null,"action_acceptability":null},{"n":0,"mean_signal":null,"action_acceptability":null},{"n":2,"mean_signal":0.539821655,"action_acceptability":0},{"n":12,"mean_signal":0.6556520583333333,"action_acceptability":0.6666666666666666},{"n":11,"mean_signal":0.7484121527272727,"action_acceptability":0.5454545454545454},{"n":17,"mean_signal":0.8369080635294117,"action_acceptability":1},{"n":1,"mean_signal":0.90560985,"action_acceptability":1}],"mass_bins":[{"n":5,"mean_signal":0.588138762,"action_acceptability":0.4},{"n":5,"mean_signal":0.656968542,"action_acceptability":0.6},{"n":5,"mean_signal":0.685727458,"action_acceptability":0.6},{"n":4,"mean_signal":0.7337317999999999,"action_acceptability":0.25},{"n":4,"mean_signal":0.7558175375,"action_acceptability":1},{"n":4,"mean_signal":0.79518077,"action_acceptability":0.75},{"n":4,"mean_signal":0.81704775,"action_acceptability":1},{"n":4,"mean_signal":0.8315381875,"action_acceptability":1},{"n":4,"mean_signal":0.847309055,"action_acceptability":1},{"n":4,"mean_signal":0.8840936025,"action_acceptability":1}]}},"probability_status":"Independent Noul probabilities, not categorical; no normalization. Reliability diagnostic differs from native Choice confidence."}},"challenge56-no_samples":{"n":56,"raw":{"n":56,"acceptable":47,"accuracy":0.8392857142857143,"unsafe":1,"unsafe_rate":0.017857142857142856,"review":8,"review_rate":0.14285714285714285,"automated_count":48,"automated_acceptable":45,"automated_coverage":0.8571428571428571,"conditional_correctness":0.9375,"unnecessary_reviews":6,"unnecessary_review_rate":0.10714285714285714},"effective":{"n":56,"acceptable":37,"accuracy":0.6607142857142857,"unsafe":0,"unsafe_rate":0.0,"review":21,"review_rate":0.375,"automated_count":35,"automated_acceptable":35,"automated_coverage":0.625,"conditional_correctness":1,"unnecessary_reviews":19,"unnecessary_review_rate":0.3392857142857143},"latency_ms":{"n":56,"median":373.5364780004602,"p90_nearest_rank":443.5646719648503},"probability_metrics":{"singleton_n":56,"multiacceptable_n":0,"nll":null,"brier":null},"calibration_singleton_labels":{},"hosted_aggregate_diagnostics":{"n":56,"unique_label_n":56,"multi_acceptable_n":0,"raw":{"n":56,"acceptable":47,"unsafe":1,"incorrect_unlabeled_unsafe":8,"reviews":8,"unnecessary_reviews":6,"automatic":48,"coverage":0.8571428571428571,"automatic_acceptable":45,"conditional_automatic_accuracy":0.9375},"guarded":{"n":56,"acceptable":37,"unsafe":0,"incorrect_unlabeled_unsafe":19,"reviews":21,"unnecessary_reviews":19,"automatic":35,"coverage":0.625,"automatic_acceptable":35,"conditional_automatic_accuracy":1},"network_roundtrip":{"n":56,"minimum_ms":278.4921579877846,"median_ms":373.5364780004602,"p90_nearest_rank_ms":443.5646719648503,"maximum_ms":516.755971009843},"categorical_probability_metrics":null,"categorical_calibration":null,"independent_suitability_score_diagnostics":{"yes_count_at_0_5":{"1":35,"0":9,"2":10,"3":2},"independent_score_sum_range":[1.199059243,2.385460005],"selected_action_yes_score_reliability_unique":{"n":56,"ece_width10":0.17254798964285714,"ece_mass10_min2":0.15846947892857144,"signed_gap":-0.08505945357142856,"absolute_gap":0.08505945357142856,"width_bins":[{"n":0,"mean_signal":null,"action_acceptability":null},{"n":0,"mean_signal":null,"action_acceptability":null},{"n":0,"mean_signal":null,"action_acceptability":null},{"n":7,"mean_signal":0.3499541442857143,"action_acceptability":0},{"n":2,"mean_signal":0.4403478,"action_acceptability":0.5},{"n":4,"mean_signal":0.52903705,"action_acceptability":0.75},{"n":2,"mean_signal":0.6849345,"action_acceptability":1},{"n":4,"mean_signal":0.759265765,"action_acceptability":1},{"n":30,"mean_signal":0.8668568843333333,"action_acceptability":1},{"n":7,"mean_signal":0.9110727428571429,"action_acceptability":1}],"mass_bins":[{"n":6,"mean_signal":0.34258011833333335,"action_acceptability":0},{"n":6,"mean_signal":0.47059865,"action_acceptability":0.5},{"n":6,"mean_signal":0.7006324,"action_acceptability":1},{"n":6,"mean_signal":0.8105057016666667,"action_acceptability":1},{"n":6,"mean_signal":0.8459214466666667,"action_acceptability":1},{"n":6,"mean_signal":0.8789734983333333,"action_acceptability":1},{"n":5,"mean_signal":0.885725942,"action_acceptability":1},{"n":5,"mean_signal":0.8893456,"action_acceptability":1},{"n":5,"mean_signal":0.89903418,"action_acceptability":1},{"n":5,"mean_signal":0.91417422,"action_acceptability":1}]},"maximum_yes_score_heuristic_reliability_whole":{"n":56,"ece_width10":0.17254798964285714,"ece_mass10_min2":0.15846947892857144,"signed_gap":-0.08505945357142856,"absolute_gap":0.08505945357142856,"width_bins":[{"n":0,"mean_signal":null,"action_acceptability":null},{"n":0,"mean_signal":null,"action_acceptability":null},{"n":0,"mean_signal":null,"action_acceptability":null},{"n":7,"mean_signal":0.3499541442857143,"action_acceptability":0},{"n":2,"mean_signal":0.4403478,"action_acceptability":0.5},{"n":4,"mean_signal":0.52903705,"action_acceptability":0.75},{"n":2,"mean_signal":0.6849345,"action_acceptability":1},{"n":4,"mean_signal":0.759265765,"action_acceptability":1},{"n":30,"mean_signal":0.8668568843333333,"action_acceptability":1},{"n":7,"mean_signal":0.9110727428571429,"action_acceptability":1}],"mass_bins":[{"n":6,"mean_signal":0.34258011833333335,"action_acceptability":0},{"n":6,"mean_signal":0.47059865,"action_acceptability":0.5},{"n":6,"mean_signal":0.7006324,"action_acceptability":1},{"n":6,"mean_signal":0.8105057016666667,"action_acceptability":1},{"n":6,"mean_signal":0.8459214466666667,"action_acceptability":1},{"n":6,"mean_signal":0.8789734983333333,"action_acceptability":1},{"n":5,"mean_signal":0.885725942,"action_acceptability":1},{"n":5,"mean_signal":0.8893456,"action_acceptability":1},{"n":5,"mean_signal":0.89903418,"action_acceptability":1},{"n":5,"mean_signal":0.91417422,"action_acceptability":1}]}},"probability_status":"Independent Noul probabilities, not categorical; no normalization. Reliability diagnostic differs from native Choice confidence."}},"challenge56-compact_no_samples":{"n":56,"raw":{"n":56,"acceptable":49,"accuracy":0.875,"unsafe":7,"unsafe_rate":0.125,"review":2,"review_rate":0.03571428571428571,"automated_count":54,"automated_acceptable":47,"automated_coverage":0.9642857142857143,"conditional_correctness":0.8703703703703703,"unnecessary_reviews":0,"unnecessary_review_rate":0.0},"effective":{"n":56,"acceptable":38,"accuracy":0.6785714285714286,"unsafe":0,"unsafe_rate":0.0,"review":20,"review_rate":0.35714285714285715,"automated_count":36,"automated_acceptable":36,"automated_coverage":0.6428571428571429,"conditional_correctness":1,"unnecessary_reviews":18,"unnecessary_review_rate":0.32142857142857145},"latency_ms":{"n":56,"median":391.3462840137072,"p90_nearest_rank":445.95185498474166},"probability_metrics":{"singleton_n":56,"multiacceptable_n":0,"nll":null,"brier":null},"calibration_singleton_labels":{},"hosted_aggregate_diagnostics":{"n":56,"unique_label_n":56,"multi_acceptable_n":0,"raw":{"n":56,"acceptable":49,"unsafe":7,"incorrect_unlabeled_unsafe":0,"reviews":2,"unnecessary_reviews":0,"automatic":54,"coverage":0.9642857142857143,"automatic_acceptable":47,"conditional_automatic_accuracy":0.8703703703703703},"guarded":{"n":56,"acceptable":38,"unsafe":0,"incorrect_unlabeled_unsafe":18,"reviews":20,"unnecessary_reviews":18,"automatic":36,"coverage":0.6428571428571429,"automatic_acceptable":36,"conditional_automatic_accuracy":1},"network_roundtrip":{"n":56,"minimum_ms":291.7697770171799,"median_ms":391.3462840137072,"p90_nearest_rank_ms":445.95185498474166,"maximum_ms":1135.5720039573498},"categorical_probability_metrics":null,"categorical_calibration":null,"independent_suitability_score_diagnostics":{"yes_count_at_0_5":{"1":34,"2":12,"0":10},"independent_score_sum_range":[1.026401351,2.306207076],"selected_action_yes_score_reliability_unique":{"n":56,"ece_width10":0.17472053910714286,"ece_mass10_min2":0.14852056910714284,"signed_gap":-0.14852056910714287,"absolute_gap":0.14852056910714287,"width_bins":[{"n":0,"mean_signal":null,"action_acceptability":null},{"n":0,"mean_signal":null,"action_acceptability":null},{"n":0,"mean_signal":null,"action_acceptability":null},{"n":6,"mean_signal":0.33238882999999997,"action_acceptability":0.3333333333333333},{"n":4,"mean_signal":0.453581215,"action_acceptability":0.75},{"n":3,"mean_signal":0.5778663866666667,"action_acceptability":0.3333333333333333},{"n":5,"mean_signal":0.649239444,"action_acceptability":1},{"n":13,"mean_signal":0.7336568323076923,"action_acceptability":1},{"n":11,"mean_signal":0.8683749445454545,"action_acceptability":1},{"n":14,"mean_signal":0.9146236214285715,"action_acceptability":1}],"mass_bins":[{"n":6,"mean_signal":0.33238882999999997,"action_acceptability":0.3333333333333333},{"n":6,"mean_signal":0.4922053483333333,"action_acceptability":0.6666666666666666},{"n":6,"mean_signal":0.6401481916666667,"action_acceptability":0.8333333333333334},{"n":6,"mean_signal":0.71576762,"action_acceptability":1},{"n":6,"mean_signal":0.7431605666666666,"action_acceptability":1},{"n":6,"mean_signal":0.8375564116666667,"action_acceptability":1},{"n":5,"mean_signal":0.8824271579999999,"action_acceptability":1},{"n":5,"mean_signal":0.901591228,"action_acceptability":1},{"n":5,"mean_signal":0.913773524,"action_acceptability":1},{"n":5,"mean_signal":0.925305354,"action_acceptability":1}]},"maximum_yes_score_heuristic_reliability_whole":{"n":56,"ece_width10":0.17472053910714286,"ece_mass10_min2":0.14852056910714284,"signed_gap":-0.14852056910714287,"absolute_gap":0.14852056910714287,"width_bins":[{"n":0,"mean_signal":null,"action_acceptability":null},{"n":0,"mean_signal":null,"action_acceptability":null},{"n":0,"mean_signal":null,"action_acceptability":null},{"n":6,"mean_signal":0.33238882999999997,"action_acceptability":0.3333333333333333},{"n":4,"mean_signal":0.453581215,"action_acceptability":0.75},{"n":3,"mean_signal":0.5778663866666667,"action_acceptability":0.3333333333333333},{"n":5,"mean_signal":0.649239444,"action_acceptability":1},{"n":13,"mean_signal":0.7336568323076923,"action_acceptability":1},{"n":11,"mean_signal":0.8683749445454545,"action_acceptability":1},{"n":14,"mean_signal":0.9146236214285715,"action_acceptability":1}],"mass_bins":[{"n":6,"mean_signal":0.33238882999999997,"action_acceptability":0.3333333333333333},{"n":6,"mean_signal":0.4922053483333333,"action_acceptability":0.6666666666666666},{"n":6,"mean_signal":0.6401481916666667,"action_acceptability":0.8333333333333334},{"n":6,"mean_signal":0.71576762,"action_acceptability":1},{"n":6,"mean_signal":0.7431605666666666,"action_acceptability":1},{"n":6,"mean_signal":0.8375564116666667,"action_acceptability":1},{"n":5,"mean_signal":0.8824271579999999,"action_acceptability":1},{"n":5,"mean_signal":0.901591228,"action_acceptability":1},{"n":5,"mean_signal":0.913773524,"action_acceptability":1},{"n":5,"mean_signal":0.925305354,"action_acceptability":1}]}},"probability_status":"Independent Noul probabilities, not categorical; no normalization. Reliability diagnostic differs from native Choice confidence."}}},"composite":{"n":198,"scope":"descriptive sum across four correlated panels","raw":{"n":198,"acceptable":166,"accuracy":0.8383838383838383,"unsafe":12,"unsafe_rate":0.06060606060606061,"review":14,"review_rate":0.0707070707070707,"automated_count":184,"automated_acceptable":160,"automated_coverage":0.9292929292929293,"conditional_correctness":0.8695652173913043,"unnecessary_reviews":8,"unnecessary_review_rate":0.04040404040404041},"effective":{"n":198,"acceptable":143,"accuracy":0.7222222222222222,"unsafe":3,"unsafe_rate":0.015151515151515152,"review":52,"review_rate":0.26262626262626265,"automated_count":146,"automated_acceptable":136,"automated_coverage":0.7373737373737373,"conditional_correctness":0.9315068493150684,"unnecessary_reviews":45,"unnecessary_review_rate":0.22727272727272727}},"recorded_whole_run_latency_ms":{"n":198,"median":389.59625249844976,"p90_nearest_rank":461.60155499819666},"benchmark_reported_cost_usd":0.00463668,"hosted_run_metadata":{"key":"span_full","name":"Span-01","requested_model_id":"respan/span-01","returned_model_id":"respan/span-01-20260925","provider":"Respan","endpoint":"https://openrouter.ai/api/alpha/decisions","interface":"six_independent_noul_suitability_questions","prediction_count":198,"benchmark_http_attempts":198,"benchmark_retry_count":0,"benchmark_usage_complete":true,"benchmark_input_tokens":231834,"benchmark_output_tokens":0,"benchmark_reported_cost_usd":0.00463668,"observed_input_price_usd_per_million":0.02,"observed_output_price_usd_per_million":0,"known_success_probe_cost_usd":2.02e-06,"failed_interface_probes_without_usage":1,"probe_billing_note":"Full Span Choice rejection lacks usage; retain $0.00131072 conservative reserve.","network_roundtrip":{"n":198,"minimum_ms":278.4921579877846,"median_ms":389.59625249844976,"p90_nearest_rank_ms":461.60155499819666,"maximum_ms":1192.144847009331},"per_request_roundtrip_available":true,"provider_compute_timing_ms":null,"provider_compute_timing_status":"not returned","per_job_walltime_available":false,"load_warmup_status":"Hosted service; no controllable model load/warmup measurement. D1/Solar have one recorded interface probe before benchmark; full Span has one successful Noul probe after a Choice rejection.","recorded_start_utc":"2026-10-02T15:39:46.134043+00:00","recorded_end_utc":"2026-10-02T15:41:10.075032+00:00","source_prediction_journal_sha256":"99238a16e63aeb21e81a9668eb0dc1b4ae2687d8ee4db9bdc8643f246a1e99d1","job_walltime_including_retries":null}},{"id":"openrouter-span_lite","platform":"OpenRouter","model_id":"respan/span-01-lite","display_name":"Span-01 Lite","revision":"respan/span-01-lite-20260925","loading":"Provider undisclosed","scoring_kind":"Adapted independent suitability","settings":{"guard_threshold":0.45,"interface":"six_independent_noul_suitability_questions","guard_signal":"maximum independent yes score","provider":"Respan","weights_revision":null,"dtype":null,"quantization":null},"scoring_metadata":{"probability_source":"six independent Bernoulli yes scores; not normalized","latency_scope":"client wall-clock network-inclusive request roundtrip; provider compute time unavailable","equal_mass_convention":"balanced larger-first chunks; minimum two rows"},"panels":{"diagnostic43-no_samples":{"n":43,"raw":{"n":43,"acceptable":37,"accuracy":0.8604651162790697,"unsafe":1,"unsafe_rate":0.023255813953488372,"review":4,"review_rate":0.09302325581395349,"automated_count":39,"automated_acceptable":35,"automated_coverage":0.9069767441860465,"conditional_correctness":0.8974358974358975,"unnecessary_reviews":2,"unnecessary_review_rate":0.046511627906976744},"effective":{"n":43,"acceptable":35,"accuracy":0.813953488372093,"unsafe":1,"unsafe_rate":0.023255813953488372,"review":9,"review_rate":0.20930232558139536,"automated_count":34,"automated_acceptable":33,"automated_coverage":0.7906976744186046,"conditional_correctness":0.9705882352941176,"unnecessary_reviews":7,"unnecessary_review_rate":0.16279069767441862},"latency_ms":{"n":43,"median":720.3699230158236,"p90_nearest_rank":1704.2048620060086},"probability_metrics":{"singleton_n":38,"multiacceptable_n":5,"nll":null,"brier":null},"calibration_singleton_labels":{},"hosted_aggregate_diagnostics":{"n":43,"unique_label_n":38,"multi_acceptable_n":5,"raw":{"n":43,"acceptable":37,"unsafe":1,"incorrect_unlabeled_unsafe":5,"reviews":4,"unnecessary_reviews":2,"automatic":39,"coverage":0.9069767441860465,"automatic_acceptable":35,"conditional_automatic_accuracy":0.8974358974358975},"guarded":{"n":43,"acceptable":35,"unsafe":1,"incorrect_unlabeled_unsafe":7,"reviews":9,"unnecessary_reviews":7,"automatic":34,"coverage":0.7906976744186046,"automatic_acceptable":33,"conditional_automatic_accuracy":0.9705882352941176},"network_roundtrip":{"n":43,"minimum_ms":432.0015239936765,"median_ms":720.3699230158236,"p90_nearest_rank_ms":1704.2048620060086,"maximum_ms":10564.573217998259},"categorical_probability_metrics":null,"categorical_calibration":null,"independent_suitability_score_diagnostics":{"yes_count_at_0_5":{"2":14,"3":9,"1":14,"0":6},"independent_score_sum_range":[1.24217775,3.07387519],"selected_action_yes_score_reliability_unique":{"n":38,"ece_width10":0.19215373736842103,"ece_mass10_min2":0.19566879789473685,"signed_gap":-0.12198963842105263,"absolute_gap":0.12198963842105263,"width_bins":[{"n":0,"mean_signal":null,"action_acceptability":null},{"n":0,"mean_signal":null,"action_acceptability":null},{"n":0,"mean_signal":null,"action_acceptability":null},{"n":3,"mean_signal":0.37869556,"action_acceptability":0.3333333333333333},{"n":3,"mean_signal":0.4688671466666667,"action_acceptability":0.3333333333333333},{"n":5,"mean_signal":0.553273148,"action_acceptability":0.8},{"n":1,"mean_signal":0.6920362,"action_acceptability":1},{"n":10,"mean_signal":0.764128213,"action_acceptability":1},{"n":13,"mean_signal":0.8408916761538462,"action_acceptability":1},{"n":3,"mean_signal":0.9301432533333334,"action_acceptability":0.6666666666666666}],"mass_bins":[{"n":4,"mean_signal":0.394339345,"action_acceptability":0.25},{"n":4,"mean_signal":0.5080292225,"action_acceptability":0.5},{"n":4,"mean_signal":0.5979039475,"action_acceptability":1},{"n":4,"mean_signal":0.7418550525000001,"action_acceptability":1},{"n":4,"mean_signal":0.772375955,"action_acceptability":1},{"n":4,"mean_signal":0.7981833549999999,"action_acceptability":1},{"n":4,"mean_signal":0.8157671575000001,"action_acceptability":1},{"n":4,"mean_signal":0.84688105,"action_acceptability":1},{"n":3,"mean_signal":0.8908745466666667,"action_acceptability":1},{"n":3,"mean_signal":0.9301432533333334,"action_acceptability":0.6666666666666666}]},"maximum_yes_score_heuristic_reliability_whole":{"n":43,"ece_width10":0.20116800023255815,"ece_mass10_min2":0.22034111837209303,"signed_gap":-0.13916251744186045,"absolute_gap":0.13916251744186045,"width_bins":[{"n":0,"mean_signal":null,"action_acceptability":null},{"n":0,"mean_signal":null,"action_acceptability":null},{"n":0,"mean_signal":null,"action_acceptability":null},{"n":3,"mean_signal":0.37869556,"action_acceptability":0.3333333333333333},{"n":3,"mean_signal":0.4688671466666667,"action_acceptability":0.3333333333333333},{"n":5,"mean_signal":0.553273148,"action_acceptability":0.8},{"n":3,"mean_signal":0.67293117,"action_acceptability":1},{"n":12,"mean_signal":0.7626963691666666,"action_acceptability":1},{"n":14,"mean_signal":0.838955585,"action_acceptability":1},{"n":3,"mean_signal":0.9301432533333334,"action_acceptability":0.6666666666666666}],"mass_bins":[{"n":5,"mean_signal":0.41100243200000003,"action_acceptability":0.2},{"n":5,"mean_signal":0.53353675,"action_acceptability":0.8},{"n":5,"mean_signal":0.664518192,"action_acceptability":1},{"n":4,"mean_signal":0.7479429025000001,"action_acceptability":1},{"n":4,"mean_signal":0.7681312674999999,"action_acceptability":1},{"n":4,"mean_signal":0.793406875,"action_acceptability":1},{"n":4,"mean_signal":0.8107266325,"action_acceptability":1},{"n":4,"mean_signal":0.8311236425,"action_acceptability":1},{"n":4,"mean_signal":0.86876796,"action_acceptability":1},{"n":4,"mean_signal":0.92258194,"action_acceptability":0.75}]}},"probability_status":"Independent Noul probabilities, not categorical; no normalization. Reliability diagnostic differs from native Choice confidence."}},"diagnostic43-compact_no_samples":{"n":43,"raw":{"n":43,"acceptable":31,"accuracy":0.7209302325581395,"unsafe":3,"unsafe_rate":0.06976744186046512,"review":0,"review_rate":0.0,"automated_count":43,"automated_acceptable":31,"automated_coverage":1.0,"conditional_correctness":0.7209302325581395,"unnecessary_reviews":0,"unnecessary_review_rate":0.0},"effective":{"n":43,"acceptable":32,"accuracy":0.7441860465116279,"unsafe":2,"unsafe_rate":0.046511627906976744,"review":2,"review_rate":0.046511627906976744,"automated_count":41,"automated_acceptable":31,"automated_coverage":0.9534883720930233,"conditional_correctness":0.7560975609756098,"unnecessary_reviews":1,"unnecessary_review_rate":0.023255813953488372},"latency_ms":{"n":43,"median":731.8282400083262,"p90_nearest_rank":1084.7979309910443},"probability_metrics":{"singleton_n":38,"multiacceptable_n":5,"nll":null,"brier":null},"calibration_singleton_labels":{},"hosted_aggregate_diagnostics":{"n":43,"unique_label_n":38,"multi_acceptable_n":5,"raw":{"n":43,"acceptable":31,"unsafe":3,"incorrect_unlabeled_unsafe":9,"reviews":0,"unnecessary_reviews":0,"automatic":43,"coverage":1.0,"automatic_acceptable":31,"conditional_automatic_accuracy":0.7209302325581395},"guarded":{"n":43,"acceptable":32,"unsafe":2,"incorrect_unlabeled_unsafe":9,"reviews":2,"unnecessary_reviews":1,"automatic":41,"coverage":0.9534883720930233,"automatic_acceptable":31,"conditional_automatic_accuracy":0.7560975609756098},"network_roundtrip":{"n":43,"minimum_ms":433.6072879959829,"median_ms":731.8282400083262,"p90_nearest_rank_ms":1084.7979309910443,"maximum_ms":4020.2299960074015},"categorical_probability_metrics":null,"categorical_calibration":null,"independent_suitability_score_diagnostics":{"yes_count_at_0_5":{"1":21,"2":18,"3":3,"4":1},"independent_score_sum_range":[1.77335821,2.8519963],"selected_action_yes_score_reliability_unique":{"n":38,"ece_width10":0.1848255594736842,"ece_mass10_min2":0.22864737210526315,"signed_gap":0.041978512105263155,"absolute_gap":0.041978512105263155,"width_bins":[{"n":0,"mean_signal":null,"action_acceptability":null},{"n":0,"mean_signal":null,"action_acceptability":null},{"n":0,"mean_signal":null,"action_acceptability":null},{"n":0,"mean_signal":null,"action_acceptability":null},{"n":0,"mean_signal":null,"action_acceptability":null},{"n":2,"mean_signal":0.5377420500000001,"action_acceptability":0},{"n":10,"mean_signal":0.649607073,"action_acceptability":0.6},{"n":9,"mean_signal":0.7486358366666667,"action_acceptability":0.4444444444444444},{"n":16,"mean_signal":0.835942125,"action_acceptability":1},{"n":1,"mean_signal":0.9108321,"action_acceptability":1}],"mass_bins":[{"n":4,"mean_signal":0.5748059249999999,"action_acceptability":0.25},{"n":4,"mean_signal":0.6416909675,"action_acceptability":0.75},{"n":4,"mean_signal":0.676391815,"action_acceptability":0.5},{"n":4,"mean_signal":0.7293809,"action_acceptability":0},{"n":4,"mean_signal":0.7563804475,"action_acceptability":1},{"n":4,"mean_signal":0.80489431,"action_acceptability":0.75},{"n":4,"mean_signal":0.819419525,"action_acceptability":1},{"n":4,"mean_signal":0.8333134875,"action_acceptability":1},{"n":3,"mean_signal":0.8528310166666667,"action_acceptability":1},{"n":3,"mean_signal":0.8971936333333334,"action_acceptability":1}]},"maximum_yes_score_heuristic_reliability_whole":{"n":43,"ece_width10":0.1627222339534884,"ece_mass10_min2":0.2239074958139535,"signed_gap":0.02939490046511628,"absolute_gap":0.02939490046511628,"width_bins":[{"n":0,"mean_signal":null,"action_acceptability":null},{"n":0,"mean_signal":null,"action_acceptability":null},{"n":0,"mean_signal":null,"action_acceptability":null},{"n":0,"mean_signal":null,"action_acceptability":null},{"n":0,"mean_signal":null,"action_acceptability":null},{"n":2,"mean_signal":0.5377420500000001,"action_acceptability":0},{"n":12,"mean_signal":0.6532915725,"action_acceptability":0.5833333333333334},{"n":11,"mean_signal":0.7468668563636364,"action_acceptability":0.5454545454545454},{"n":17,"mean_signal":0.8366253076470588,"action_acceptability":1},{"n":1,"mean_signal":0.9108321,"action_acceptability":1}],"mass_bins":[{"n":5,"mean_signal":0.584496672,"action_acceptability":0.2},{"n":5,"mean_signal":0.654486756,"action_acceptability":0.8},{"n":5,"mean_signal":0.686116826,"action_acceptability":0.4},{"n":4,"mean_signal":0.73135596,"action_acceptability":0.25},{"n":4,"mean_signal":0.7563481599999999,"action_acceptability":1},{"n":4,"mean_signal":0.7918775475,"action_acceptability":0.75},{"n":4,"mean_signal":0.8159275625,"action_acceptability":1},{"n":4,"mean_signal":0.8280645625,"action_acceptability":1},{"n":4,"mean_signal":0.848843545,"action_acceptability":1},{"n":4,"mean_signal":0.887202525,"action_acceptability":1}]}},"probability_status":"Independent Noul probabilities, not categorical; no normalization. Reliability diagnostic differs from native Choice confidence."}},"challenge56-no_samples":{"n":56,"raw":{"n":56,"acceptable":47,"accuracy":0.8392857142857143,"unsafe":1,"unsafe_rate":0.017857142857142856,"review":9,"review_rate":0.16071428571428573,"automated_count":47,"automated_acceptable":45,"automated_coverage":0.8392857142857143,"conditional_correctness":0.9574468085106383,"unnecessary_reviews":7,"unnecessary_review_rate":0.125},"effective":{"n":56,"acceptable":37,"accuracy":0.6607142857142857,"unsafe":0,"unsafe_rate":0.0,"review":21,"review_rate":0.375,"automated_count":35,"automated_acceptable":35,"automated_coverage":0.625,"conditional_correctness":1,"unnecessary_reviews":19,"unnecessary_review_rate":0.3392857142857143},"latency_ms":{"n":56,"median":683.5382274875883,"p90_nearest_rank":1235.671734000789},"probability_metrics":{"singleton_n":56,"multiacceptable_n":0,"nll":null,"brier":null},"calibration_singleton_labels":{},"hosted_aggregate_diagnostics":{"n":56,"unique_label_n":56,"multi_acceptable_n":0,"raw":{"n":56,"acceptable":47,"unsafe":1,"incorrect_unlabeled_unsafe":8,"reviews":9,"unnecessary_reviews":7,"automatic":47,"coverage":0.8392857142857143,"automatic_acceptable":45,"conditional_automatic_accuracy":0.9574468085106383},"guarded":{"n":56,"acceptable":37,"unsafe":0,"incorrect_unlabeled_unsafe":19,"reviews":21,"unnecessary_reviews":19,"automatic":35,"coverage":0.625,"automatic_acceptable":35,"conditional_automatic_accuracy":1},"network_roundtrip":{"n":56,"minimum_ms":319.0690600022208,"median_ms":683.5382274875883,"p90_nearest_rank_ms":1235.671734000789,"maximum_ms":2149.511175986845},"categorical_probability_metrics":null,"categorical_calibration":null,"independent_suitability_score_diagnostics":{"yes_count_at_0_5":{"1":36,"0":9,"2":9,"3":2},"independent_score_sum_range":[1.2045579180000001,2.3820982500000003],"selected_action_yes_score_reliability_unique":{"n":56,"ece_width10":0.17183119339285713,"ece_mass10_min2":0.1579337669642857,"signed_gap":-0.08494661089285714,"absolute_gap":0.08494661089285714,"width_bins":[{"n":0,"mean_signal":null,"action_acceptability":null},{"n":0,"mean_signal":null,"action_acceptability":null},{"n":0,"mean_signal":null,"action_acceptability":null},{"n":7,"mean_signal":0.34753833,"action_acceptability":0},{"n":2,"mean_signal":0.4430406,"action_acceptability":0.5},{"n":4,"mean_signal":0.531033125,"action_acceptability":0.75},{"n":2,"mean_signal":0.6925828,"action_acceptability":1},{"n":4,"mean_signal":0.7598031275,"action_acceptability":1},{"n":30,"mean_signal":0.86661723,"action_acceptability":1},{"n":7,"mean_signal":0.91101611,"action_acceptability":1}],"mass_bins":[{"n":6,"mean_signal":0.34060672833333333,"action_acceptability":0},{"n":6,"mean_signal":0.4719819066666667,"action_acceptability":0.5},{"n":6,"mean_signal":0.703540075,"action_acceptability":1},{"n":6,"mean_signal":0.8094924766666667,"action_acceptability":1},{"n":6,"mean_signal":0.845552095,"action_acceptability":1},{"n":6,"mean_signal":0.8793640466666667,"action_acceptability":1},{"n":5,"mean_signal":0.885963208,"action_acceptability":1},{"n":5,"mean_signal":0.889442582,"action_acceptability":1},{"n":5,"mean_signal":0.89845244,"action_acceptability":1},{"n":5,"mean_signal":0.914094934,"action_acceptability":1}]},"maximum_yes_score_heuristic_reliability_whole":{"n":56,"ece_width10":0.17183119339285713,"ece_mass10_min2":0.1579337669642857,"signed_gap":-0.08494661089285714,"absolute_gap":0.08494661089285714,"width_bins":[{"n":0,"mean_signal":null,"action_acceptability":null},{"n":0,"mean_signal":null,"action_acceptability":null},{"n":0,"mean_signal":null,"action_acceptability":null},{"n":7,"mean_signal":0.34753833,"action_acceptability":0},{"n":2,"mean_signal":0.4430406,"action_acceptability":0.5},{"n":4,"mean_signal":0.531033125,"action_acceptability":0.75},{"n":2,"mean_signal":0.6925828,"action_acceptability":1},{"n":4,"mean_signal":0.7598031275,"action_acceptability":1},{"n":30,"mean_signal":0.86661723,"action_acceptability":1},{"n":7,"mean_signal":0.91101611,"action_acceptability":1}],"mass_bins":[{"n":6,"mean_signal":0.34060672833333333,"action_acceptability":0},{"n":6,"mean_signal":0.4719819066666667,"action_acceptability":0.5},{"n":6,"mean_signal":0.703540075,"action_acceptability":1},{"n":6,"mean_signal":0.8094924766666667,"action_acceptability":1},{"n":6,"mean_signal":0.845552095,"action_acceptability":1},{"n":6,"mean_signal":0.8793640466666667,"action_acceptability":1},{"n":5,"mean_signal":0.885963208,"action_acceptability":1},{"n":5,"mean_signal":0.889442582,"action_acceptability":1},{"n":5,"mean_signal":0.89845244,"action_acceptability":1},{"n":5,"mean_signal":0.914094934,"action_acceptability":1}]}},"probability_status":"Independent Noul probabilities, not categorical; no normalization. Reliability diagnostic differs from native Choice confidence."}},"challenge56-compact_no_samples":{"n":56,"raw":{"n":56,"acceptable":49,"accuracy":0.875,"unsafe":7,"unsafe_rate":0.125,"review":2,"review_rate":0.03571428571428571,"automated_count":54,"automated_acceptable":47,"automated_coverage":0.9642857142857143,"conditional_correctness":0.8703703703703703,"unnecessary_reviews":0,"unnecessary_review_rate":0.0},"effective":{"n":56,"acceptable":38,"accuracy":0.6785714285714286,"unsafe":0,"unsafe_rate":0.0,"review":20,"review_rate":0.35714285714285715,"automated_count":36,"automated_acceptable":36,"automated_coverage":0.6428571428571429,"conditional_correctness":1,"unnecessary_reviews":18,"unnecessary_review_rate":0.32142857142857145},"latency_ms":{"n":56,"median":653.1604365009116,"p90_nearest_rank":1101.53717600042},"probability_metrics":{"singleton_n":56,"multiacceptable_n":0,"nll":null,"brier":null},"calibration_singleton_labels":{},"hosted_aggregate_diagnostics":{"n":56,"unique_label_n":56,"multi_acceptable_n":0,"raw":{"n":56,"acceptable":49,"unsafe":7,"incorrect_unlabeled_unsafe":0,"reviews":2,"unnecessary_reviews":0,"automatic":54,"coverage":0.9642857142857143,"automatic_acceptable":47,"conditional_automatic_accuracy":0.8703703703703703},"guarded":{"n":56,"acceptable":38,"unsafe":0,"incorrect_unlabeled_unsafe":18,"reviews":20,"unnecessary_reviews":18,"automatic":36,"coverage":0.6428571428571429,"automatic_acceptable":36,"conditional_automatic_accuracy":1},"network_roundtrip":{"n":56,"minimum_ms":336.5410399856046,"median_ms":653.1604365009116,"p90_nearest_rank_ms":1101.53717600042,"maximum_ms":2721.477004000917},"categorical_probability_metrics":null,"categorical_calibration":null,"independent_suitability_score_diagnostics":{"yes_count_at_0_5":{"1":34,"2":12,"0":10},"independent_score_sum_range":[1.026401351,2.306207076],"selected_action_yes_score_reliability_unique":{"n":56,"ece_width10":0.19556466875,"ece_mass10_min2":0.14864590089285715,"signed_gap":-0.14786434017857142,"absolute_gap":0.14786434017857142,"width_bins":[{"n":0,"mean_signal":null,"action_acceptability":null},{"n":0,"mean_signal":null,"action_acceptability":null},{"n":0,"mean_signal":null,"action_acceptability":null},{"n":5,"mean_signal":0.3206723,"action_acceptability":0.2},{"n":5,"mean_signal":0.449574304,"action_acceptability":0.8},{"n":3,"mean_signal":0.5774159,"action_acceptability":0.3333333333333333},{"n":5,"mean_signal":0.6513354,"action_acceptability":1},{"n":13,"mean_signal":0.7331894169230769,"action_acceptability":1},{"n":12,"mean_signal":0.8697609666666667,"action_acceptability":1},{"n":13,"mean_signal":0.9162188623076923,"action_acceptability":1}],"mass_bins":[{"n":6,"mean_signal":0.3369806166666667,"action_acceptability":0.3333333333333333},{"n":6,"mean_signal":0.49437317833333333,"action_acceptability":0.6666666666666666},{"n":6,"mean_signal":0.642005825,"action_acceptability":0.8333333333333334},{"n":6,"mean_signal":0.714871565,"action_acceptability":1},{"n":6,"mean_signal":0.7430438883333333,"action_acceptability":1},{"n":6,"mean_signal":0.8355714733333334,"action_acceptability":1},{"n":5,"mean_signal":0.882435038,"action_acceptability":1},{"n":5,"mean_signal":0.901898914,"action_acceptability":1},{"n":5,"mean_signal":0.914055608,"action_acceptability":1},{"n":5,"mean_signal":0.9253139739999999,"action_acceptability":1}]},"maximum_yes_score_heuristic_reliability_whole":{"n":56,"ece_width10":0.19556466875,"ece_mass10_min2":0.14864590089285715,"signed_gap":-0.14786434017857142,"absolute_gap":0.14786434017857142,"width_bins":[{"n":0,"mean_signal":null,"action_acceptability":null},{"n":0,"mean_signal":null,"action_acceptability":null},{"n":0,"mean_signal":null,"action_acceptability":null},{"n":5,"mean_signal":0.3206723,"action_acceptability":0.2},{"n":5,"mean_signal":0.449574304,"action_acceptability":0.8},{"n":3,"mean_signal":0.5774159,"action_acceptability":0.3333333333333333},{"n":5,"mean_signal":0.6513354,"action_acceptability":1},{"n":13,"mean_signal":0.7331894169230769,"action_acceptability":1},{"n":12,"mean_signal":0.8697609666666667,"action_acceptability":1},{"n":13,"mean_signal":0.9162188623076923,"action_acceptability":1}],"mass_bins":[{"n":6,"mean_signal":0.3369806166666667,"action_acceptability":0.3333333333333333},{"n":6,"mean_signal":0.49437317833333333,"action_acceptability":0.6666666666666666},{"n":6,"mean_signal":0.642005825,"action_acceptability":0.8333333333333334},{"n":6,"mean_signal":0.714871565,"action_acceptability":1},{"n":6,"mean_signal":0.7430438883333333,"action_acceptability":1},{"n":6,"mean_signal":0.8355714733333334,"action_acceptability":1},{"n":5,"mean_signal":0.882435038,"action_acceptability":1},{"n":5,"mean_signal":0.901898914,"action_acceptability":1},{"n":5,"mean_signal":0.914055608,"action_acceptability":1},{"n":5,"mean_signal":0.9253139739999999,"action_acceptability":1}]}},"probability_status":"Independent Noul probabilities, not categorical; no normalization. Reliability diagnostic differs from native Choice confidence."}}},"composite":{"n":198,"scope":"descriptive sum across four correlated panels","raw":{"n":198,"acceptable":164,"accuracy":0.8282828282828283,"unsafe":12,"unsafe_rate":0.06060606060606061,"review":15,"review_rate":0.07575757575757576,"automated_count":183,"automated_acceptable":158,"automated_coverage":0.9242424242424242,"conditional_correctness":0.8633879781420765,"unnecessary_reviews":9,"unnecessary_review_rate":0.045454545454545456},"effective":{"n":198,"acceptable":142,"accuracy":0.7171717171717171,"unsafe":3,"unsafe_rate":0.015151515151515152,"review":52,"review_rate":0.26262626262626265,"automated_count":146,"automated_acceptable":135,"automated_coverage":0.7373737373737373,"conditional_correctness":0.9246575342465754,"unnecessary_reviews":45,"unnecessary_review_rate":0.22727272727272727}},"recorded_whole_run_latency_ms":{"n":198,"median":686.8579224974383,"p90_nearest_rank":1224.692504008999},"benchmark_reported_cost_usd":0,"hosted_run_metadata":{"key":"span_lite","name":"Span-01 Lite","requested_model_id":"respan/span-01-lite","returned_model_id":"respan/span-01-lite-20260925","provider":"Respan","endpoint":"https://openrouter.ai/api/v1/systemone","interface":"six_independent_noul_suitability_questions","prediction_count":198,"benchmark_http_attempts":198,"benchmark_retry_count":0,"benchmark_usage_complete":true,"benchmark_input_tokens":231834,"benchmark_output_tokens":0,"benchmark_reported_cost_usd":0,"observed_input_price_usd_per_million":0,"observed_output_price_usd_per_million":0,"known_success_probe_cost_usd":0,"failed_interface_probes_without_usage":2,"probe_billing_note":"Historical Lite Choice/Score rejections lack usage; reported successful Noul probe cost is zero under zero-price gates.","network_roundtrip":{"n":198,"minimum_ms":319.0690600022208,"median_ms":686.8579224974383,"p90_nearest_rank_ms":1224.692504008999,"maximum_ms":10564.573217998259},"per_request_roundtrip_available":true,"provider_compute_timing_ms":null,"provider_compute_timing_status":"not returned","per_job_walltime_available":false,"load_warmup_status":"Hosted service; no controllable model load/warmup measurement. D1/Solar have one recorded interface probe before benchmark; full Span has one successful Noul probe after a Choice rejection.","recorded_start_utc":null,"recorded_end_utc":null,"source_prediction_journal_sha256":"86248d7df547ff9977f699487c28eefd75b2e57b40cffd534ffb06c5ab6b9cd3","job_walltime_including_retries":null}},{"id":"openrouter-d1","platform":"OpenRouter","model_id":"liquid/d1","display_name":"Liquid D1","revision":"liquid/d1-20260930","loading":"Provider undisclosed","scoring_kind":"Native hosted Choice","settings":{"guard_threshold":0.45,"interface":"native_choice","guard_signal":"selected Choice probability","provider":"Liquid","weights_revision":null,"dtype":null,"quantization":null},"scoring_metadata":{"probability_source":"returned categorical Choice probabilities","latency_scope":"client wall-clock network-inclusive request roundtrip; provider compute time unavailable","equal_mass_convention":"balanced larger-first chunks; minimum two rows"},"panels":{"diagnostic43-no_samples":{"n":43,"raw":{"n":43,"acceptable":42,"accuracy":0.9767441860465116,"unsafe":0,"unsafe_rate":0.0,"review":0,"review_rate":0.0,"automated_count":43,"automated_acceptable":42,"automated_coverage":1.0,"conditional_correctness":0.9767441860465116,"unnecessary_reviews":0,"unnecessary_review_rate":0.0},"effective":{"n":43,"acceptable":41,"accuracy":0.9534883720930233,"unsafe":0,"unsafe_rate":0.0,"review":2,"review_rate":0.046511627906976744,"automated_count":41,"automated_acceptable":40,"automated_coverage":0.9534883720930233,"conditional_correctness":0.975609756097561,"unnecessary_reviews":1,"unnecessary_review_rate":0.023255813953488372},"latency_ms":{"n":43,"median":253.97903402335942,"p90_nearest_rank":322.60817795759067},"probability_metrics":{"singleton_n":38,"multiacceptable_n":5,"nll":0.06123922622782393,"brier":0.0230237773308594},"calibration_singleton_labels":{"equal_width":{"n":38,"ece":0.05096803734717513},"equal_mass":{"n":38,"ece":0.025524413511653094}},"hosted_aggregate_diagnostics":{"n":43,"unique_label_n":38,"multi_acceptable_n":5,"raw":{"n":43,"acceptable":42,"unsafe":0,"incorrect_unlabeled_unsafe":1,"reviews":0,"unnecessary_reviews":0,"automatic":43,"coverage":1.0,"automatic_acceptable":42,"automatic_conditional_accuracy":0.9767441860465116},"guarded":{"n":43,"acceptable":41,"unsafe":0,"incorrect_unlabeled_unsafe":2,"reviews":2,"unnecessary_reviews":1,"automatic":41,"coverage":0.9534883720930233,"automatic_acceptable":40,"automatic_conditional_accuracy":0.975609756097561},"network_roundtrip":{"n":43,"minimum_ms":198.52255197474733,"median_ms":253.97903402335942,"p90_nearest_rank_ms":322.60817795759067,"maximum_ms":1049.3378459941596},"categorical_probability_metrics":{"unique_nll":0.06123922622782393,"unique_brier_sum":0.0230237773308594,"multi_acceptable_mass":0.959290433322948,"multi_acceptable_event_logloss":0.04290436479806176,"mean_unsafe_probability_mass":0.00875244902956858},"categorical_calibration":{"ece_width10":0.05096803734717513,"ece_mass10_min2":0.025524413511653094,"signed_gap":-0.022317630066706173,"absolute_gap":0.022317630066706173,"width_bins":[{"n":0,"mean_confidence":null,"accuracy":null,"min":null,"max":null},{"n":0,"mean_confidence":null,"accuracy":null,"min":null,"max":null},{"n":0,"mean_confidence":null,"accuracy":null,"min":null,"max":null},{"n":0,"mean_confidence":null,"accuracy":null,"min":null,"max":null},{"n":0,"mean_confidence":null,"accuracy":null,"min":null,"max":null},{"n":1,"mean_confidence":0.544357738328911,"accuracy":0,"min":0.544357738328911,"max":0.544357738328911},{"n":0,"mean_confidence":null,"accuracy":null,"min":null,"max":null},{"n":1,"mean_confidence":0.7878206304700108,"accuracy":1,"min":0.7878206304700108,"max":0.7878206304700108},{"n":4,"mean_confidence":0.8716869564504519,"accuracy":1,"min":0.8571051236207957,"max":0.8806317173761447},{"n":32,"mean_confidence":0.9791563707145137,"accuracy":1,"min":0.9324787182772223,"max":0.9991715679530025}],"mass_bins":[{"n":4,"mean_confidence":0.7652322213634979,"accuracy":0.75,"min":0.544357738328911,"max":0.8716453930342741},{"n":4,"mean_confidence":0.9081661576335076,"accuracy":1,"min":0.8773655917705934,"max":0.94218860311007},{"n":4,"mean_confidence":0.9488187538861061,"accuracy":1,"min":0.9439695649088762,"max":0.9582290060903642},{"n":4,"mean_confidence":0.9631897746476583,"accuracy":1,"min":0.9588246158461377,"max":0.9704684962214724},{"n":4,"mean_confidence":0.9779600568681209,"accuracy":1,"min":0.971390259642568,"max":0.981140344023499},{"n":4,"mean_confidence":0.987027089302581,"accuracy":1,"min":0.9830449350639533,"max":0.9906929897575226},{"n":4,"mean_confidence":0.9932152795964245,"accuracy":1,"min":0.9914681150618629,"max":0.9946178662271734},{"n":4,"mean_confidence":0.9965593644366499,"accuracy":1,"min":0.9959263012690107,"max":0.9972277291813921},{"n":3,"mean_confidence":0.9982075405319134,"accuracy":1,"min":0.9976297487933712,"max":0.998512684334612},{"n":3,"mean_confidence":0.9988775483104133,"accuracy":1,"min":0.9985425820645456,"max":0.9991715679530025}]},"independent_suitability_score_diagnostics":null,"probability_status":"Native categorical distribution preserved exactly.","nll_zero_gold_mass_n":0}},"diagnostic43-compact_no_samples":{"n":43,"raw":{"n":43,"acceptable":43,"accuracy":1.0,"unsafe":0,"unsafe_rate":0.0,"review":1,"review_rate":0.023255813953488372,"automated_count":42,"automated_acceptable":42,"automated_coverage":0.9767441860465116,"conditional_correctness":1,"unnecessary_reviews":0,"unnecessary_review_rate":0.0},"effective":{"n":43,"acceptable":42,"accuracy":0.9767441860465116,"unsafe":0,"unsafe_rate":0.0,"review":2,"review_rate":0.046511627906976744,"automated_count":41,"automated_acceptable":41,"automated_coverage":0.9534883720930233,"conditional_correctness":1,"unnecessary_reviews":1,"unnecessary_review_rate":0.023255813953488372},"latency_ms":{"n":43,"median":260.1358270039782,"p90_nearest_rank":330.6018100120127},"probability_metrics":{"singleton_n":38,"multiacceptable_n":5,"nll":0.01903171976486584,"brier":0.0016673625790412342},"calibration_singleton_labels":{"equal_width":{"n":38,"ece":0.018451859503445137},"equal_mass":{"n":38,"ece":0.018451859503445113}},"hosted_aggregate_diagnostics":{"n":43,"unique_label_n":38,"multi_acceptable_n":5,"raw":{"n":43,"acceptable":43,"unsafe":0,"incorrect_unlabeled_unsafe":0,"reviews":1,"unnecessary_reviews":0,"automatic":42,"coverage":0.9767441860465116,"automatic_acceptable":42,"automatic_conditional_accuracy":1},"guarded":{"n":43,"acceptable":42,"unsafe":0,"incorrect_unlabeled_unsafe":1,"reviews":2,"unnecessary_reviews":1,"automatic":41,"coverage":0.9534883720930233,"automatic_acceptable":41,"automatic_conditional_accuracy":1},"network_roundtrip":{"n":43,"minimum_ms":207.5534249888733,"median_ms":260.1358270039782,"p90_nearest_rank_ms":330.6018100120127,"maximum_ms":881.9266090285964},"categorical_probability_metrics":{"unique_nll":0.01903171976486584,"unique_brier_sum":0.0016673625790412342,"multi_acceptable_mass":0.9722778505771744,"multi_acceptable_event_logloss":0.028651630170763526,"mean_unsafe_probability_mass":0.0081197506232939},"categorical_calibration":{"ece_width10":0.018451859503445137,"ece_mass10_min2":0.018451859503445113,"signed_gap":-0.0184518595034451,"absolute_gap":0.0184518595034451,"width_bins":[{"n":0,"mean_confidence":null,"accuracy":null,"min":null,"max":null},{"n":0,"mean_confidence":null,"accuracy":null,"min":null,"max":null},{"n":0,"mean_confidence":null,"accuracy":null,"min":null,"max":null},{"n":0,"mean_confidence":null,"accuracy":null,"min":null,"max":null},{"n":0,"mean_confidence":null,"accuracy":null,"min":null,"max":null},{"n":0,"mean_confidence":null,"accuracy":null,"min":null,"max":null},{"n":0,"mean_confidence":null,"accuracy":null,"min":null,"max":null},{"n":0,"mean_confidence":null,"accuracy":null,"min":null,"max":null},{"n":1,"mean_confidence":0.898556912108661,"accuracy":1,"min":0.898556912108661,"max":0.898556912108661},{"n":37,"mean_confidence":0.9837911466692006,"accuracy":1,"min":0.9120039210614502,"max":0.9999646788514669}],"mass_bins":[{"n":4,"mean_confidence":0.921725284642689,"accuracy":1,"min":0.898556912108661,"max":0.94097483899914},{"n":4,"mean_confidence":0.9471517149812055,"accuracy":1,"min":0.9447881569253571,"max":0.9495635653262772},{"n":4,"mean_confidence":0.9690570161639066,"accuracy":1,"min":0.9506672334306688,"max":0.9899158264110397},{"n":4,"mean_confidence":0.9935530120446284,"accuracy":1,"min":0.990559843329147,"max":0.9958854130842356},{"n":4,"mean_confidence":0.997143186970758,"accuracy":1,"min":0.9964036527389882,"max":0.9979945131988462},{"n":4,"mean_confidence":0.9983112212511855,"accuracy":1,"min":0.9980507055978404,"max":0.998521098055892},{"n":4,"mean_confidence":0.9988036897197264,"accuracy":1,"min":0.9985241676021098,"max":0.9990627285419886},{"n":4,"mean_confidence":0.9993577298229068,"accuracy":1,"min":0.9992214538785699,"max":0.9994870196712076},{"n":3,"mean_confidence":0.999612245348087,"accuracy":1,"min":0.9994905998715056,"max":0.9996956238966765},{"n":3,"mean_confidence":0.9998603934789331,"accuracy":1,"min":0.9997363004772796,"max":0.9999646788514669}]},"independent_suitability_score_diagnostics":null,"probability_status":"Native categorical distribution preserved exactly.","nll_zero_gold_mass_n":0}},"challenge56-no_samples":{"n":56,"raw":{"n":56,"acceptable":55,"accuracy":0.9821428571428571,"unsafe":0,"unsafe_rate":0.0,"review":3,"review_rate":0.05357142857142857,"automated_count":53,"automated_acceptable":53,"automated_coverage":0.9464285714285714,"conditional_correctness":1,"unnecessary_reviews":1,"unnecessary_review_rate":0.017857142857142856},"effective":{"n":56,"acceptable":44,"accuracy":0.7857142857142857,"unsafe":0,"unsafe_rate":0.0,"review":14,"review_rate":0.25,"automated_count":42,"automated_acceptable":42,"automated_coverage":0.75,"conditional_correctness":1,"unnecessary_reviews":12,"unnecessary_review_rate":0.21428571428571427},"latency_ms":{"n":56,"median":260.8261810382828,"p90_nearest_rank":388.5000160080381},"probability_metrics":{"singleton_n":56,"multiacceptable_n":0,"nll":0.090018483151333,"brier":0.03817412694865919},"calibration_singleton_labels":{"equal_width":{"n":56,"ece":0.053582917055793655},"equal_mass":{"n":56,"ece":0.05301476473400533}},"hosted_aggregate_diagnostics":{"n":56,"unique_label_n":56,"multi_acceptable_n":0,"raw":{"n":56,"acceptable":55,"unsafe":0,"incorrect_unlabeled_unsafe":1,"reviews":3,"unnecessary_reviews":1,"automatic":53,"coverage":0.9464285714285714,"automatic_acceptable":53,"automatic_conditional_accuracy":1},"guarded":{"n":56,"acceptable":44,"unsafe":0,"incorrect_unlabeled_unsafe":12,"reviews":14,"unnecessary_reviews":12,"automatic":42,"coverage":0.75,"automatic_acceptable":42,"automatic_conditional_accuracy":1},"network_roundtrip":{"n":56,"minimum_ms":201.69761904980987,"median_ms":260.8261810382828,"p90_nearest_rank_ms":388.5000160080381,"maximum_ms":1737.0098069659434},"categorical_probability_metrics":{"unique_nll":0.090018483151333,"unique_brier_sum":0.03817412694865919,"multi_acceptable_mass":null,"multi_acceptable_event_logloss":null,"mean_unsafe_probability_mass":0.015531778653365},"categorical_calibration":{"ece_width10":0.053582917055793655,"ece_mass10_min2":0.05301476473400533,"signed_gap":-0.053014764734005314,"absolute_gap":0.053014764734005314,"width_bins":[{"n":0,"mean_confidence":null,"accuracy":null,"min":null,"max":null},{"n":0,"mean_confidence":null,"accuracy":null,"min":null,"max":null},{"n":0,"mean_confidence":null,"accuracy":null,"min":null,"max":null},{"n":0,"mean_confidence":null,"accuracy":null,"min":null,"max":null},{"n":1,"mean_confidence":0.406301193569889,"accuracy":1,"min":0.406301193569889,"max":0.406301193569889},{"n":2,"mean_confidence":0.5079541325050362,"accuracy":0.5,"min":0.502607959480586,"max":0.5133003055294864},{"n":0,"mean_confidence":null,"accuracy":null,"min":null,"max":null},{"n":4,"mean_confidence":0.7640759898238878,"accuracy":1,"min":0.7018522504712962,"max":0.7949202587740621},{"n":5,"mean_confidence":0.8687519459294012,"accuracy":1,"min":0.8186927736374725,"max":0.8935917841557558},{"n":44,"mean_confidence":0.9820204551675723,"accuracy":1,"min":0.9047226945766164,"max":0.999207890770618}],"mass_bins":[{"n":6,"mean_confidence":0.6139321931835751,"accuracy":0.8333333333333334,"min":0.406301193569889,"max":0.7881178791400286},{"n":6,"mean_confidence":0.8564466647368447,"accuracy":1,"min":0.7949202587740621,"max":0.8935917841557558},{"n":6,"mean_confidence":0.9310705754774933,"accuracy":1,"min":0.9047226945766164,"max":0.9537617585767247},{"n":6,"mean_confidence":0.9710382227866833,"accuracy":1,"min":0.9549042797479922,"max":0.9835977032383247},{"n":6,"mean_confidence":0.9855068626148217,"accuracy":1,"min":0.9842010792040795,"max":0.9878638646716318},{"n":6,"mean_confidence":0.9905237885254542,"accuracy":1,"min":0.9890394962115323,"max":0.9932554739517757},{"n":5,"mean_confidence":0.9949914196249253,"accuracy":1,"min":0.993820105711888,"max":0.9953876504111147},{"n":5,"mean_confidence":0.9962037391325654,"accuracy":1,"min":0.9955613290814082,"max":0.9968652922117399},{"n":5,"mean_confidence":0.9977696616846635,"accuracy":1,"min":0.9971233182289795,"max":0.9986639458597174},{"n":5,"mean_confidence":0.9990478457471395,"accuracy":1,"min":0.9988874086744486,"max":0.999207890770618}]},"independent_suitability_score_diagnostics":null,"probability_status":"Native categorical distribution preserved exactly.","nll_zero_gold_mass_n":0}},"challenge56-compact_no_samples":{"n":56,"raw":{"n":56,"acceptable":56,"accuracy":1.0,"unsafe":0,"unsafe_rate":0.0,"review":2,"review_rate":0.03571428571428571,"automated_count":54,"automated_acceptable":54,"automated_coverage":0.9642857142857143,"conditional_correctness":1,"unnecessary_reviews":0,"unnecessary_review_rate":0.0},"effective":{"n":56,"acceptable":46,"accuracy":0.8214285714285714,"unsafe":0,"unsafe_rate":0.0,"review":12,"review_rate":0.21428571428571427,"automated_count":44,"automated_acceptable":44,"automated_coverage":0.7857142857142857,"conditional_correctness":1,"unnecessary_reviews":10,"unnecessary_review_rate":0.17857142857142858},"latency_ms":{"n":56,"median":249.3554925022181,"p90_nearest_rank":404.6333330334164},"probability_metrics":{"singleton_n":56,"multiacceptable_n":0,"nll":0.08952380222824155,"brier":0.027899440495565832},"calibration_singleton_labels":{"equal_width":{"n":56,"ece":0.07737529648702286},"equal_mass":{"n":56,"ece":0.07737529648702288}},"hosted_aggregate_diagnostics":{"n":56,"unique_label_n":56,"multi_acceptable_n":0,"raw":{"n":56,"acceptable":56,"unsafe":0,"incorrect_unlabeled_unsafe":0,"reviews":2,"unnecessary_reviews":0,"automatic":54,"coverage":0.9642857142857143,"automatic_acceptable":54,"automatic_conditional_accuracy":1},"guarded":{"n":56,"acceptable":46,"unsafe":0,"incorrect_unlabeled_unsafe":10,"reviews":12,"unnecessary_reviews":10,"automatic":44,"coverage":0.7857142857142857,"automatic_acceptable":44,"automatic_conditional_accuracy":1},"network_roundtrip":{"n":56,"minimum_ms":204.9646829837002,"median_ms":249.3554925022181,"p90_nearest_rank_ms":404.6333330334164,"maximum_ms":1350.0243659946136},"categorical_probability_metrics":{"unique_nll":0.08952380222824155,"unique_brier_sum":0.027899440495565832,"multi_acceptable_mass":null,"multi_acceptable_event_logloss":null,"mean_unsafe_probability_mass":0.02918724585287952},"categorical_calibration":{"ece_width10":0.07737529648702286,"ece_mass10_min2":0.07737529648702288,"signed_gap":-0.07737529648702288,"absolute_gap":0.07737529648702288,"width_bins":[{"n":0,"mean_confidence":null,"accuracy":null,"min":null,"max":null},{"n":0,"mean_confidence":null,"accuracy":null,"min":null,"max":null},{"n":0,"mean_confidence":null,"accuracy":null,"min":null,"max":null},{"n":0,"mean_confidence":null,"accuracy":null,"min":null,"max":null},{"n":0,"mean_confidence":null,"accuracy":null,"min":null,"max":null},{"n":1,"mean_confidence":0.599398326832684,"accuracy":1,"min":0.599398326832684,"max":0.599398326832684},{"n":3,"mean_confidence":0.6281670805749666,"accuracy":1,"min":0.6039355799401788,"max":0.6731300402939453},{"n":5,"mean_confidence":0.7571881394536989,"accuracy":1,"min":0.7072183472159945,"max":0.7842765746326074},{"n":8,"mean_confidence":0.8540031928514166,"accuracy":1,"min":0.8066338379729127,"max":0.899515137979546},{"n":39,"mean_confidence":0.9888491689253669,"accuracy":1,"min":0.9491479534927738,"max":0.9996470399158263}],"mass_bins":[{"n":6,"mean_confidence":0.6575133116476044,"accuracy":1,"min":0.599398326832684,"max":0.7539619541120481},{"n":6,"mean_confidence":0.7975671166965612,"accuracy":1,"min":0.7609548421048821,"max":0.8437021389489473},{"n":6,"mean_confidence":0.8867551986775317,"accuracy":1,"min":0.8485667910221065,"max":0.9491479534927738},{"n":6,"mean_confidence":0.9698624896017362,"accuracy":1,"min":0.9525022304512697,"max":0.9800115449402471},{"n":6,"mean_confidence":0.984629691329947,"accuracy":1,"min":0.9801612875327885,"max":0.9869728960718289},{"n":6,"mean_confidence":0.9912007604608898,"accuracy":1,"min":0.9883759079784556,"max":0.9932027607450162},{"n":5,"mean_confidence":0.9941778438134772,"accuracy":1,"min":0.9938799065328577,"max":0.9945602465755913},{"n":5,"mean_confidence":0.9961968858593176,"accuracy":1,"min":0.9947463966500872,"max":0.9977266624648872},{"n":5,"mean_confidence":0.9986908273698628,"accuracy":1,"min":0.998302270508992,"max":0.9988626940962042},{"n":5,"mean_confidence":0.9992968402055618,"accuracy":1,"min":0.9989043187644988,"max":0.9996470399158263}]},"independent_suitability_score_diagnostics":null,"probability_status":"Native categorical distribution preserved exactly.","nll_zero_gold_mass_n":0}}},"composite":{"n":198,"scope":"descriptive sum across four correlated panels","raw":{"n":198,"acceptable":196,"accuracy":0.98989898989899,"unsafe":0,"unsafe_rate":0.0,"review":6,"review_rate":0.030303030303030304,"automated_count":192,"automated_acceptable":191,"automated_coverage":0.9696969696969697,"conditional_correctness":0.9947916666666666,"unnecessary_reviews":1,"unnecessary_review_rate":0.005050505050505051},"effective":{"n":198,"acceptable":173,"accuracy":0.8737373737373737,"unsafe":0,"unsafe_rate":0.0,"review":30,"review_rate":0.15151515151515152,"automated_count":168,"automated_acceptable":167,"automated_coverage":0.8484848484848485,"conditional_correctness":0.9940476190476191,"unnecessary_reviews":24,"unnecessary_review_rate":0.12121212121212122}},"recorded_whole_run_latency_ms":{"n":198,"median":255.84394999896176,"p90_nearest_rank":360.5844709672965},"benchmark_reported_cost_usd":0.00260556,"hosted_run_metadata":{"key":"d1","name":"Liquid D1","requested_model_id":"liquid/d1","returned_model_id":"liquid/d1-20260930","provider":"Liquid","endpoint":"https://openrouter.ai/api/alpha/decisions","interface":"native_choice","prediction_count":198,"benchmark_http_attempts":198,"benchmark_retry_count":0,"benchmark_usage_complete":true,"benchmark_input_tokens":65139,"benchmark_output_tokens":0,"benchmark_reported_cost_usd":0.00260556,"observed_input_price_usd_per_million":0.04,"observed_output_price_usd_per_million":0,"known_success_probe_cost_usd":1.96e-06,"failed_interface_probes_without_usage":0,"probe_billing_note":"No unreported errors in this recorded run.","network_roundtrip":{"n":198,"minimum_ms":198.52255197474733,"median_ms":255.84394999896176,"p90_nearest_rank_ms":360.5844709672965,"maximum_ms":1737.0098069659434},"per_request_roundtrip_available":true,"provider_compute_timing_ms":null,"provider_compute_timing_status":"not returned","per_job_walltime_available":true,"load_warmup_status":"Hosted service; no controllable model load/warmup measurement. D1/Solar have one recorded interface probe before benchmark; full Span has one successful Noul probe after a Choice rejection.","recorded_start_utc":"2026-10-02T19:20:09.128153+00:00","recorded_end_utc":"2026-10-02T19:21:13.468209+00:00","source_prediction_journal_sha256":"6f6a269a38f46760660fe7af751dec8320820347f12ae7e59fdab2feef51d089","job_walltime_including_retries":{"n":198,"minimum_ms":201.1816009762697,"median_ms":270.94872150337324,"p90_nearest_rank_ms":382.21612299093977,"maximum_ms":1739.3364710151218}}},{"id":"openrouter-solar","platform":"OpenRouter","model_id":"upstage/solar-decide","display_name":"Solar Decide","revision":"upstage/solar-decide-20260928","loading":"Provider undisclosed","scoring_kind":"Native hosted Choice","settings":{"guard_threshold":0.45,"interface":"native_choice","guard_signal":"selected Choice probability","provider":"Upstage","weights_revision":null,"dtype":null,"quantization":null},"scoring_metadata":{"probability_source":"returned categorical Choice probabilities","latency_scope":"client wall-clock network-inclusive request roundtrip; provider compute time unavailable","equal_mass_convention":"balanced larger-first chunks; minimum two rows"},"panels":{"diagnostic43-no_samples":{"n":43,"raw":{"n":43,"acceptable":39,"accuracy":0.9069767441860465,"unsafe":2,"unsafe_rate":0.046511627906976744,"review":4,"review_rate":0.09302325581395349,"automated_count":39,"automated_acceptable":35,"automated_coverage":0.9069767441860465,"conditional_correctness":0.8974358974358975,"unnecessary_reviews":0,"unnecessary_review_rate":0.0},"effective":{"n":43,"acceptable":38,"accuracy":0.8837209302325582,"unsafe":2,"unsafe_rate":0.046511627906976744,"review":7,"review_rate":0.16279069767441862,"automated_count":36,"automated_acceptable":34,"automated_coverage":0.8372093023255814,"conditional_correctness":0.9444444444444444,"unnecessary_reviews":3,"unnecessary_review_rate":0.06976744186046512},"latency_ms":{"n":43,"median":4932.302484987304,"p90_nearest_rank":9033.103040012065},"probability_metrics":{"singleton_n":38,"multiacceptable_n":5,"nll":0.5812719424273888,"brier":0.1941279984728421},"calibration_singleton_labels":{"equal_width":{"n":38,"ece":0.08741805263157898},"equal_mass":{"n":38,"ece":0.08977926315789474}},"hosted_aggregate_diagnostics":{"n":43,"unique_label_n":38,"multi_acceptable_n":5,"raw":{"n":43,"acceptable":39,"unsafe":2,"incorrect_unlabeled_unsafe":2,"reviews":4,"unnecessary_reviews":0,"automatic":39,"coverage":0.9069767441860465,"automatic_acceptable":35,"automatic_conditional_accuracy":0.8974358974358975},"guarded":{"n":43,"acceptable":38,"unsafe":2,"incorrect_unlabeled_unsafe":3,"reviews":7,"unnecessary_reviews":3,"automatic":36,"coverage":0.8372093023255814,"automatic_acceptable":34,"automatic_conditional_accuracy":0.9444444444444444},"network_roundtrip":{"n":43,"minimum_ms":397.69809000426903,"median_ms":4932.302484987304,"p90_nearest_rank_ms":9033.103040012065,"maximum_ms":13088.117185980082},"categorical_probability_metrics":{"unique_nll":0.5812719424273888,"unique_brier_sum":0.1941279984728421,"multi_acceptable_mass":0.9937692,"multi_acceptable_event_logloss":0.0062552993305911185,"mean_unsafe_probability_mass":0.04521160465116279},"categorical_calibration":{"ece_width10":0.08741805263157898,"ece_mass10_min2":0.08977926315789474,"signed_gap":0.08741805263157895,"absolute_gap":0.08741805263157895,"width_bins":[{"n":0,"mean_confidence":null,"accuracy":null,"min":null,"max":null},{"n":0,"mean_confidence":null,"accuracy":null,"min":null,"max":null},{"n":0,"mean_confidence":null,"accuracy":null,"min":null,"max":null},{"n":0,"mean_confidence":null,"accuracy":null,"min":null,"max":null},{"n":0,"mean_confidence":null,"accuracy":null,"min":null,"max":null},{"n":0,"mean_confidence":null,"accuracy":null,"min":null,"max":null},{"n":0,"mean_confidence":null,"accuracy":null,"min":null,"max":null},{"n":0,"mean_confidence":null,"accuracy":null,"min":null,"max":null},{"n":3,"mean_confidence":0.8807363333333333,"accuracy":0.6666666666666666,"min":0.855074,"max":0.898433},{"n":35,"mean_confidence":0.9908479142857143,"accuracy":0.9142857142857143,"min":0.90337,"max":0.999885}],"mass_bins":[{"n":4,"mean_confidence":0.88639475,"accuracy":0.5,"min":0.855074,"max":0.90337},{"n":4,"mean_confidence":0.9552925,"accuracy":0.5,"min":0.907943,"max":0.992427},{"n":4,"mean_confidence":0.99529725,"accuracy":1,"min":0.993148,"max":0.996812},{"n":4,"mean_confidence":0.9977615,"accuracy":1,"min":0.996862,"max":0.998323},{"n":4,"mean_confidence":0.99863825,"accuracy":1,"min":0.998468,"max":0.998865},{"n":4,"mean_confidence":0.9990275,"accuracy":1,"min":0.998974,"max":0.999095},{"n":4,"mean_confidence":0.999182,"accuracy":1,"min":0.999117,"max":0.999307},{"n":4,"mean_confidence":0.9993595,"accuracy":1,"min":0.999332,"max":0.999431},{"n":3,"mean_confidence":0.9995303333333333,"accuracy":1,"min":0.999436,"max":0.999611},{"n":3,"mean_confidence":0.9998273333333334,"accuracy":1,"min":0.999719,"max":0.999885}]},"independent_suitability_score_diagnostics":null,"probability_status":"Native categorical distribution preserved exactly.","nll_zero_gold_mass_n":0}},"diagnostic43-compact_no_samples":{"n":43,"raw":{"n":43,"acceptable":33,"accuracy":0.7674418604651163,"unsafe":2,"unsafe_rate":0.046511627906976744,"review":3,"review_rate":0.06976744186046512,"automated_count":40,"automated_acceptable":30,"automated_coverage":0.9302325581395349,"conditional_correctness":0.75,"unnecessary_reviews":0,"unnecessary_review_rate":0.0},"effective":{"n":43,"acceptable":33,"accuracy":0.7674418604651163,"unsafe":2,"unsafe_rate":0.046511627906976744,"review":5,"review_rate":0.11627906976744186,"automated_count":38,"automated_acceptable":30,"automated_coverage":0.8837209302325582,"conditional_correctness":0.7894736842105263,"unnecessary_reviews":2,"unnecessary_review_rate":0.046511627906976744},"latency_ms":{"n":43,"median":4224.005416035652,"p90_nearest_rank":10580.810690997168},"probability_metrics":{"singleton_n":38,"multiacceptable_n":5,"nll":1.376812559569427,"brier":0.47793897556557896},"calibration_singleton_labels":{"equal_width":{"n":38,"ece":0.2183768421052631},"equal_mass":{"n":38,"ece":0.2200146842105263}},"hosted_aggregate_diagnostics":{"n":43,"unique_label_n":38,"multi_acceptable_n":5,"raw":{"n":43,"acceptable":33,"unsafe":2,"incorrect_unlabeled_unsafe":8,"reviews":3,"unnecessary_reviews":0,"automatic":40,"coverage":0.9302325581395349,"automatic_acceptable":30,"automatic_conditional_accuracy":0.75},"guarded":{"n":43,"acceptable":33,"unsafe":2,"incorrect_unlabeled_unsafe":8,"reviews":5,"unnecessary_reviews":2,"automatic":38,"coverage":0.8837209302325582,"automatic_acceptable":30,"automatic_conditional_accuracy":0.7894736842105263},"network_roundtrip":{"n":43,"minimum_ms":340.3225479996763,"median_ms":4224.005416035652,"p90_nearest_rank_ms":10580.810690997168,"maximum_ms":12886.147380981129},"categorical_probability_metrics":{"unique_nll":1.376812559569427,"unique_brier_sum":0.47793897556557896,"multi_acceptable_mass":0.9934422,"multi_acceptable_event_logloss":0.006582867919599414,"mean_unsafe_probability_mass":0.0458873488372093},"categorical_calibration":{"ece_width10":0.2183768421052631,"ece_mass10_min2":0.2200146842105263,"signed_gap":0.21837684210526315,"absolute_gap":0.21837684210526315,"width_bins":[{"n":0,"mean_confidence":null,"accuracy":null,"min":null,"max":null},{"n":0,"mean_confidence":null,"accuracy":null,"min":null,"max":null},{"n":0,"mean_confidence":null,"accuracy":null,"min":null,"max":null},{"n":0,"mean_confidence":null,"accuracy":null,"min":null,"max":null},{"n":0,"mean_confidence":null,"accuracy":null,"min":null,"max":null},{"n":0,"mean_confidence":null,"accuracy":null,"min":null,"max":null},{"n":0,"mean_confidence":null,"accuracy":null,"min":null,"max":null},{"n":0,"mean_confidence":null,"accuracy":null,"min":null,"max":null},{"n":8,"mean_confidence":0.8532689999999999,"accuracy":0.375,"min":0.802046,"max":0.88928},{"n":30,"mean_confidence":0.9824056,"accuracy":0.8333333333333334,"min":0.905679,"max":0.999805}],"mass_bins":[{"n":4,"mean_confidence":0.83376825,"accuracy":0.5,"min":0.802046,"max":0.857579},{"n":4,"mean_confidence":0.87276975,"accuracy":0.25,"min":0.859161,"max":0.88928},{"n":4,"mean_confidence":0.9142705,"accuracy":0.5,"min":0.905679,"max":0.920185},{"n":4,"mean_confidence":0.96172775,"accuracy":0.5,"min":0.932305,"max":0.994041},{"n":4,"mean_confidence":0.996263,"accuracy":1,"min":0.995397,"max":0.997497},{"n":4,"mean_confidence":0.998203,"accuracy":1,"min":0.997967,"max":0.998462},{"n":4,"mean_confidence":0.99892575,"accuracy":1,"min":0.998752,"max":0.999042},{"n":4,"mean_confidence":0.9991755,"accuracy":1,"min":0.999054,"max":0.999396},{"n":3,"mean_confidence":0.9995373333333333,"accuracy":1,"min":0.999431,"max":0.999692},{"n":3,"mean_confidence":0.9997646666666667,"accuracy":0.6666666666666666,"min":0.999739,"max":0.999805}]},"independent_suitability_score_diagnostics":null,"probability_status":"Native categorical distribution preserved exactly.","nll_zero_gold_mass_n":0}},"challenge56-no_samples":{"n":56,"raw":{"n":56,"acceptable":56,"accuracy":1.0,"unsafe":0,"unsafe_rate":0.0,"review":2,"review_rate":0.03571428571428571,"automated_count":54,"automated_acceptable":54,"automated_coverage":0.9642857142857143,"conditional_correctness":1,"unnecessary_reviews":0,"unnecessary_review_rate":0.0},"effective":{"n":56,"acceptable":46,"accuracy":0.8214285714285714,"unsafe":0,"unsafe_rate":0.0,"review":12,"review_rate":0.21428571428571427,"automated_count":44,"automated_acceptable":44,"automated_coverage":0.7857142857142857,"conditional_correctness":1,"unnecessary_reviews":10,"unnecessary_review_rate":0.17857142857142858},"latency_ms":{"n":56,"median":540.8749784983229,"p90_nearest_rank":6505.255135009065},"probability_metrics":{"singleton_n":56,"multiacceptable_n":0,"nll":0.05725405740627031,"brier":0.00761930265225},"calibration_singleton_labels":{"equal_width":{"n":56,"ece":0.05410057142857147},"equal_mass":{"n":56,"ece":0.054100571428571435}},"hosted_aggregate_diagnostics":{"n":56,"unique_label_n":56,"multi_acceptable_n":0,"raw":{"n":56,"acceptable":56,"unsafe":0,"incorrect_unlabeled_unsafe":0,"reviews":2,"unnecessary_reviews":0,"automatic":54,"coverage":0.9642857142857143,"automatic_acceptable":54,"automatic_conditional_accuracy":1},"guarded":{"n":56,"acceptable":46,"unsafe":0,"incorrect_unlabeled_unsafe":10,"reviews":12,"unnecessary_reviews":10,"automatic":44,"coverage":0.7857142857142857,"automatic_acceptable":44,"automatic_conditional_accuracy":1},"network_roundtrip":{"n":56,"minimum_ms":332.2739309514873,"median_ms":540.8749784983229,"p90_nearest_rank_ms":6505.255135009065,"maximum_ms":9699.907156988047},"categorical_probability_metrics":{"unique_nll":0.05725405740627031,"unique_brier_sum":0.00761930265225,"multi_acceptable_mass":null,"multi_acceptable_event_logloss":null,"mean_unsafe_probability_mass":0.0105565},"categorical_calibration":{"ece_width10":0.05410057142857147,"ece_mass10_min2":0.054100571428571435,"signed_gap":-0.05410057142857143,"absolute_gap":0.05410057142857143,"width_bins":[{"n":0,"mean_confidence":null,"accuracy":null,"min":null,"max":null},{"n":0,"mean_confidence":null,"accuracy":null,"min":null,"max":null},{"n":0,"mean_confidence":null,"accuracy":null,"min":null,"max":null},{"n":0,"mean_confidence":null,"accuracy":null,"min":null,"max":null},{"n":0,"mean_confidence":null,"accuracy":null,"min":null,"max":null},{"n":0,"mean_confidence":null,"accuracy":null,"min":null,"max":null},{"n":0,"mean_confidence":null,"accuracy":null,"min":null,"max":null},{"n":0,"mean_confidence":null,"accuracy":null,"min":null,"max":null},{"n":11,"mean_confidence":0.8566863636363636,"accuracy":1,"min":0.805224,"max":0.892413},{"n":45,"mean_confidence":0.9677070666666666,"accuracy":1,"min":0.906236,"max":0.999681}],"mass_bins":[{"n":6,"mean_confidence":0.8354041666666667,"accuracy":1,"min":0.805224,"max":0.85408},{"n":6,"mean_confidence":0.8862268333333333,"accuracy":1,"min":0.874398,"max":0.906236},{"n":6,"mean_confidence":0.916218,"accuracy":1,"min":0.907448,"max":0.927191},{"n":6,"mean_confidence":0.9406776666666666,"accuracy":1,"min":0.932442,"max":0.952474},{"n":6,"mean_confidence":0.9630056666666666,"accuracy":1,"min":0.955252,"max":0.969409},{"n":6,"mean_confidence":0.9726543333333333,"accuracy":1,"min":0.969952,"max":0.978719},{"n":5,"mean_confidence":0.9844524,"accuracy":1,"min":0.980129,"max":0.989573},{"n":5,"mean_confidence":0.9941694,"accuracy":1,"min":0.990463,"max":0.998425},{"n":5,"mean_confidence":0.9988246,"accuracy":1,"min":0.998502,"max":0.999404},{"n":5,"mean_confidence":0.9996032,"accuracy":1,"min":0.999542,"max":0.999681}]},"independent_suitability_score_diagnostics":null,"probability_status":"Native categorical distribution preserved exactly.","nll_zero_gold_mass_n":0}},"challenge56-compact_no_samples":{"n":56,"raw":{"n":56,"acceptable":56,"accuracy":1.0,"unsafe":0,"unsafe_rate":0.0,"review":2,"review_rate":0.03571428571428571,"automated_count":54,"automated_acceptable":54,"automated_coverage":0.9642857142857143,"conditional_correctness":1,"unnecessary_reviews":0,"unnecessary_review_rate":0.0},"effective":{"n":56,"acceptable":46,"accuracy":0.8214285714285714,"unsafe":0,"unsafe_rate":0.0,"review":12,"review_rate":0.21428571428571427,"automated_count":44,"automated_acceptable":44,"automated_coverage":0.7857142857142857,"conditional_correctness":1,"unnecessary_reviews":10,"unnecessary_review_rate":0.17857142857142858},"latency_ms":{"n":56,"median":538.4896020113956,"p90_nearest_rank":5740.804545988794},"probability_metrics":{"singleton_n":56,"multiacceptable_n":0,"nll":0.04720199444081137,"brier":0.005310418654571428},"calibration_singleton_labels":{"equal_width":{"n":56,"ece":0.04498291071428567},"equal_mass":{"n":56,"ece":0.04498291071428571}},"hosted_aggregate_diagnostics":{"n":56,"unique_label_n":56,"multi_acceptable_n":0,"raw":{"n":56,"acceptable":56,"unsafe":0,"incorrect_unlabeled_unsafe":0,"reviews":2,"unnecessary_reviews":0,"automatic":54,"coverage":0.9642857142857143,"automatic_acceptable":54,"automatic_conditional_accuracy":1},"guarded":{"n":56,"acceptable":46,"unsafe":0,"incorrect_unlabeled_unsafe":10,"reviews":12,"unnecessary_reviews":10,"automatic":44,"coverage":0.7857142857142857,"automatic_acceptable":44,"automatic_conditional_accuracy":1},"network_roundtrip":{"n":56,"minimum_ms":328.4370899782516,"median_ms":538.4896020113956,"p90_nearest_rank_ms":5740.804545988794,"maximum_ms":10004.962383012753},"categorical_probability_metrics":{"unique_nll":0.04720199444081137,"unique_brier_sum":0.005310418654571428,"multi_acceptable_mass":null,"multi_acceptable_event_logloss":null,"mean_unsafe_probability_mass":0.011849892857142857},"categorical_calibration":{"ece_width10":0.04498291071428567,"ece_mass10_min2":0.04498291071428571,"signed_gap":-0.04498291071428571,"absolute_gap":0.04498291071428571,"width_bins":[{"n":0,"mean_confidence":null,"accuracy":null,"min":null,"max":null},{"n":0,"mean_confidence":null,"accuracy":null,"min":null,"max":null},{"n":0,"mean_confidence":null,"accuracy":null,"min":null,"max":null},{"n":0,"mean_confidence":null,"accuracy":null,"min":null,"max":null},{"n":0,"mean_confidence":null,"accuracy":null,"min":null,"max":null},{"n":0,"mean_confidence":null,"accuracy":null,"min":null,"max":null},{"n":0,"mean_confidence":null,"accuracy":null,"min":null,"max":null},{"n":0,"mean_confidence":null,"accuracy":null,"min":null,"max":null},{"n":8,"mean_confidence":0.8663682500000001,"accuracy":1,"min":0.820111,"max":0.894864},{"n":48,"mean_confidence":0.9697918958333334,"accuracy":1,"min":0.901227,"max":0.999812}],"mass_bins":[{"n":6,"mean_confidence":0.8584535,"accuracy":1,"min":0.820111,"max":0.873679},{"n":6,"mean_confidence":0.9036031666666666,"accuracy":1,"min":0.885361,"max":0.921698},{"n":6,"mean_confidence":0.92937,"accuracy":1,"min":0.925144,"max":0.944137},{"n":6,"mean_confidence":0.9598116666666666,"accuracy":1,"min":0.954369,"max":0.964401},{"n":6,"mean_confidence":0.9692093333333334,"accuracy":1,"min":0.964634,"max":0.974556},{"n":6,"mean_confidence":0.9770718333333334,"accuracy":1,"min":0.974739,"max":0.980536},{"n":5,"mean_confidence":0.9859372000000001,"accuracy":1,"min":0.980809,"max":0.988236},{"n":5,"mean_confidence":0.9941646,"accuracy":1,"min":0.988505,"max":0.999024},{"n":5,"mean_confidence":0.9993578,"accuracy":1,"min":0.999191,"max":0.999533},{"n":5,"mean_confidence":0.9997084,"accuracy":1,"min":0.999637,"max":0.999812}]},"independent_suitability_score_diagnostics":null,"probability_status":"Native categorical distribution preserved exactly.","nll_zero_gold_mass_n":0}}},"composite":{"n":198,"scope":"descriptive sum across four correlated panels","raw":{"n":198,"acceptable":184,"accuracy":0.9292929292929293,"unsafe":4,"unsafe_rate":0.020202020202020204,"review":11,"review_rate":0.05555555555555555,"automated_count":187,"automated_acceptable":173,"automated_coverage":0.9444444444444444,"conditional_correctness":0.9251336898395722,"unnecessary_reviews":0,"unnecessary_review_rate":0.0},"effective":{"n":198,"acceptable":163,"accuracy":0.8232323232323232,"unsafe":4,"unsafe_rate":0.020202020202020204,"review":36,"review_rate":0.18181818181818182,"automated_count":162,"automated_acceptable":152,"automated_coverage":0.8181818181818182,"conditional_correctness":0.9382716049382716,"unnecessary_reviews":25,"unnecessary_review_rate":0.12626262626262627}},"recorded_whole_run_latency_ms":{"n":198,"median":609.7010789962951,"p90_nearest_rank":7736.532112001441},"benchmark_reported_cost_usd":0.0063107,"hosted_run_metadata":{"key":"solar","name":"Solar Decide","requested_model_id":"upstage/solar-decide","returned_model_id":"upstage/solar-decide-20260928","provider":"Upstage","endpoint":"https://openrouter.ai/api/alpha/decisions","interface":"native_choice","prediction_count":198,"benchmark_http_attempts":198,"benchmark_retry_count":0,"benchmark_usage_complete":true,"benchmark_input_tokens":126214,"benchmark_output_tokens":198,"benchmark_reported_cost_usd":0.0063107,"observed_input_price_usd_per_million":0.05,"observed_output_price_usd_per_million":0,"known_success_probe_cost_usd":1.765e-05,"failed_interface_probes_without_usage":0,"probe_billing_note":"No unreported errors in this recorded run.","network_roundtrip":{"n":198,"minimum_ms":328.4370899782516,"median_ms":609.7010789962951,"p90_nearest_rank_ms":7736.532112001441,"maximum_ms":13088.117185980082},"per_request_roundtrip_available":true,"provider_compute_timing_ms":null,"provider_compute_timing_status":"not returned","per_job_walltime_available":true,"load_warmup_status":"Hosted service; no controllable model load/warmup measurement. D1/Solar have one recorded interface probe before benchmark; full Span has one successful Noul probe after a Choice rejection.","recorded_start_utc":"2026-10-02T19:21:13.469606+00:00","recorded_end_utc":"2026-10-02T19:31:11.161216+00:00","source_prediction_journal_sha256":"9b4599a66f85d50590f1dc3c9fd66eb217cbb7910043fe6fd8d0b24af991f1b6","job_walltime_including_retries":{"n":198,"minimum_ms":344.83690600609407,"median_ms":633.7517600331921,"p90_nearest_rank_ms":7858.851590019185,"maximum_ms":13090.417194995098}}}],"repeats":[{"model_id":"Mapika/decider-4b","loading":"bfloat16 / nf4","n":198,"changed_choices":0,"maximum_absolute_probability_difference":0.0,"included_as_extra_distinct_configuration":false}],"non_evaluated_attempts":[{"model_id":"Cloudflare/clef-flash","revision":"17f0b0ad64efb65d273590632833508766b2aae6","status":"not_evaluated","reason":"Temporary cloud provisioning did not establish access; work paused before inference."},{"model_id":"Cloudflare/clef","revision":"2f3de3dd85f379784083b0814d997ab627200f0c","status":"not_evaluated","reason":"Temporary cloud provisioning did not establish access; work paused before inference."},{"model_id":"respan/span-01","status":"rejected_interface_probe","reason":"Pre-benchmark Choice probe rejected; no usage returned. Successful Noul benchmark is included above."},{"model_id":"respan/span-01-lite","status":"rejected_interface_probes","reason":"Pre-benchmark Choice and Score probes rejected; no usage returned. Successful Noul benchmark is included above."}],"hosted_source_sha256":"7cf0c8431b5212f699b2a350f154d319c3806284e093f27abcb6d2f34726649e","hosted_dataset":{"diagnostic_cases":43,"challenge_cases":56,"underlying_cases":99,"representations":["no_samples","compact_no_samples"],"jobs_per_model":198,"total_hosted_predictions":1188,"actions":["keep","drop","mask","pseudonymize","generalize","review"],"language":"English","exposure":"Exposed authored synthetic diagnostic/challenge cases; not an independent held-out population.","provenance":"Synthetic per-column release-policy cases; challenge v1 has eight template-correlated scenario families. Cell samples removed from all provider states; schema/type/format metadata and fictional release-policy prose retained.","fixtures":[{"name":"compact_fresh_test.json","sha256":"4692aa8c449d17b1b7bcab1593da32ec6436125dfb17ed70e46024b458012f9e"},{"name":"generality_challenge_v1.json","sha256":"4ae20dc31799e2cc40d2651edbeb9ff216666d48a6072e9fcb1b4e0cb6ac1f3d"}],"native_choice_prepared_jobs_sha256":"8365c1702266640ff8706c59d4b464b24ec8bccaf021a1a0f1029891845a954c","case_label_contract":"Acceptable actions and independently specified unsafe actions; unsafe is not the complement of acceptable. Review is acceptable only where permitted by the rubric."},"hosted_method":{"identical_state_strings_all_models":true,"identical_choice_questions_mercury_jev_d1_solar":true,"identical_six_noul_questions_full_and_lite_span":true,"same_guard_implementation_sha256":"4dc05527fb910dea64efed3f0725a71df6d8e0e42d2950155287ad4a789baa29","guard_threshold":0.45,"native_guard_signal":"Probability of selected Choice action; provider confidence remains opaque metadata.","span_adapter":"Six independent action-suitability Noul questions; exact maximum chooses action, exact ties choose review. Guard receives maximum independent yes score, a heuristic with a different estimand from Choice probability.","scoring":"Separate raw and guarded acceptability, unsafe actions, reviews, automatic coverage and conditional correctness. No tuning, probability normalization or clipping.","categorical_metrics":"Unique-label cases only: 38/38/56/56. NLL and sum-of-squares multiclass Brier; five set-valued diagnostic cases excluded, acceptable-set event scores separately retained.","ece":"10 equal-width bins and original Mercury balanced larger-first equal-mass chunks, minimum two cases per chunk; primary selected-action confidence reliability on unique-label cases.","timing":"Serial client wall-clock request roundtrip; no provider compute/GPU split. No local model loading or controllable hosted warmup. Runs not contemporaneous, provider/routing/network conditions differ.","limits":["99 underlying cases appear in two representations; 198 rows are not independent samples.","Synthetic policy correctness and labeled unsafety do not certify anonymity, legal compliance or empirical re-identification resistance.","Span is a domain/interface adaptation, not an identical native Choice model comparison.","Provider-returned dated model IDs are service identities; weights, quantization and provider hardware were not independently disclosed."]},"hosted_cost_summary":{"benchmark_reported_cost_usd":0.018827174,"known_success_probe_cost_usd":2.1629999999999998e-05,"failed_interface_probes_without_usage":3,"complete_historical_account_spend_known":false,"note":"All 1,188 benchmark rows have token usage and reported cost. Costs are reported OpenRouter usage, not an account invoice. Missing failed-probe billing and unrecorded earlier discovery prevent an all-research-spend total."}}