{ "categories": [ "polarity", "semantic-inversion", "hallucination", "numeric", "brand", "tech", "culture", "proper-noun", "dialect", "mishear-severe", "content-drop", "paraphrase", "mishear-minor", "filler", "format", "unscoreable" ], "tiers": { "tier1": [ "polarity", "semantic-inversion", "hallucination", "numeric" ], "tier2": [ "brand", "tech", "culture", "proper-noun", "dialect" ], "tier3": [ "mishear-severe", "content-drop", "paraphrase", "mishear-minor" ], "tier4": [ "filler", "format" ], "unscoreable": [ "unscoreable" ] }, "groups": { "meaning-flip": [ "polarity", "semantic-inversion", "hallucination", "numeric" ], "entity": [ "brand", "tech", "proper-noun" ], "cultural": [ "culture", "dialect" ], "fidelity": [ "mishear-severe", "content-drop", "paraphrase", "mishear-minor", "filler", "format" ] }, "group_label": { "meaning-flip": "意思翻转", "entity": "实体识别", "cultural": "文化/方言", "fidelity": "Verbatim 保真" }, "bucket_order": [ "general", "idiom-test", "long-mixed" ], "bucket_label": { "general": "常规中英混排", "idiom-test": "成语 / 方言 / 文化典故专项", "long-mixed": "长中英混合 · Echo Notes 典型" }, "bucket_map": { "1": "general", "2": "general", "3": "general", "4": "general", "5": "general", "6": "general", "7": "general", "8": "general", "9": "general", "10": "general", "11": "general", "12": "general", "13": "general", "14": "general", "15": "general", "16": "general", "17": "general", "18": "general", "19": "general", "20": "general", "21": "general", "22": "general", "23": "general", "24": "general", "25": "general", "26": "general", "27": "general", "28": "general", "29": "general", "30": "general", "31": "general", "32": "general", "33": "general", "34": "general", "35": "general", "36": "general", "37": "general", "38": "general", "39": "general", "40": "general", "41": "idiom-test", "42": "idiom-test", "43": "idiom-test", "44": "idiom-test", "45": "idiom-test", "46": "long-mixed", "47": "long-mixed", "48": "long-mixed", "49": "long-mixed", "50": "long-mixed", "51": "batch2-telegram", "52": "batch2-telegram", "53": "batch2-telegram", "54": "batch2-telegram", "55": "batch2-telegram", "56": "batch2-telegram", "57": "batch2-telegram", "58": "batch2-telegram", "59": "batch2-telegram", "60": "batch2-telegram", "61": "batch2-telegram", "62": "batch2-telegram", "63": "batch2-telegram", "64": "batch2-telegram", "65": "batch2-telegram", "66": "batch2-telegram", "67": "batch2-telegram", "68": "batch2-telegram", "69": "batch2-telegram", "70": "batch2-telegram", "71": "batch2-telegram", "72": "batch2-telegram", "73": "batch2-telegram", "74": "batch2-telegram", "75": "batch2-telegram", "76": "batch2-telegram", "77": "batch3-echo-notes-zh", "78": "batch3-echo-notes-zh", "79": "batch3-echo-notes-zh", "80": "batch3-echo-notes-zh", "81": "batch3-echo-notes-zh", "82": "batch3-echo-notes-zh", "83": "batch3-echo-notes-zh", "84": "batch3-echo-notes-zh", "85": "batch3-echo-notes-zh", "86": "batch3-echo-notes-zh", "87": "batch3-echo-notes-zh", "88": "batch3-echo-notes-zh", "89": "batch3-echo-notes-zh", "90": "batch3-echo-notes-zh", "91": "batch3-echo-notes-zh", "92": "batch3-echo-notes-zh", "93": "batch3-echo-notes-zh", "94": "batch3-echo-notes-zh", "95": "batch3-echo-notes-zh", "96": "batch3-echo-notes-zh", "97": "batch3-echo-notes-zh", "98": "batch3-echo-notes-zh", "99": "batch3-echo-notes-zh", "100": "batch3-echo-notes-zh" }, "tag_map": { "1": [ "numeric", "pure-chinese" ], "2": [], "3": [ "pure-chinese" ], "4": [], "5": [ "numeric", "polarity-dense" ], "6": [ "numeric" ], "7": [ "numeric", "has-tech", "brand-heavy" ], "8": [ "numeric", "polarity-dense" ], "9": [ "polarity-dense" ], "10": [ "idiom-classical", "pure-chinese" ], "11": [], "12": [ "numeric", "dialect", "polarity-dense" ], "13": [ "numeric", "has-tech", "brand-heavy", "polarity-dense" ], "14": [ "polarity-dense", "long" ], "15": [ "numeric", "has-tech", "polarity-dense", "long" ], "16": [ "numeric", "polarity-dense", "long" ], "17": [ "numeric", "has-tech", "brand-heavy", "dialect", "polarity-dense", "long" ], "18": [ "numeric", "has-tech", "polarity-dense", "long" ], "19": [ "numeric", "has-tech", "brand-heavy", "dialect", "polarity-dense", "long" ], "20": [ "numeric", "polarity-dense", "long" ], "21": [ "numeric" ], "22": [], "23": [ "polarity-dense" ], "24": [ "numeric" ], "25": [], "26": [ "numeric", "polarity-dense" ], "27": [ "numeric", "polarity-dense" ], "28": [ "numeric", "polarity-dense" ], "29": [ "numeric", "has-tech", "brand-heavy" ], "30": [ "numeric", "has-tech", "brand-heavy", "idiom-classical", "polarity-dense" ], "31": [ "numeric", "brand-heavy" ], "32": [ "numeric", "pure-chinese", "polarity-dense" ], "33": [ "numeric", "has-tech", "brand-heavy", "polarity-dense", "long" ], "34": [ "numeric", "polarity-dense", "long" ], "35": [ "numeric", "has-tech", "long" ], "36": [ "numeric", "polarity-dense", "long" ], "37": [ "numeric", "dialect", "polarity-dense", "long" ], "38": [ "numeric", "dialect", "polarity-dense", "long" ], "39": [ "numeric", "has-tech", "brand-heavy", "polarity-dense", "long" ], "40": [ "numeric", "idiom-classical", "dialect", "polarity-dense", "long" ], "41": [ "idiom-classical", "pure-chinese" ], "42": [ "dialect", "polarity-dense" ], "43": [ "idiom-classical", "dialect", "pure-chinese" ], "44": [ "numeric", "idiom-classical", "long" ], "45": [ "dialect", "pure-chinese", "polarity-dense" ], "46": [ "numeric", "pure-chinese", "polarity-dense", "long" ], "47": [ "numeric", "has-tech", "polarity-dense", "long" ], "48": [ "numeric", "has-tech", "dialect", "polarity-dense", "long" ], "49": [ "numeric", "has-tech", "brand-heavy", "polarity-dense", "long" ], "50": [ "numeric", "idiom-classical", "dialect", "pure-chinese", "long" ], "51": [ "numeric", "idiom-classical" ], "52": [ "numeric", "dialect", "pure-chinese", "polarity-dense" ], "53": [ "numeric", "pure-chinese", "polarity-dense" ], "54": [ "numeric", "pure-chinese", "polarity-dense" ], "55": [ "numeric", "polarity-dense" ], "56": [ "numeric", "brand-heavy", "polarity-dense" ], "57": [ "numeric", "has-tech", "polarity-dense" ], "58": [ "has-tech", "polarity-dense" ], "59": [ "polarity-dense" ], "60": [ "has-tech", "dialect" ], "61": [ "polarity-dense" ], "62": [ "numeric", "pure-chinese" ], "63": [ "numeric" ], "64": [ "dialect", "pure-chinese", "polarity-dense" ], "65": [ "numeric" ], "66": [ "numeric", "polarity-dense" ], "67": [ "polarity-dense" ], "68": [ "numeric" ], "69": [ "numeric", "polarity-dense" ], "70": [ "polarity-dense" ], "71": [], "72": [ "numeric", "pure-chinese", "polarity-dense" ], "73": [ "numeric", "has-tech" ], "74": [ "polarity-dense" ], "75": [ "polarity-dense", "long" ], "76": [ "numeric", "dialect", "polarity-dense" ], "77": [ "numeric", "dialect", "polarity-dense", "long" ], "78": [ "numeric", "pure-chinese", "polarity-dense", "long" ], "79": [ "numeric", "pure-chinese", "polarity-dense" ], "80": [ "pure-chinese", "polarity-dense" ], "81": [ "numeric", "pure-chinese", "polarity-dense" ], "82": [ "numeric", "polarity-dense" ], "83": [ "numeric", "dialect", "pure-chinese", "polarity-dense" ], "84": [ "numeric", "polarity-dense" ], "85": [ "numeric", "pure-chinese" ], "86": [ "numeric", "pure-chinese" ], "87": [ "numeric", "pure-chinese" ], "88": [ "pure-chinese" ], "89": [ "numeric", "dialect", "pure-chinese", "polarity-dense" ], "90": [ "numeric", "has-tech", "dialect", "pure-chinese", "polarity-dense" ], "91": [ "numeric", "pure-chinese", "polarity-dense" ], "92": [ "numeric", "pure-chinese", "polarity-dense" ], "93": [ "pure-chinese", "polarity-dense" ], "94": [ "numeric", "pure-chinese", "polarity-dense" ], "95": [ "numeric", "pure-chinese", "polarity-dense" ], "96": [ "numeric", "dialect", "pure-chinese" ], "97": [ "numeric", "pure-chinese", "polarity-dense" ], "98": [ "numeric", "pure-chinese", "polarity-dense" ], "99": [ "pure-chinese" ], "100": [ "dialect", "pure-chinese" ] }, "conditions_map": { "1": [ "clean" ], "2": [ "bg-noise" ], "3": [ "clean" ], "4": [ "bg-noise" ], "5": [ "mic-far" ], "6": [ "bg-noise" ], "7": [ "bg-noise" ], "8": [ "close-mic" ], "9": [ "clean" ], "10": [ "bg-noise" ], "11": [ "bg-noise" ], "12": [ "bg-noise" ], "13": [ "bg-noise" ], "14": [ "clean" ], "15": [ "clean" ], "16": [ "clean" ], "17": [ "bg-noise" ], "18": [ "clean" ], "19": [ "clean" ], "20": [ "clean" ], "21": [ "clean" ], "22": [ "mic-far", "bg-noise" ], "23": [ "clean" ], "24": [ "clean" ], "25": [ "clean" ], "26": [ "bg-noise", "close-mic" ], "27": [ "clean" ], "28": [ "bg-noise" ], "29": [ "bg-noise" ], "30": [ "bg-noise" ], "31": [ "bg-noise" ], "32": [ "bg-noise" ], "33": [ "bg-noise" ], "34": [ "ambient" ], "35": [ "clean" ], "36": [ "clean" ], "37": [ "bg-noise" ], "38": [ "clean" ], "39": [ "clean" ], "40": [ "clean" ], "41": [ "clean" ], "42": [ "clean" ], "43": [ "clean" ], "44": [ "clean" ], "45": [ "clean" ], "46": [ "clean" ], "47": [ "bg-noise" ], "48": [ "clean" ], "49": [ "bg-noise" ], "50": [ "clean" ], "51": [ "clean" ], "52": [ "mic-far" ], "53": [ "clean" ], "54": [ "bg-noise" ], "55": [ "bg-noise" ], "56": [ "bg-noise" ], "57": [ "bg-noise" ], "58": [ "clean" ], "59": [ "ambient" ], "60": [ "bg-noise" ], "61": [ "bg-noise" ], "62": [ "bg-noise" ], "63": [ "bg-noise" ], "64": [ "clean" ], "65": [ "clean" ], "66": [ "close-mic" ], "67": [ "clean" ], "68": [ "bg-noise" ], "69": [ "bg-noise" ], "70": [ "bg-noise", "mic-far" ], "71": [ "bg-noise" ], "72": [ "close-mic" ], "73": [ "clean" ], "74": [ "bg-noise", "mic-far" ], "75": [ "bg-noise", "mic-far" ], "76": [ "ambient" ], "77": [ "clean", "close-mic" ], "78": [ "clean" ], "79": [ "clean" ], "80": [ "clean" ], "81": [ "clean" ], "82": [ "clean", "bg-noise" ], "83": [ "clean" ], "84": [ "clean", "bg-noise" ], "85": [ "clean" ], "86": [ "clean" ], "87": [ "clean", "ambient", "bg-noise" ], "88": [ "clean", "bg-noise" ], "89": [ "clean" ], "90": [ "bg-noise" ], "91": [ "clean", "mic-far" ], "92": [ "bg-noise" ], "93": [ "clean", "mic-far", "bg-noise" ], "94": [ "clean" ], "95": [ "clean", "mic-far", "bg-noise" ], "96": [ "clean", "bg-noise" ], "97": [ "clean" ], "98": [ "clean", "bg-noise" ], "99": [ "bg-noise" ], "100": [ "clean", "bg-noise" ] }, "emotion_map": { "1": "neutral", "2": "neutral", "3": "frustrated", "4": "neutral", "5": "frustrated", "6": "neutral", "7": "neutral", "8": "frustrated", "9": "neutral", "10": "frustrated", "11": "neutral", "12": "neutral", "13": "neutral", "14": "neutral", "15": "neutral", "16": "neutral", "17": "neutral", "18": "neutral", "19": "neutral", "20": "neutral", "21": "frustrated", "22": "neutral", "23": "frustrated", "24": "angry", "25": "angry", "26": "frustrated", "27": "neutral", "28": "frustrated", "29": "neutral", "30": "neutral", "31": "frustrated", "32": "frustrated", "33": "neutral", "34": "neutral", "35": "neutral", "36": "neutral", "37": "neutral", "38": "frustrated", "39": "neutral", "40": "frustrated", "41": "neutral", "42": "frustrated", "43": "neutral", "44": "excited", "45": "neutral", "46": "neutral", "47": "angry", "48": "neutral", "49": "neutral", "50": "neutral", "51": "neutral", "52": "neutral", "53": "neutral", "54": "neutral", "55": "neutral", "56": "neutral", "57": "neutral", "58": "frustrated", "59": "frustrated", "60": "angry", "61": "frustrated", "62": "neutral", "63": "frustrated", "64": "frustrated", "65": "neutral", "66": "neutral", "67": "frustrated", "68": "frustrated", "69": "frustrated", "70": "frustrated", "71": "neutral", "72": "neutral", "73": "neutral", "74": "neutral", "75": "neutral", "76": "neutral", "77": "neutral", "78": "neutral", "79": "neutral", "80": "neutral", "81": "neutral", "82": "frustrated", "83": "neutral", "84": "neutral", "85": "neutral", "86": "neutral", "87": "neutral", "88": "frustrated", "89": "neutral", "90": "neutral", "91": "angry", "92": "neutral", "93": "neutral", "94": "frustrated", "95": "frustrated", "96": "neutral", "97": "neutral", "98": "excited", "99": "frustrated", "100": "neutral" }, "per_tag_stats": { "brand-heavy": { "apple_speechanalyzer": { "total_deductions": 535.0, "sample_count": 11, "avg_deductions_per_sample": 48.64 }, "doubao": { "total_deductions": 185.0, "sample_count": 11, "avg_deductions_per_sample": 16.82 }, "elevenlabs": { "total_deductions": 112.0, "sample_count": 11, "avg_deductions_per_sample": 10.18 }, "firered": { "total_deductions": 736.0, "sample_count": 11, "avg_deductions_per_sample": 66.91 }, "firered_aed": { "total_deductions": 444.0, "sample_count": 11, "avg_deductions_per_sample": 40.36 }, "flash": { "total_deductions": 169.0, "sample_count": 11, "avg_deductions_per_sample": 15.36 }, "funasr_nano": { "total_deductions": 447.0, "sample_count": 11, "avg_deductions_per_sample": 40.64 }, "gemini_3_5_flash_lite_stt": { "total_deductions": 309.0, "sample_count": 11, "avg_deductions_per_sample": 28.09 }, "gemini_3_5_flash_stt": { "total_deductions": 171.0, "sample_count": 11, "avg_deductions_per_sample": 15.55 }, "gemini_3_5_transcribe": { "total_deductions": 198.0, "sample_count": 11, "avg_deductions_per_sample": 18.0 }, "gemini_3_6_flash_stt": { "total_deductions": 169.0, "sample_count": 11, "avg_deductions_per_sample": 15.36 }, "gemini_3_7_flash_stt": { "total_deductions": 159.0, "sample_count": 11, "avg_deductions_per_sample": 14.45 }, "gemini_3_8_flash_stt": { "total_deductions": 74.0, "sample_count": 11, "avg_deductions_per_sample": 6.73 }, "gemini_flash_stt": { "total_deductions": 144.0, "sample_count": 11, "avg_deductions_per_sample": 13.09 }, "gemini_pro_stt": { "total_deductions": 124.0, "sample_count": 11, "avg_deductions_per_sample": 11.27 }, "gemini_stt": { "total_deductions": 227.0, "sample_count": 11, "avg_deductions_per_sample": 20.64 }, "glm_asr_nano": { "total_deductions": 298.0, "sample_count": 11, "avg_deductions_per_sample": 27.09 }, "gpt_transcribe": { "total_deductions": 146.0, "sample_count": 11, "avg_deductions_per_sample": 13.27 }, "mai_transcribe_2": { "total_deductions": 99.0, "sample_count": 11, "avg_deductions_per_sample": 9.0 }, "mimo": { "total_deductions": 119.0, "sample_count": 11, "avg_deductions_per_sample": 10.82 }, "mimo_stt": { "total_deductions": 152.0, "sample_count": 11, "avg_deductions_per_sample": 13.82 }, "moss_audio": { "total_deductions": 309.0, "sample_count": 10, "avg_deductions_per_sample": 30.9 }, "qwen3_oss": { "total_deductions": 238.0, "sample_count": 11, "avg_deductions_per_sample": 21.64 }, "qwen_audio_3_1_asr": { "total_deductions": 157.0, "sample_count": 11, "avg_deductions_per_sample": 14.27 }, "r2t2": { "total_deductions": 178.0, "sample_count": 11, "avg_deductions_per_sample": 16.18 }, "soniox": { "total_deductions": 172.0, "sample_count": 11, "avg_deductions_per_sample": 15.64 }, "stepfun": { "total_deductions": 311.0, "sample_count": 11, "avg_deductions_per_sample": 28.27 }, "vibevoice": { "total_deductions": 202.0, "sample_count": 11, "avg_deductions_per_sample": 18.36 }, "voxtral": { "total_deductions": 339.0, "sample_count": 11, "avg_deductions_per_sample": 30.82 }, "whisper": { "total_deductions": 255.0, "sample_count": 11, "avg_deductions_per_sample": 23.18 }, "whisper_v3_turbo": { "total_deductions": 363.0, "sample_count": 11, "avg_deductions_per_sample": 33.0 } }, "dialect": { "apple_speechanalyzer": { "total_deductions": 751.0, "sample_count": 21, "avg_deductions_per_sample": 35.76 }, "doubao": { "total_deductions": 207.0, "sample_count": 21, "avg_deductions_per_sample": 9.86 }, "elevenlabs": { "total_deductions": 264.0, "sample_count": 21, "avg_deductions_per_sample": 12.57 }, "firered": { "total_deductions": 854.0, "sample_count": 20, "avg_deductions_per_sample": 42.7 }, "firered_aed": { "total_deductions": 434.0, "sample_count": 21, "avg_deductions_per_sample": 20.67 }, "flash": { "total_deductions": 187.0, "sample_count": 21, "avg_deductions_per_sample": 8.9 }, "funasr_nano": { "total_deductions": 571.0, "sample_count": 21, "avg_deductions_per_sample": 27.19 }, "gemini_3_5_flash_lite_stt": { "total_deductions": 815.0, "sample_count": 21, "avg_deductions_per_sample": 38.81 }, "gemini_3_5_flash_stt": { "total_deductions": 306.0, "sample_count": 21, "avg_deductions_per_sample": 14.57 }, "gemini_3_5_transcribe": { "total_deductions": 466.0, "sample_count": 21, "avg_deductions_per_sample": 22.19 }, "gemini_3_6_flash_stt": { "total_deductions": 445.0, "sample_count": 21, "avg_deductions_per_sample": 21.19 }, "gemini_3_7_flash_stt": { "total_deductions": 338.0, "sample_count": 20, "avg_deductions_per_sample": 16.9 }, "gemini_3_8_flash_stt": { "total_deductions": 217.0, "sample_count": 21, "avg_deductions_per_sample": 10.33 }, "gemini_flash_stt": { "total_deductions": 392.0, "sample_count": 21, "avg_deductions_per_sample": 18.67 }, "gemini_pro_stt": { "total_deductions": 247.0, "sample_count": 21, "avg_deductions_per_sample": 11.76 }, "gemini_stt": { "total_deductions": 591.0, "sample_count": 21, "avg_deductions_per_sample": 28.14 }, "glm_asr_nano": { "total_deductions": 424.0, "sample_count": 21, "avg_deductions_per_sample": 20.19 }, "gpt_transcribe": { "total_deductions": 298.0, "sample_count": 21, "avg_deductions_per_sample": 14.19 }, "mai_transcribe_2": { "total_deductions": 161.0, "sample_count": 21, "avg_deductions_per_sample": 7.67 }, "mimo": { "total_deductions": 235.0, "sample_count": 21, "avg_deductions_per_sample": 11.19 }, "mimo_stt": { "total_deductions": 308.0, "sample_count": 21, "avg_deductions_per_sample": 14.67 }, "moss_audio": { "total_deductions": 593.0, "sample_count": 19, "avg_deductions_per_sample": 31.21 }, "qwen3_oss": { "total_deductions": 424.0, "sample_count": 21, "avg_deductions_per_sample": 20.19 }, "qwen_audio_3_1_asr": { "total_deductions": 225.0, "sample_count": 21, "avg_deductions_per_sample": 10.71 }, "r2t2": { "total_deductions": 259.0, "sample_count": 21, "avg_deductions_per_sample": 12.33 }, "soniox": { "total_deductions": 304.0, "sample_count": 21, "avg_deductions_per_sample": 14.48 }, "stepfun": { "total_deductions": 308.0, "sample_count": 21, "avg_deductions_per_sample": 14.67 }, "vibevoice": { "total_deductions": 283.0, "sample_count": 21, "avg_deductions_per_sample": 13.48 }, "voxtral": { "total_deductions": 595.0, "sample_count": 21, "avg_deductions_per_sample": 28.33 }, "whisper": { "total_deductions": 366.0, "sample_count": 21, "avg_deductions_per_sample": 17.43 }, "whisper_v3_turbo": { "total_deductions": 513.0, "sample_count": 21, "avg_deductions_per_sample": 24.43 } }, "has-tech": { "apple_speechanalyzer": { "total_deductions": 813.0, "sample_count": 19, "avg_deductions_per_sample": 42.79 }, "doubao": { "total_deductions": 245.0, "sample_count": 19, "avg_deductions_per_sample": 12.89 }, "elevenlabs": { "total_deductions": 231.0, "sample_count": 19, "avg_deductions_per_sample": 12.16 }, "firered": { "total_deductions": 1079.0, "sample_count": 19, "avg_deductions_per_sample": 56.79 }, "firered_aed": { "total_deductions": 627.0, "sample_count": 19, "avg_deductions_per_sample": 33.0 }, "flash": { "total_deductions": 192.0, "sample_count": 19, "avg_deductions_per_sample": 10.11 }, "funasr_nano": { "total_deductions": 547.0, "sample_count": 18, "avg_deductions_per_sample": 30.39 }, "gemini_3_5_flash_lite_stt": { "total_deductions": 608.0, "sample_count": 19, "avg_deductions_per_sample": 32.0 }, "gemini_3_5_flash_stt": { "total_deductions": 334.0, "sample_count": 19, "avg_deductions_per_sample": 17.58 }, "gemini_3_5_transcribe": { "total_deductions": 330.0, "sample_count": 19, "avg_deductions_per_sample": 17.37 }, "gemini_3_6_flash_stt": { "total_deductions": 344.0, "sample_count": 19, "avg_deductions_per_sample": 18.11 }, "gemini_3_7_flash_stt": { "total_deductions": 285.0, "sample_count": 19, "avg_deductions_per_sample": 15.0 }, "gemini_3_8_flash_stt": { "total_deductions": 203.0, "sample_count": 19, "avg_deductions_per_sample": 10.68 }, "gemini_flash_stt": { "total_deductions": 318.0, "sample_count": 19, "avg_deductions_per_sample": 16.74 }, "gemini_pro_stt": { "total_deductions": 251.0, "sample_count": 19, "avg_deductions_per_sample": 13.21 }, "gemini_stt": { "total_deductions": 530.0, "sample_count": 19, "avg_deductions_per_sample": 27.89 }, "glm_asr_nano": { "total_deductions": 467.0, "sample_count": 19, "avg_deductions_per_sample": 24.58 }, "gpt_transcribe": { "total_deductions": 244.0, "sample_count": 19, "avg_deductions_per_sample": 12.84 }, "mai_transcribe_2": { "total_deductions": 144.0, "sample_count": 19, "avg_deductions_per_sample": 7.58 }, "mimo": { "total_deductions": 212.0, "sample_count": 19, "avg_deductions_per_sample": 11.16 }, "mimo_stt": { "total_deductions": 272.0, "sample_count": 19, "avg_deductions_per_sample": 14.32 }, "moss_audio": { "total_deductions": 651.0, "sample_count": 18, "avg_deductions_per_sample": 36.17 }, "qwen3_oss": { "total_deductions": 373.0, "sample_count": 19, "avg_deductions_per_sample": 19.63 }, "qwen_audio_3_1_asr": { "total_deductions": 182.0, "sample_count": 19, "avg_deductions_per_sample": 9.58 }, "r2t2": { "total_deductions": 266.0, "sample_count": 19, "avg_deductions_per_sample": 14.0 }, "soniox": { "total_deductions": 258.0, "sample_count": 19, "avg_deductions_per_sample": 13.58 }, "stepfun": { "total_deductions": 381.0, "sample_count": 19, "avg_deductions_per_sample": 20.05 }, "vibevoice": { "total_deductions": 249.0, "sample_count": 19, "avg_deductions_per_sample": 13.11 }, "voxtral": { "total_deductions": 507.0, "sample_count": 19, "avg_deductions_per_sample": 26.68 }, "whisper": { "total_deductions": 293.0, "sample_count": 19, "avg_deductions_per_sample": 15.42 }, "whisper_v3_turbo": { "total_deductions": 591.0, "sample_count": 19, "avg_deductions_per_sample": 31.11 } }, "idiom-classical": { "apple_speechanalyzer": { "total_deductions": 278.0, "sample_count": 8, "avg_deductions_per_sample": 34.75 }, "doubao": { "total_deductions": 65.0, "sample_count": 8, "avg_deductions_per_sample": 8.12 }, "elevenlabs": { "total_deductions": 93.0, "sample_count": 8, "avg_deductions_per_sample": 11.62 }, "firered": { "total_deductions": 257.0, "sample_count": 7, "avg_deductions_per_sample": 36.71 }, "firered_aed": { "total_deductions": 174.0, "sample_count": 8, "avg_deductions_per_sample": 21.75 }, "flash": { "total_deductions": 60.0, "sample_count": 8, "avg_deductions_per_sample": 7.5 }, "funasr_nano": { "total_deductions": 218.0, "sample_count": 8, "avg_deductions_per_sample": 27.25 }, "gemini_3_5_flash_lite_stt": { "total_deductions": 356.0, "sample_count": 8, "avg_deductions_per_sample": 44.5 }, "gemini_3_5_flash_stt": { "total_deductions": 140.0, "sample_count": 8, "avg_deductions_per_sample": 17.5 }, "gemini_3_5_transcribe": { "total_deductions": 176.0, "sample_count": 8, "avg_deductions_per_sample": 22.0 }, "gemini_3_6_flash_stt": { "total_deductions": 169.0, "sample_count": 8, "avg_deductions_per_sample": 21.12 }, "gemini_3_7_flash_stt": { "total_deductions": 144.0, "sample_count": 8, "avg_deductions_per_sample": 18.0 }, "gemini_3_8_flash_stt": { "total_deductions": 94.0, "sample_count": 8, "avg_deductions_per_sample": 11.75 }, "gemini_flash_stt": { "total_deductions": 144.0, "sample_count": 8, "avg_deductions_per_sample": 18.0 }, "gemini_pro_stt": { "total_deductions": 82.0, "sample_count": 8, "avg_deductions_per_sample": 10.25 }, "gemini_stt": { "total_deductions": 229.0, "sample_count": 8, "avg_deductions_per_sample": 28.62 }, "glm_asr_nano": { "total_deductions": 103.0, "sample_count": 8, "avg_deductions_per_sample": 12.88 }, "gpt_transcribe": { "total_deductions": 71.0, "sample_count": 8, "avg_deductions_per_sample": 8.88 }, "mai_transcribe_2": { "total_deductions": 59.0, "sample_count": 8, "avg_deductions_per_sample": 7.38 }, "mimo": { "total_deductions": 45.0, "sample_count": 8, "avg_deductions_per_sample": 5.62 }, "mimo_stt": { "total_deductions": 79.0, "sample_count": 8, "avg_deductions_per_sample": 9.88 }, "moss_audio": { "total_deductions": 199.0, "sample_count": 8, "avg_deductions_per_sample": 24.88 }, "qwen3_oss": { "total_deductions": 173.0, "sample_count": 8, "avg_deductions_per_sample": 21.62 }, "qwen_audio_3_1_asr": { "total_deductions": 51.0, "sample_count": 8, "avg_deductions_per_sample": 6.38 }, "r2t2": { "total_deductions": 57.0, "sample_count": 8, "avg_deductions_per_sample": 7.12 }, "soniox": { "total_deductions": 162.0, "sample_count": 8, "avg_deductions_per_sample": 20.25 }, "stepfun": { "total_deductions": 152.0, "sample_count": 8, "avg_deductions_per_sample": 19.0 }, "vibevoice": { "total_deductions": 79.0, "sample_count": 8, "avg_deductions_per_sample": 9.88 }, "voxtral": { "total_deductions": 298.0, "sample_count": 8, "avg_deductions_per_sample": 37.25 }, "whisper": { "total_deductions": 192.0, "sample_count": 8, "avg_deductions_per_sample": 24.0 }, "whisper_v3_turbo": { "total_deductions": 323.0, "sample_count": 8, "avg_deductions_per_sample": 40.38 } }, "long": { "apple_speechanalyzer": { "total_deductions": 1101.0, "sample_count": 24, "avg_deductions_per_sample": 45.88 }, "doubao": { "total_deductions": 287.0, "sample_count": 24, "avg_deductions_per_sample": 11.96 }, "elevenlabs": { "total_deductions": 257.0, "sample_count": 24, "avg_deductions_per_sample": 10.71 }, "firered": { "total_deductions": 1296.0, "sample_count": 23, "avg_deductions_per_sample": 56.35 }, "firered_aed": { "total_deductions": 642.0, "sample_count": 24, "avg_deductions_per_sample": 26.75 }, "flash": { "total_deductions": 313.0, "sample_count": 24, "avg_deductions_per_sample": 13.04 }, "funasr_nano": { "total_deductions": 876.0, "sample_count": 23, "avg_deductions_per_sample": 38.09 }, "gemini_3_5_flash_lite_stt": { "total_deductions": 1018.0, "sample_count": 24, "avg_deductions_per_sample": 42.42 }, "gemini_3_5_flash_stt": { "total_deductions": 444.0, "sample_count": 24, "avg_deductions_per_sample": 18.5 }, "gemini_3_5_transcribe": { "total_deductions": 522.0, "sample_count": 24, "avg_deductions_per_sample": 21.75 }, "gemini_3_6_flash_stt": { "total_deductions": 539.0, "sample_count": 24, "avg_deductions_per_sample": 22.46 }, "gemini_3_7_flash_stt": { "total_deductions": 389.0, "sample_count": 24, "avg_deductions_per_sample": 16.21 }, "gemini_3_8_flash_stt": { "total_deductions": 277.0, "sample_count": 23, "avg_deductions_per_sample": 12.04 }, "gemini_flash_stt": { "total_deductions": 466.0, "sample_count": 24, "avg_deductions_per_sample": 19.42 }, "gemini_pro_stt": { "total_deductions": 423.0, "sample_count": 24, "avg_deductions_per_sample": 17.62 }, "gemini_stt": { "total_deductions": 682.0, "sample_count": 24, "avg_deductions_per_sample": 28.42 }, "glm_asr_nano": { "total_deductions": 485.0, "sample_count": 24, "avg_deductions_per_sample": 20.21 }, "gpt_transcribe": { "total_deductions": 315.0, "sample_count": 24, "avg_deductions_per_sample": 13.12 }, "mai_transcribe_2": { "total_deductions": 209.0, "sample_count": 24, "avg_deductions_per_sample": 8.71 }, "mimo": { "total_deductions": 271.0, "sample_count": 24, "avg_deductions_per_sample": 11.29 }, "mimo_stt": { "total_deductions": 357.0, "sample_count": 24, "avg_deductions_per_sample": 14.88 }, "moss_audio": { "total_deductions": 915.0, "sample_count": 22, "avg_deductions_per_sample": 41.59 }, "qwen3_oss": { "total_deductions": 507.0, "sample_count": 24, "avg_deductions_per_sample": 21.12 }, "qwen_audio_3_1_asr": { "total_deductions": 268.0, "sample_count": 24, "avg_deductions_per_sample": 11.17 }, "r2t2": { "total_deductions": 313.0, "sample_count": 24, "avg_deductions_per_sample": 13.04 }, "soniox": { "total_deductions": 360.0, "sample_count": 24, "avg_deductions_per_sample": 15.0 }, "stepfun": { "total_deductions": 458.0, "sample_count": 24, "avg_deductions_per_sample": 19.08 }, "vibevoice": { "total_deductions": 353.0, "sample_count": 24, "avg_deductions_per_sample": 14.71 }, "voxtral": { "total_deductions": 759.0, "sample_count": 24, "avg_deductions_per_sample": 31.62 }, "whisper": { "total_deductions": 413.0, "sample_count": 24, "avg_deductions_per_sample": 17.21 }, "whisper_v3_turbo": { "total_deductions": 792.0, "sample_count": 23, "avg_deductions_per_sample": 34.43 } }, "numeric": { "apple_speechanalyzer": { "total_deductions": 1831.0, "sample_count": 71, "avg_deductions_per_sample": 25.79 }, "doubao": { "total_deductions": 671.0, "sample_count": 71, "avg_deductions_per_sample": 9.45 }, "elevenlabs": { "total_deductions": 492.0, "sample_count": 71, "avg_deductions_per_sample": 6.93 }, "firered": { "total_deductions": 2144.0, "sample_count": 70, "avg_deductions_per_sample": 30.63 }, "firered_aed": { "total_deductions": 1094.0, "sample_count": 71, "avg_deductions_per_sample": 15.41 }, "flash": { "total_deductions": 557.0, "sample_count": 71, "avg_deductions_per_sample": 7.85 }, "funasr_nano": { "total_deductions": 1358.0, "sample_count": 70, "avg_deductions_per_sample": 19.4 }, "gemini_3_5_flash_lite_stt": { "total_deductions": 1731.0, "sample_count": 71, "avg_deductions_per_sample": 24.38 }, "gemini_3_5_flash_stt": { "total_deductions": 851.0, "sample_count": 71, "avg_deductions_per_sample": 11.99 }, "gemini_3_5_transcribe": { "total_deductions": 866.0, "sample_count": 71, "avg_deductions_per_sample": 12.2 }, "gemini_3_6_flash_stt": { "total_deductions": 999.0, "sample_count": 71, "avg_deductions_per_sample": 14.07 }, "gemini_3_7_flash_stt": { "total_deductions": 740.0, "sample_count": 70, "avg_deductions_per_sample": 10.57 }, "gemini_3_8_flash_stt": { "total_deductions": 525.0, "sample_count": 70, "avg_deductions_per_sample": 7.5 }, "gemini_flash_stt": { "total_deductions": 808.0, "sample_count": 71, "avg_deductions_per_sample": 11.38 }, "gemini_pro_stt": { "total_deductions": 592.0, "sample_count": 71, "avg_deductions_per_sample": 8.34 }, "gemini_stt": { "total_deductions": 1169.0, "sample_count": 71, "avg_deductions_per_sample": 16.46 }, "glm_asr_nano": { "total_deductions": 983.0, "sample_count": 71, "avg_deductions_per_sample": 13.85 }, "gpt_transcribe": { "total_deductions": 631.0, "sample_count": 71, "avg_deductions_per_sample": 8.89 }, "mai_transcribe_2": { "total_deductions": 390.0, "sample_count": 71, "avg_deductions_per_sample": 5.49 }, "mimo": { "total_deductions": 408.0, "sample_count": 71, "avg_deductions_per_sample": 5.75 }, "mimo_stt": { "total_deductions": 601.0, "sample_count": 71, "avg_deductions_per_sample": 8.46 }, "moss_audio": { "total_deductions": 1776.0, "sample_count": 69, "avg_deductions_per_sample": 25.74 }, "qwen3_oss": { "total_deductions": 797.0, "sample_count": 70, "avg_deductions_per_sample": 11.39 }, "qwen_audio_3_1_asr": { "total_deductions": 515.0, "sample_count": 71, "avg_deductions_per_sample": 7.25 }, "r2t2": { "total_deductions": 616.0, "sample_count": 71, "avg_deductions_per_sample": 8.68 }, "soniox": { "total_deductions": 631.0, "sample_count": 71, "avg_deductions_per_sample": 8.89 }, "stepfun": { "total_deductions": 970.0, "sample_count": 71, "avg_deductions_per_sample": 13.66 }, "vibevoice": { "total_deductions": 716.0, "sample_count": 71, "avg_deductions_per_sample": 10.08 }, "voxtral": { "total_deductions": 1481.0, "sample_count": 71, "avg_deductions_per_sample": 20.86 }, "whisper": { "total_deductions": 782.0, "sample_count": 71, "avg_deductions_per_sample": 11.01 }, "whisper_v3_turbo": { "total_deductions": 1455.0, "sample_count": 71, "avg_deductions_per_sample": 20.49 } }, "polarity-dense": { "apple_speechanalyzer": { "total_deductions": 1630.0, "sample_count": 66, "avg_deductions_per_sample": 24.7 }, "doubao": { "total_deductions": 612.0, "sample_count": 66, "avg_deductions_per_sample": 9.27 }, "elevenlabs": { "total_deductions": 542.0, "sample_count": 66, "avg_deductions_per_sample": 8.21 }, "firered": { "total_deductions": 2012.0, "sample_count": 65, "avg_deductions_per_sample": 30.95 }, "firered_aed": { "total_deductions": 1021.0, "sample_count": 66, "avg_deductions_per_sample": 15.47 }, "flash": { "total_deductions": 521.0, "sample_count": 66, "avg_deductions_per_sample": 7.89 }, "funasr_nano": { "total_deductions": 1183.0, "sample_count": 65, "avg_deductions_per_sample": 18.2 }, "gemini_3_5_flash_lite_stt": { "total_deductions": 1717.0, "sample_count": 66, "avg_deductions_per_sample": 26.02 }, "gemini_3_5_flash_stt": { "total_deductions": 800.0, "sample_count": 66, "avg_deductions_per_sample": 12.12 }, "gemini_3_5_transcribe": { "total_deductions": 895.0, "sample_count": 66, "avg_deductions_per_sample": 13.56 }, "gemini_3_6_flash_stt": { "total_deductions": 938.0, "sample_count": 66, "avg_deductions_per_sample": 14.21 }, "gemini_3_7_flash_stt": { "total_deductions": 716.0, "sample_count": 64, "avg_deductions_per_sample": 11.19 }, "gemini_3_8_flash_stt": { "total_deductions": 507.0, "sample_count": 64, "avg_deductions_per_sample": 7.92 }, "gemini_flash_stt": { "total_deductions": 766.0, "sample_count": 66, "avg_deductions_per_sample": 11.61 }, "gemini_pro_stt": { "total_deductions": 589.0, "sample_count": 66, "avg_deductions_per_sample": 8.92 }, "gemini_stt": { "total_deductions": 1169.0, "sample_count": 66, "avg_deductions_per_sample": 17.71 }, "glm_asr_nano": { "total_deductions": 887.0, "sample_count": 66, "avg_deductions_per_sample": 13.44 }, "gpt_transcribe": { "total_deductions": 605.0, "sample_count": 66, "avg_deductions_per_sample": 9.17 }, "mai_transcribe_2": { "total_deductions": 330.0, "sample_count": 66, "avg_deductions_per_sample": 5.0 }, "mimo": { "total_deductions": 415.0, "sample_count": 66, "avg_deductions_per_sample": 6.29 }, "mimo_stt": { "total_deductions": 659.0, "sample_count": 66, "avg_deductions_per_sample": 9.98 }, "moss_audio": { "total_deductions": 1711.0, "sample_count": 64, "avg_deductions_per_sample": 26.73 }, "qwen3_oss": { "total_deductions": 819.0, "sample_count": 66, "avg_deductions_per_sample": 12.41 }, "qwen_audio_3_1_asr": { "total_deductions": 510.0, "sample_count": 66, "avg_deductions_per_sample": 7.73 }, "r2t2": { "total_deductions": 626.0, "sample_count": 66, "avg_deductions_per_sample": 9.48 }, "soniox": { "total_deductions": 571.0, "sample_count": 66, "avg_deductions_per_sample": 8.65 }, "stepfun": { "total_deductions": 909.0, "sample_count": 66, "avg_deductions_per_sample": 13.77 }, "vibevoice": { "total_deductions": 629.0, "sample_count": 66, "avg_deductions_per_sample": 9.53 }, "voxtral": { "total_deductions": 1441.0, "sample_count": 66, "avg_deductions_per_sample": 21.83 }, "whisper": { "total_deductions": 749.0, "sample_count": 66, "avg_deductions_per_sample": 11.35 }, "whisper_v3_turbo": { "total_deductions": 1314.0, "sample_count": 65, "avg_deductions_per_sample": 20.22 } }, "pure-chinese": { "apple_speechanalyzer": { "total_deductions": 452.0, "sample_count": 36, "avg_deductions_per_sample": 12.56 }, "doubao": { "total_deductions": 112.0, "sample_count": 36, "avg_deductions_per_sample": 3.11 }, "elevenlabs": { "total_deductions": 169.0, "sample_count": 36, "avg_deductions_per_sample": 4.69 }, "firered": { "total_deductions": 452.0, "sample_count": 36, "avg_deductions_per_sample": 12.56 }, "firered_aed": { "total_deductions": 228.0, "sample_count": 36, "avg_deductions_per_sample": 6.33 }, "flash": { "total_deductions": 118.0, "sample_count": 36, "avg_deductions_per_sample": 3.28 }, "funasr_nano": { "total_deductions": 202.0, "sample_count": 36, "avg_deductions_per_sample": 5.61 }, "gemini_3_5_flash_lite_stt": { "total_deductions": 669.0, "sample_count": 36, "avg_deductions_per_sample": 18.58 }, "gemini_3_5_flash_stt": { "total_deductions": 302.0, "sample_count": 36, "avg_deductions_per_sample": 8.39 }, "gemini_3_5_transcribe": { "total_deductions": 227.0, "sample_count": 36, "avg_deductions_per_sample": 6.31 }, "gemini_3_6_flash_stt": { "total_deductions": 361.0, "sample_count": 36, "avg_deductions_per_sample": 10.03 }, "gemini_3_7_flash_stt": { "total_deductions": 192.0, "sample_count": 33, "avg_deductions_per_sample": 5.82 }, "gemini_3_8_flash_stt": { "total_deductions": 188.0, "sample_count": 34, "avg_deductions_per_sample": 5.53 }, "gemini_flash_stt": { "total_deductions": 278.0, "sample_count": 36, "avg_deductions_per_sample": 7.72 }, "gemini_pro_stt": { "total_deductions": 164.0, "sample_count": 36, "avg_deductions_per_sample": 4.56 }, "gemini_stt": { "total_deductions": 378.0, "sample_count": 36, "avg_deductions_per_sample": 10.5 }, "glm_asr_nano": { "total_deductions": 250.0, "sample_count": 36, "avg_deductions_per_sample": 6.94 }, "gpt_transcribe": { "total_deductions": 242.0, "sample_count": 36, "avg_deductions_per_sample": 6.72 }, "mai_transcribe_2": { "total_deductions": 92.0, "sample_count": 36, "avg_deductions_per_sample": 2.56 }, "mimo": { "total_deductions": 101.0, "sample_count": 36, "avg_deductions_per_sample": 2.81 }, "mimo_stt": { "total_deductions": 233.0, "sample_count": 36, "avg_deductions_per_sample": 6.47 }, "moss_audio": { "total_deductions": 648.0, "sample_count": 36, "avg_deductions_per_sample": 18.0 }, "qwen3_oss": { "total_deductions": 182.0, "sample_count": 36, "avg_deductions_per_sample": 5.06 }, "qwen_audio_3_1_asr": { "total_deductions": 129.0, "sample_count": 36, "avg_deductions_per_sample": 3.58 }, "r2t2": { "total_deductions": 156.0, "sample_count": 36, "avg_deductions_per_sample": 4.33 }, "soniox": { "total_deductions": 269.0, "sample_count": 36, "avg_deductions_per_sample": 7.47 }, "stepfun": { "total_deductions": 340.0, "sample_count": 36, "avg_deductions_per_sample": 9.44 }, "vibevoice": { "total_deductions": 176.0, "sample_count": 36, "avg_deductions_per_sample": 4.89 }, "voxtral": { "total_deductions": 496.0, "sample_count": 36, "avg_deductions_per_sample": 13.78 }, "whisper": { "total_deductions": 283.0, "sample_count": 36, "avg_deductions_per_sample": 7.86 }, "whisper_v3_turbo": { "total_deductions": 440.0, "sample_count": 36, "avg_deductions_per_sample": 12.22 } } }, "per_condition_stats": { "ambient": { "apple_speechanalyzer": { "total_deductions": 121.0, "n_samples": 4, "avg_ded_per_sample": 30.25 }, "doubao": { "total_deductions": 43.0, "n_samples": 4, "avg_ded_per_sample": 10.75 }, "elevenlabs": { "total_deductions": 36.0, "n_samples": 4, "avg_ded_per_sample": 9.0 }, "firered": { "total_deductions": 162.0, "n_samples": 4, "avg_ded_per_sample": 40.5 }, "firered_aed": { "total_deductions": 99.0, "n_samples": 4, "avg_ded_per_sample": 24.75 }, "flash": { "total_deductions": 78.0, "n_samples": 4, "avg_ded_per_sample": 19.5 }, "funasr_nano": { "total_deductions": 91.0, "n_samples": 4, "avg_ded_per_sample": 22.75 }, "gemini_3_5_flash_lite_stt": { "total_deductions": 119.0, "n_samples": 4, "avg_ded_per_sample": 29.75 }, "gemini_3_5_flash_stt": { "total_deductions": 49.0, "n_samples": 4, "avg_ded_per_sample": 12.25 }, "gemini_3_5_transcribe": { "total_deductions": 114.0, "n_samples": 4, "avg_ded_per_sample": 28.5 }, "gemini_3_6_flash_stt": { "total_deductions": 69.0, "n_samples": 4, "avg_ded_per_sample": 17.25 }, "gemini_3_7_flash_stt": { "total_deductions": 40.0, "n_samples": 4, "avg_ded_per_sample": 10.0 }, "gemini_3_8_flash_stt": { "total_deductions": 35.0, "n_samples": 4, "avg_ded_per_sample": 8.75 }, "gemini_flash_stt": { "total_deductions": 61.0, "n_samples": 4, "avg_ded_per_sample": 15.25 }, "gemini_pro_stt": { "total_deductions": 19.0, "n_samples": 4, "avg_ded_per_sample": 4.75 }, "gemini_stt": { "total_deductions": 82.0, "n_samples": 4, "avg_ded_per_sample": 20.5 }, "glm_asr_nano": { "total_deductions": 89.0, "n_samples": 4, "avg_ded_per_sample": 22.25 }, "gpt_transcribe": { "total_deductions": 60.0, "n_samples": 4, "avg_ded_per_sample": 15.0 }, "mai_transcribe_2": { "total_deductions": 30.0, "n_samples": 4, "avg_ded_per_sample": 7.5 }, "mimo": { "total_deductions": 17.0, "n_samples": 4, "avg_ded_per_sample": 4.25 }, "mimo_stt": { "total_deductions": 75.0, "n_samples": 4, "avg_ded_per_sample": 18.75 }, "moss_audio": { "total_deductions": 128.0, "n_samples": 4, "avg_ded_per_sample": 32.0 }, "qwen3_oss": { "total_deductions": 68.0, "n_samples": 4, "avg_ded_per_sample": 17.0 }, "qwen_audio_3_1_asr": { "total_deductions": 45.0, "n_samples": 4, "avg_ded_per_sample": 11.25 }, "r2t2": { "total_deductions": 81.0, "n_samples": 4, "avg_ded_per_sample": 20.25 }, "soniox": { "total_deductions": 48.0, "n_samples": 4, "avg_ded_per_sample": 12.0 }, "stepfun": { "total_deductions": 90.0, "n_samples": 4, "avg_ded_per_sample": 22.5 }, "vibevoice": { "total_deductions": 95.0, "n_samples": 4, "avg_ded_per_sample": 23.75 }, "voxtral": { "total_deductions": 150.0, "n_samples": 4, "avg_ded_per_sample": 37.5 }, "whisper": { "total_deductions": 69.0, "n_samples": 4, "avg_ded_per_sample": 17.25 }, "whisper_v3_turbo": { "total_deductions": 143.0, "n_samples": 4, "avg_ded_per_sample": 35.75 } }, "bg-noise": { "apple_speechanalyzer": { "total_deductions": 907.0, "n_samples": 46, "avg_ded_per_sample": 19.72 }, "doubao": { "total_deductions": 460.0, "n_samples": 46, "avg_ded_per_sample": 10.0 }, "elevenlabs": { "total_deductions": 314.0, "n_samples": 46, "avg_ded_per_sample": 6.83 }, "firered": { "total_deductions": 1165.0, "n_samples": 46, "avg_ded_per_sample": 25.33 }, "firered_aed": { "total_deductions": 638.0, "n_samples": 46, "avg_ded_per_sample": 13.87 }, "flash": { "total_deductions": 325.0, "n_samples": 46, "avg_ded_per_sample": 7.07 }, "funasr_nano": { "total_deductions": 672.0, "n_samples": 46, "avg_ded_per_sample": 14.61 }, "gemini_3_5_flash_lite_stt": { "total_deductions": 776.0, "n_samples": 46, "avg_ded_per_sample": 16.87 }, "gemini_3_5_flash_stt": { "total_deductions": 384.0, "n_samples": 46, "avg_ded_per_sample": 8.35 }, "gemini_3_5_transcribe": { "total_deductions": 397.0, "n_samples": 46, "avg_ded_per_sample": 8.63 }, "gemini_3_6_flash_stt": { "total_deductions": 492.0, "n_samples": 46, "avg_ded_per_sample": 10.7 }, "gemini_3_7_flash_stt": { "total_deductions": 409.0, "n_samples": 46, "avg_ded_per_sample": 8.89 }, "gemini_3_8_flash_stt": { "total_deductions": 277.0, "n_samples": 46, "avg_ded_per_sample": 6.02 }, "gemini_flash_stt": { "total_deductions": 362.0, "n_samples": 46, "avg_ded_per_sample": 7.87 }, "gemini_pro_stt": { "total_deductions": 296.0, "n_samples": 46, "avg_ded_per_sample": 6.43 }, "gemini_stt": { "total_deductions": 619.0, "n_samples": 46, "avg_ded_per_sample": 13.46 }, "glm_asr_nano": { "total_deductions": 555.0, "n_samples": 46, "avg_ded_per_sample": 12.07 }, "gpt_transcribe": { "total_deductions": 311.0, "n_samples": 46, "avg_ded_per_sample": 6.76 }, "mai_transcribe_2": { "total_deductions": 148.0, "n_samples": 46, "avg_ded_per_sample": 3.22 }, "mimo": { "total_deductions": 222.0, "n_samples": 46, "avg_ded_per_sample": 4.83 }, "mimo_stt": { "total_deductions": 386.0, "n_samples": 46, "avg_ded_per_sample": 8.39 }, "moss_audio": { "total_deductions": 1005.0, "n_samples": 46, "avg_ded_per_sample": 21.85 }, "qwen3_oss": { "total_deductions": 444.0, "n_samples": 46, "avg_ded_per_sample": 9.65 }, "qwen_audio_3_1_asr": { "total_deductions": 330.0, "n_samples": 46, "avg_ded_per_sample": 7.17 }, "r2t2": { "total_deductions": 387.0, "n_samples": 46, "avg_ded_per_sample": 8.41 }, "soniox": { "total_deductions": 310.0, "n_samples": 46, "avg_ded_per_sample": 6.74 }, "stepfun": { "total_deductions": 649.0, "n_samples": 46, "avg_ded_per_sample": 14.11 }, "vibevoice": { "total_deductions": 378.0, "n_samples": 46, "avg_ded_per_sample": 8.22 }, "voxtral": { "total_deductions": 792.0, "n_samples": 46, "avg_ded_per_sample": 17.22 }, "whisper": { "total_deductions": 390.0, "n_samples": 46, "avg_ded_per_sample": 8.48 }, "whisper_v3_turbo": { "total_deductions": 659.0, "n_samples": 46, "avg_ded_per_sample": 14.33 } }, "clean": { "apple_speechanalyzer": { "total_deductions": 1385.0, "n_samples": 55, "avg_ded_per_sample": 25.18 }, "doubao": { "total_deductions": 385.0, "n_samples": 55, "avg_ded_per_sample": 7.0 }, "elevenlabs": { "total_deductions": 408.0, "n_samples": 55, "avg_ded_per_sample": 7.42 }, "firered": { "total_deductions": 1434.0, "n_samples": 55, "avg_ded_per_sample": 26.07 }, "firered_aed": { "total_deductions": 744.0, "n_samples": 55, "avg_ded_per_sample": 13.53 }, "flash": { "total_deductions": 376.0, "n_samples": 55, "avg_ded_per_sample": 6.84 }, "funasr_nano": { "total_deductions": 931.0, "n_samples": 55, "avg_ded_per_sample": 16.93 }, "gemini_3_5_flash_lite_stt": { "total_deductions": 1479.0, "n_samples": 55, "avg_ded_per_sample": 26.89 }, "gemini_3_5_flash_stt": { "total_deductions": 686.0, "n_samples": 55, "avg_ded_per_sample": 12.47 }, "gemini_3_5_transcribe": { "total_deductions": 674.0, "n_samples": 55, "avg_ded_per_sample": 12.25 }, "gemini_3_6_flash_stt": { "total_deductions": 837.0, "n_samples": 55, "avg_ded_per_sample": 15.22 }, "gemini_3_7_flash_stt": { "total_deductions": 498.0, "n_samples": 55, "avg_ded_per_sample": 9.05 }, "gemini_3_8_flash_stt": { "total_deductions": 411.0, "n_samples": 55, "avg_ded_per_sample": 7.47 }, "gemini_flash_stt": { "total_deductions": 654.0, "n_samples": 55, "avg_ded_per_sample": 11.89 }, "gemini_pro_stt": { "total_deductions": 464.0, "n_samples": 55, "avg_ded_per_sample": 8.44 }, "gemini_stt": { "total_deductions": 975.0, "n_samples": 55, "avg_ded_per_sample": 17.73 }, "glm_asr_nano": { "total_deductions": 679.0, "n_samples": 55, "avg_ded_per_sample": 12.35 }, "gpt_transcribe": { "total_deductions": 499.0, "n_samples": 55, "avg_ded_per_sample": 9.07 }, "mai_transcribe_2": { "total_deductions": 294.0, "n_samples": 55, "avg_ded_per_sample": 5.35 }, "mimo": { "total_deductions": 358.0, "n_samples": 55, "avg_ded_per_sample": 6.51 }, "mimo_stt": { "total_deductions": 513.0, "n_samples": 55, "avg_ded_per_sample": 9.33 }, "moss_audio": { "total_deductions": 1198.0, "n_samples": 55, "avg_ded_per_sample": 21.78 }, "qwen3_oss": { "total_deductions": 586.0, "n_samples": 55, "avg_ded_per_sample": 10.65 }, "qwen_audio_3_1_asr": { "total_deductions": 405.0, "n_samples": 55, "avg_ded_per_sample": 7.36 }, "r2t2": { "total_deductions": 410.0, "n_samples": 55, "avg_ded_per_sample": 7.45 }, "soniox": { "total_deductions": 561.0, "n_samples": 55, "avg_ded_per_sample": 10.2 }, "stepfun": { "total_deductions": 668.0, "n_samples": 55, "avg_ded_per_sample": 12.15 }, "vibevoice": { "total_deductions": 445.0, "n_samples": 55, "avg_ded_per_sample": 8.09 }, "voxtral": { "total_deductions": 1089.0, "n_samples": 55, "avg_ded_per_sample": 19.8 }, "whisper": { "total_deductions": 634.0, "n_samples": 55, "avg_ded_per_sample": 11.53 }, "whisper_v3_turbo": { "total_deductions": 1093.0, "n_samples": 55, "avg_ded_per_sample": 19.87 } }, "close-mic": { "apple_speechanalyzer": { "total_deductions": 73.0, "n_samples": 5, "avg_ded_per_sample": 14.6 }, "doubao": { "total_deductions": 63.0, "n_samples": 5, "avg_ded_per_sample": 12.6 }, "elevenlabs": { "total_deductions": 43.0, "n_samples": 5, "avg_ded_per_sample": 8.6 }, "firered": { "total_deductions": 168.0, "n_samples": 5, "avg_ded_per_sample": 33.6 }, "firered_aed": { "total_deductions": 46.0, "n_samples": 5, "avg_ded_per_sample": 9.2 }, "flash": { "total_deductions": 51.0, "n_samples": 5, "avg_ded_per_sample": 10.2 }, "funasr_nano": { "total_deductions": 56.0, "n_samples": 5, "avg_ded_per_sample": 11.2 }, "gemini_3_5_flash_lite_stt": { "total_deductions": 81.0, "n_samples": 5, "avg_ded_per_sample": 16.2 }, "gemini_3_5_flash_stt": { "total_deductions": 54.0, "n_samples": 5, "avg_ded_per_sample": 10.8 }, "gemini_3_5_transcribe": { "total_deductions": 66.0, "n_samples": 5, "avg_ded_per_sample": 13.2 }, "gemini_3_6_flash_stt": { "total_deductions": 76.0, "n_samples": 5, "avg_ded_per_sample": 15.2 }, "gemini_3_7_flash_stt": { "total_deductions": 99.0, "n_samples": 5, "avg_ded_per_sample": 19.8 }, "gemini_3_8_flash_stt": { "total_deductions": 26.0, "n_samples": 5, "avg_ded_per_sample": 5.2 }, "gemini_flash_stt": { "total_deductions": 58.0, "n_samples": 5, "avg_ded_per_sample": 11.6 }, "gemini_pro_stt": { "total_deductions": 54.0, "n_samples": 5, "avg_ded_per_sample": 10.8 }, "gemini_stt": { "total_deductions": 66.0, "n_samples": 5, "avg_ded_per_sample": 13.2 }, "glm_asr_nano": { "total_deductions": 32.0, "n_samples": 5, "avg_ded_per_sample": 6.4 }, "gpt_transcribe": { "total_deductions": 71.0, "n_samples": 5, "avg_ded_per_sample": 14.2 }, "mai_transcribe_2": { "total_deductions": 32.0, "n_samples": 5, "avg_ded_per_sample": 6.4 }, "mimo": { "total_deductions": 40.0, "n_samples": 5, "avg_ded_per_sample": 8.0 }, "mimo_stt": { "total_deductions": 43.0, "n_samples": 5, "avg_ded_per_sample": 8.6 }, "moss_audio": { "total_deductions": 133.0, "n_samples": 5, "avg_ded_per_sample": 26.6 }, "qwen3_oss": { "total_deductions": 66.0, "n_samples": 5, "avg_ded_per_sample": 13.2 }, "qwen_audio_3_1_asr": { "total_deductions": 52.0, "n_samples": 5, "avg_ded_per_sample": 10.4 }, "r2t2": { "total_deductions": 40.0, "n_samples": 5, "avg_ded_per_sample": 8.0 }, "soniox": { "total_deductions": 23.0, "n_samples": 5, "avg_ded_per_sample": 4.6 }, "stepfun": { "total_deductions": 65.0, "n_samples": 5, "avg_ded_per_sample": 13.0 }, "vibevoice": { "total_deductions": 75.0, "n_samples": 5, "avg_ded_per_sample": 15.0 }, "voxtral": { "total_deductions": 75.0, "n_samples": 5, "avg_ded_per_sample": 15.0 }, "whisper": { "total_deductions": 56.0, "n_samples": 5, "avg_ded_per_sample": 11.2 }, "whisper_v3_turbo": { "total_deductions": 55.0, "n_samples": 5, "avg_ded_per_sample": 11.0 } }, "mic-far": { "apple_speechanalyzer": { "total_deductions": 124.0, "n_samples": 9, "avg_ded_per_sample": 13.78 }, "doubao": { "total_deductions": 50.0, "n_samples": 9, "avg_ded_per_sample": 5.56 }, "elevenlabs": { "total_deductions": 70.0, "n_samples": 9, "avg_ded_per_sample": 7.78 }, "firered": { "total_deductions": 159.0, "n_samples": 9, "avg_ded_per_sample": 17.67 }, "firered_aed": { "total_deductions": 57.0, "n_samples": 9, "avg_ded_per_sample": 6.33 }, "flash": { "total_deductions": 66.0, "n_samples": 9, "avg_ded_per_sample": 7.33 }, "funasr_nano": { "total_deductions": 87.0, "n_samples": 9, "avg_ded_per_sample": 9.67 }, "gemini_3_5_flash_lite_stt": { "total_deductions": 220.0, "n_samples": 9, "avg_ded_per_sample": 24.44 }, "gemini_3_5_flash_stt": { "total_deductions": 101.0, "n_samples": 9, "avg_ded_per_sample": 11.22 }, "gemini_3_5_transcribe": { "total_deductions": 75.0, "n_samples": 9, "avg_ded_per_sample": 8.33 }, "gemini_3_6_flash_stt": { "total_deductions": 128.0, "n_samples": 9, "avg_ded_per_sample": 14.22 }, "gemini_3_7_flash_stt": { "total_deductions": 59.0, "n_samples": 9, "avg_ded_per_sample": 6.56 }, "gemini_3_8_flash_stt": { "total_deductions": 48.0, "n_samples": 9, "avg_ded_per_sample": 5.33 }, "gemini_flash_stt": { "total_deductions": 65.0, "n_samples": 9, "avg_ded_per_sample": 7.22 }, "gemini_pro_stt": { "total_deductions": 51.0, "n_samples": 9, "avg_ded_per_sample": 5.67 }, "gemini_stt": { "total_deductions": 107.0, "n_samples": 9, "avg_ded_per_sample": 11.89 }, "glm_asr_nano": { "total_deductions": 118.0, "n_samples": 9, "avg_ded_per_sample": 13.11 }, "gpt_transcribe": { "total_deductions": 36.0, "n_samples": 9, "avg_ded_per_sample": 4.0 }, "mai_transcribe_2": { "total_deductions": 14.0, "n_samples": 9, "avg_ded_per_sample": 1.56 }, "mimo": { "total_deductions": 27.0, "n_samples": 9, "avg_ded_per_sample": 3.0 }, "mimo_stt": { "total_deductions": 99.0, "n_samples": 9, "avg_ded_per_sample": 11.0 }, "moss_audio": { "total_deductions": 137.0, "n_samples": 9, "avg_ded_per_sample": 15.22 }, "qwen3_oss": { "total_deductions": 81.0, "n_samples": 9, "avg_ded_per_sample": 9.0 }, "qwen_audio_3_1_asr": { "total_deductions": 75.0, "n_samples": 9, "avg_ded_per_sample": 8.33 }, "r2t2": { "total_deductions": 89.0, "n_samples": 9, "avg_ded_per_sample": 9.89 }, "soniox": { "total_deductions": 54.0, "n_samples": 9, "avg_ded_per_sample": 6.0 }, "stepfun": { "total_deductions": 194.0, "n_samples": 9, "avg_ded_per_sample": 21.56 }, "vibevoice": { "total_deductions": 40.0, "n_samples": 9, "avg_ded_per_sample": 4.44 }, "voxtral": { "total_deductions": 115.0, "n_samples": 9, "avg_ded_per_sample": 12.78 }, "whisper": { "total_deductions": 45.0, "n_samples": 9, "avg_ded_per_sample": 5.0 }, "whisper_v3_turbo": { "total_deductions": 69.0, "n_samples": 9, "avg_ded_per_sample": 7.67 } } }, "per_emotion_stats": { "angry": { "apple_speechanalyzer": { "total_deductions": 112.0, "n_samples": 5, "avg_ded_per_sample": 22.4 }, "doubao": { "total_deductions": 29.0, "n_samples": 5, "avg_ded_per_sample": 5.8 }, "elevenlabs": { "total_deductions": 59.0, "n_samples": 5, "avg_ded_per_sample": 11.8 }, "firered": { "total_deductions": 116.0, "n_samples": 5, "avg_ded_per_sample": 23.2 }, "firered_aed": { "total_deductions": 79.0, "n_samples": 5, "avg_ded_per_sample": 15.8 }, "flash": { "total_deductions": 28.0, "n_samples": 5, "avg_ded_per_sample": 5.6 }, "funasr_nano": { "total_deductions": 75.0, "n_samples": 5, "avg_ded_per_sample": 15.0 }, "gemini_3_5_flash_lite_stt": { "total_deductions": 120.0, "n_samples": 5, "avg_ded_per_sample": 24.0 }, "gemini_3_5_flash_stt": { "total_deductions": 29.0, "n_samples": 5, "avg_ded_per_sample": 5.8 }, "gemini_3_5_transcribe": { "total_deductions": 74.0, "n_samples": 5, "avg_ded_per_sample": 14.8 }, "gemini_3_6_flash_stt": { "total_deductions": 95.0, "n_samples": 5, "avg_ded_per_sample": 19.0 }, "gemini_3_7_flash_stt": { "total_deductions": 55.0, "n_samples": 5, "avg_ded_per_sample": 11.0 }, "gemini_3_8_flash_stt": { "total_deductions": 53.0, "n_samples": 5, "avg_ded_per_sample": 10.6 }, "gemini_flash_stt": { "total_deductions": 77.0, "n_samples": 5, "avg_ded_per_sample": 15.4 }, "gemini_pro_stt": { "total_deductions": 74.0, "n_samples": 5, "avg_ded_per_sample": 14.8 }, "gemini_stt": { "total_deductions": 111.0, "n_samples": 5, "avg_ded_per_sample": 22.2 }, "glm_asr_nano": { "total_deductions": 106.0, "n_samples": 5, "avg_ded_per_sample": 21.2 }, "gpt_transcribe": { "total_deductions": 57.0, "n_samples": 5, "avg_ded_per_sample": 11.4 }, "mai_transcribe_2": { "total_deductions": 20.0, "n_samples": 5, "avg_ded_per_sample": 4.0 }, "mimo": { "total_deductions": 66.0, "n_samples": 5, "avg_ded_per_sample": 13.2 }, "mimo_stt": { "total_deductions": 64.0, "n_samples": 5, "avg_ded_per_sample": 12.8 }, "moss_audio": { "total_deductions": 156.0, "n_samples": 5, "avg_ded_per_sample": 31.2 }, "qwen3_oss": { "total_deductions": 87.0, "n_samples": 5, "avg_ded_per_sample": 17.4 }, "qwen_audio_3_1_asr": { "total_deductions": 48.0, "n_samples": 5, "avg_ded_per_sample": 9.6 }, "r2t2": { "total_deductions": 55.0, "n_samples": 5, "avg_ded_per_sample": 11.0 }, "soniox": { "total_deductions": 26.0, "n_samples": 5, "avg_ded_per_sample": 5.2 }, "stepfun": { "total_deductions": 100.0, "n_samples": 5, "avg_ded_per_sample": 20.0 }, "vibevoice": { "total_deductions": 71.0, "n_samples": 5, "avg_ded_per_sample": 14.2 }, "voxtral": { "total_deductions": 107.0, "n_samples": 5, "avg_ded_per_sample": 21.4 }, "whisper": { "total_deductions": 39.0, "n_samples": 5, "avg_ded_per_sample": 7.8 }, "whisper_v3_turbo": { "total_deductions": 59.0, "n_samples": 5, "avg_ded_per_sample": 11.8 } }, "excited": { "apple_speechanalyzer": { "total_deductions": 56.0, "n_samples": 2, "avg_ded_per_sample": 28.0 }, "doubao": { "total_deductions": 9.0, "n_samples": 2, "avg_ded_per_sample": 4.5 }, "elevenlabs": { "total_deductions": 27.0, "n_samples": 2, "avg_ded_per_sample": 13.5 }, "firered": { "total_deductions": 75.0, "n_samples": 2, "avg_ded_per_sample": 37.5 }, "firered_aed": { "total_deductions": 16.0, "n_samples": 2, "avg_ded_per_sample": 8.0 }, "flash": { "total_deductions": 11.0, "n_samples": 2, "avg_ded_per_sample": 5.5 }, "funasr_nano": { "total_deductions": 25.0, "n_samples": 2, "avg_ded_per_sample": 12.5 }, "gemini_3_5_flash_lite_stt": { "total_deductions": 49.0, "n_samples": 2, "avg_ded_per_sample": 24.5 }, "gemini_3_5_flash_stt": { "total_deductions": 14.0, "n_samples": 2, "avg_ded_per_sample": 7.0 }, "gemini_3_5_transcribe": { "total_deductions": 12.0, "n_samples": 2, "avg_ded_per_sample": 6.0 }, "gemini_3_6_flash_stt": { "total_deductions": 23.0, "n_samples": 2, "avg_ded_per_sample": 11.5 }, "gemini_3_7_flash_stt": { "total_deductions": 19.0, "n_samples": 2, "avg_ded_per_sample": 9.5 }, "gemini_3_8_flash_stt": { "total_deductions": 11.0, "n_samples": 2, "avg_ded_per_sample": 5.5 }, "gemini_flash_stt": { "total_deductions": 24.0, "n_samples": 2, "avg_ded_per_sample": 12.0 }, "gemini_pro_stt": { "total_deductions": 0.0, "n_samples": 2, "avg_ded_per_sample": 0.0 }, "gemini_stt": { "total_deductions": 41.0, "n_samples": 2, "avg_ded_per_sample": 20.5 }, "glm_asr_nano": { "total_deductions": 17.0, "n_samples": 2, "avg_ded_per_sample": 8.5 }, "gpt_transcribe": { "total_deductions": 12.0, "n_samples": 2, "avg_ded_per_sample": 6.0 }, "mai_transcribe_2": { "total_deductions": 16.0, "n_samples": 2, "avg_ded_per_sample": 8.0 }, "mimo": { "total_deductions": 1.0, "n_samples": 2, "avg_ded_per_sample": 0.5 }, "mimo_stt": { "total_deductions": 18.0, "n_samples": 2, "avg_ded_per_sample": 9.0 }, "moss_audio": { "total_deductions": 33.0, "n_samples": 2, "avg_ded_per_sample": 16.5 }, "qwen3_oss": { "total_deductions": 13.0, "n_samples": 2, "avg_ded_per_sample": 6.5 }, "qwen_audio_3_1_asr": { "total_deductions": 12.0, "n_samples": 2, "avg_ded_per_sample": 6.0 }, "r2t2": { "total_deductions": 12.0, "n_samples": 2, "avg_ded_per_sample": 6.0 }, "soniox": { "total_deductions": 27.0, "n_samples": 2, "avg_ded_per_sample": 13.5 }, "stepfun": { "total_deductions": 41.0, "n_samples": 2, "avg_ded_per_sample": 20.5 }, "vibevoice": { "total_deductions": 9.0, "n_samples": 2, "avg_ded_per_sample": 4.5 }, "voxtral": { "total_deductions": 38.0, "n_samples": 2, "avg_ded_per_sample": 19.0 }, "whisper": { "total_deductions": 32.0, "n_samples": 2, "avg_ded_per_sample": 16.0 }, "whisper_v3_turbo": { "total_deductions": 64.0, "n_samples": 2, "avg_ded_per_sample": 32.0 } }, "frustrated": { "apple_speechanalyzer": { "total_deductions": 433.0, "n_samples": 27, "avg_ded_per_sample": 16.04 }, "doubao": { "total_deductions": 186.0, "n_samples": 27, "avg_ded_per_sample": 6.89 }, "elevenlabs": { "total_deductions": 154.0, "n_samples": 27, "avg_ded_per_sample": 5.7 }, "firered": { "total_deductions": 499.0, "n_samples": 27, "avg_ded_per_sample": 18.48 }, "firered_aed": { "total_deductions": 184.0, "n_samples": 27, "avg_ded_per_sample": 6.81 }, "flash": { "total_deductions": 118.0, "n_samples": 27, "avg_ded_per_sample": 4.37 }, "funasr_nano": { "total_deductions": 304.0, "n_samples": 27, "avg_ded_per_sample": 11.26 }, "gemini_3_5_flash_lite_stt": { "total_deductions": 496.0, "n_samples": 27, "avg_ded_per_sample": 18.37 }, "gemini_3_5_flash_stt": { "total_deductions": 205.0, "n_samples": 27, "avg_ded_per_sample": 7.59 }, "gemini_3_5_transcribe": { "total_deductions": 286.0, "n_samples": 27, "avg_ded_per_sample": 10.59 }, "gemini_3_6_flash_stt": { "total_deductions": 361.0, "n_samples": 27, "avg_ded_per_sample": 13.37 }, "gemini_3_7_flash_stt": { "total_deductions": 226.0, "n_samples": 27, "avg_ded_per_sample": 8.37 }, "gemini_3_8_flash_stt": { "total_deductions": 168.0, "n_samples": 27, "avg_ded_per_sample": 6.22 }, "gemini_flash_stt": { "total_deductions": 177.0, "n_samples": 27, "avg_ded_per_sample": 6.56 }, "gemini_pro_stt": { "total_deductions": 146.0, "n_samples": 27, "avg_ded_per_sample": 5.41 }, "gemini_stt": { "total_deductions": 391.0, "n_samples": 27, "avg_ded_per_sample": 14.48 }, "glm_asr_nano": { "total_deductions": 202.0, "n_samples": 27, "avg_ded_per_sample": 7.48 }, "gpt_transcribe": { "total_deductions": 141.0, "n_samples": 27, "avg_ded_per_sample": 5.22 }, "mai_transcribe_2": { "total_deductions": 48.0, "n_samples": 27, "avg_ded_per_sample": 1.78 }, "mimo": { "total_deductions": 110.0, "n_samples": 27, "avg_ded_per_sample": 4.07 }, "mimo_stt": { "total_deductions": 158.0, "n_samples": 27, "avg_ded_per_sample": 5.85 }, "moss_audio": { "total_deductions": 445.0, "n_samples": 27, "avg_ded_per_sample": 16.48 }, "qwen3_oss": { "total_deductions": 252.0, "n_samples": 27, "avg_ded_per_sample": 9.33 }, "qwen_audio_3_1_asr": { "total_deductions": 157.0, "n_samples": 27, "avg_ded_per_sample": 5.81 }, "r2t2": { "total_deductions": 167.0, "n_samples": 27, "avg_ded_per_sample": 6.19 }, "soniox": { "total_deductions": 142.0, "n_samples": 27, "avg_ded_per_sample": 5.26 }, "stepfun": { "total_deductions": 280.0, "n_samples": 27, "avg_ded_per_sample": 10.37 }, "vibevoice": { "total_deductions": 182.0, "n_samples": 27, "avg_ded_per_sample": 6.74 }, "voxtral": { "total_deductions": 446.0, "n_samples": 27, "avg_ded_per_sample": 16.52 }, "whisper": { "total_deductions": 239.0, "n_samples": 27, "avg_ded_per_sample": 8.85 }, "whisper_v3_turbo": { "total_deductions": 304.0, "n_samples": 27, "avg_ded_per_sample": 11.26 } }, "neutral": { "apple_speechanalyzer": { "total_deductions": 1727.0, "n_samples": 66, "avg_ded_per_sample": 26.17 }, "doubao": { "total_deductions": 588.0, "n_samples": 66, "avg_ded_per_sample": 8.91 }, "elevenlabs": { "total_deductions": 487.0, "n_samples": 66, "avg_ded_per_sample": 7.38 }, "firered": { "total_deductions": 2062.0, "n_samples": 66, "avg_ded_per_sample": 31.24 }, "firered_aed": { "total_deductions": 1111.0, "n_samples": 66, "avg_ded_per_sample": 16.83 }, "flash": { "total_deductions": 543.0, "n_samples": 66, "avg_ded_per_sample": 8.23 }, "funasr_nano": { "total_deductions": 1220.0, "n_samples": 66, "avg_ded_per_sample": 18.48 }, "gemini_3_5_flash_lite_stt": { "total_deductions": 1695.0, "n_samples": 66, "avg_ded_per_sample": 25.68 }, "gemini_3_5_flash_stt": { "total_deductions": 821.0, "n_samples": 66, "avg_ded_per_sample": 12.44 }, "gemini_3_5_transcribe": { "total_deductions": 790.0, "n_samples": 66, "avg_ded_per_sample": 11.97 }, "gemini_3_6_flash_stt": { "total_deductions": 883.0, "n_samples": 66, "avg_ded_per_sample": 13.38 }, "gemini_3_7_flash_stt": { "total_deductions": 639.0, "n_samples": 66, "avg_ded_per_sample": 9.68 }, "gemini_3_8_flash_stt": { "total_deductions": 469.0, "n_samples": 66, "avg_ded_per_sample": 7.11 }, "gemini_flash_stt": { "total_deductions": 783.0, "n_samples": 66, "avg_ded_per_sample": 11.86 }, "gemini_pro_stt": { "total_deductions": 552.0, "n_samples": 66, "avg_ded_per_sample": 8.36 }, "gemini_stt": { "total_deductions": 1093.0, "n_samples": 66, "avg_ded_per_sample": 16.56 }, "glm_asr_nano": { "total_deductions": 893.0, "n_samples": 66, "avg_ded_per_sample": 13.53 }, "gpt_transcribe": { "total_deductions": 653.0, "n_samples": 66, "avg_ded_per_sample": 9.89 }, "mai_transcribe_2": { "total_deductions": 350.0, "n_samples": 66, "avg_ded_per_sample": 5.3 }, "mimo": { "total_deductions": 387.0, "n_samples": 66, "avg_ded_per_sample": 5.86 }, "mimo_stt": { "total_deductions": 685.0, "n_samples": 66, "avg_ded_per_sample": 10.38 }, "moss_audio": { "total_deductions": 1683.0, "n_samples": 66, "avg_ded_per_sample": 25.5 }, "qwen3_oss": { "total_deductions": 728.0, "n_samples": 66, "avg_ded_per_sample": 11.03 }, "qwen_audio_3_1_asr": { "total_deductions": 491.0, "n_samples": 66, "avg_ded_per_sample": 7.44 }, "r2t2": { "total_deductions": 585.0, "n_samples": 66, "avg_ded_per_sample": 8.86 }, "soniox": { "total_deductions": 665.0, "n_samples": 66, "avg_ded_per_sample": 10.08 }, "stepfun": { "total_deductions": 876.0, "n_samples": 66, "avg_ded_per_sample": 13.27 }, "vibevoice": { "total_deductions": 625.0, "n_samples": 66, "avg_ded_per_sample": 9.47 }, "voxtral": { "total_deductions": 1412.0, "n_samples": 66, "avg_ded_per_sample": 21.39 }, "whisper": { "total_deductions": 751.0, "n_samples": 66, "avg_ded_per_sample": 11.38 }, "whisper_v3_turbo": { "total_deductions": 1414.0, "n_samples": 66, "avg_ded_per_sample": 21.42 } } }, "models": { "apple_speechanalyzer": { "scores_by_judge": { "deepseek_flash": [ 100, 92, 100, 100, 89, 90, 78, 92, 83, 98, 58, 77, 44, 73, 88, 58, 63, 43, 0, 81, 92, 90, 92, 90, 91, 91, 93, 90, 71, 90, 39, 98, 34, 24, 39, 60, 67, 3, 44, 12, 69, 78, 67, 50, 27, 85, 73, 61, 63, 63, 73, 90, 89, 85, 84, 70, 52, 82, 97, 51, 94, 100, 86, 97, 98, 83, 97, 89, 98, 89, 59, 86, 49, 89, 73, 78, 75, 98, 84, 95, 80, 83, 90, 60, 82, 73, 80, 92, 95, 93, 83, 100, 77, 93, 96, 100, 96, 94, 100, 93 ] }, "per_category_deductions": { "polarity": { "deepseek_flash": 210 }, "semantic-inversion": { "deepseek_flash": 0 }, "hallucination": { "deepseek_flash": 0 }, "numeric": { "deepseek_flash": 10 }, "brand": { "deepseek_flash": 370 }, "tech": { "deepseek_flash": 616 }, "culture": { "deepseek_flash": 256 }, "proper-noun": { "deepseek_flash": 96 }, "dialect": { "deepseek_flash": 40 }, "mishear-severe": { "deepseek_flash": 445 }, "content-drop": { "deepseek_flash": 0 }, "paraphrase": { "deepseek_flash": 15 }, "mishear-minor": { "deepseek_flash": 72 }, "filler": { "deepseek_flash": 197 }, "format": { "deepseek_flash": 1 }, "unscoreable": { "deepseek_flash": 0 } }, "per_tier_deductions": { "tier1": { "deepseek_flash": 220 }, "tier2": { "deepseek_flash": 1378 }, "tier3": { "deepseek_flash": 532 }, "tier4": { "deepseek_flash": 198 }, "unscoreable": { "deepseek_flash": 0 } }, "raw_deductions_by_judge": { "deepseek_flash": [ 0, 8, 0, 0, 11, 10, 22, 8, 17, 2, 42, 23, 56, 27, 12, 42, 37, 57, 131, 19, 8, 10, 8, 10, 9, 9, 7, 10, 29, 10, 61, 2, 66, 76, 61, 40, 33, 97, 56, 88, 31, 22, 33, 50, 73, 15, 27, 39, 37, 37, 27, 10, 11, 15, 16, 30, 48, 18, 3, 49, 6, 0, 14, 3, 2, 17, 3, 11, 2, 11, 41, 14, 51, 11, 27, 22, 25, 2, 16, 5, 20, 17, 10, 40, 18, 27, 20, 8, 5, 7, 17, 0, 23, 7, 4, 0, 4, 6, 0, 7 ] }, "sample_details": [ { "idx": 1, "duration_s": 5, "mean_score": 100.0, "mean_raw": 0.0, "scores_per_judge": [ 100 ], "deductions_per_judge": { "deepseek_flash": [] }, "judge_disagreement": 0 }, { "idx": 2, "duration_s": 7, "mean_score": 92.0, "mean_raw": 8.0, "scores_per_judge": [ 92 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "tech", "points": 8 } ] }, "judge_disagreement": 0 }, { "idx": 3, "duration_s": 8, "mean_score": 100.0, "mean_raw": 0.0, "scores_per_judge": [ 100 ], "deductions_per_judge": { "deepseek_flash": [] }, "judge_disagreement": 0 }, { "idx": 4, "duration_s": 11, "mean_score": 100.0, "mean_raw": 0.0, "scores_per_judge": [ 100 ], "deductions_per_judge": { "deepseek_flash": [] }, "judge_disagreement": 0 }, { "idx": 5, "duration_s": 16, "mean_score": 89.0, "mean_raw": 11.0, "scores_per_judge": [ 89 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "culture", "points": 8 } ] }, "judge_disagreement": 0 }, { "idx": 6, "duration_s": 17, "mean_score": 90.0, "mean_raw": 10.0, "scores_per_judge": [ 90 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "tech", "points": 8 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 7, "duration_s": 20, "mean_score": 78.0, "mean_raw": 22.0, "scores_per_judge": [ 78 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "brand", "points": 10 }, { "category": "tech", "points": 8 }, { "category": "mishear-minor", "points": 2 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 8, "duration_s": 30, "mean_score": 92.0, "mean_raw": 8.0, "scores_per_judge": [ 92 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "tech", "points": 8 } ] }, "judge_disagreement": 0 }, { "idx": 9, "duration_s": 32, "mean_score": 83.0, "mean_raw": 17.0, "scores_per_judge": [ 83 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-severe", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 10, "duration_s": 35, "mean_score": 98.0, "mean_raw": 2.0, "scores_per_judge": [ 98 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 11, "duration_s": 40, "mean_score": 58.0, "mean_raw": 42.0, "scores_per_judge": [ 58 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "brand", "points": 10 }, { "category": "tech", "points": 8 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-minor", "points": 2 }, { "category": "mishear-minor", "points": 2 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 12, "duration_s": 46, "mean_score": 77.0, "mean_raw": 23.0, "scores_per_judge": [ 77 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "brand", "points": 10 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-minor", "points": 2 }, { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 13, "duration_s": 60, "mean_score": 44.0, "mean_raw": 56.0, "scores_per_judge": [ 44 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "brand", "points": 10 }, { "category": "brand", "points": 10 }, { "category": "brand", "points": 10 }, { "category": "tech", "points": 8 }, { "category": "tech", "points": 8 }, { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 14, "duration_s": 64, "mean_score": 73.0, "mean_raw": 27.0, "scores_per_judge": [ 73 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 }, { "category": "mishear-severe", "points": 5 }, { "category": "proper-noun", "points": 6 }, { "category": "proper-noun", "points": 6 }, { "category": "mishear-minor", "points": 2 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 15, "duration_s": 69, "mean_score": 88.0, "mean_raw": 12.0, "scores_per_judge": [ 88 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "tech", "points": 8 }, { "category": "mishear-minor", "points": 2 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 16, "duration_s": 81, "mean_score": 58.0, "mean_raw": 42.0, "scores_per_judge": [ 58 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "mishear-severe", "points": 5 }, { "category": "proper-noun", "points": 6 }, { "category": "tech", "points": 8 }, { "category": "proper-noun", "points": 6 }, { "category": "tech", "points": 8 }, { "category": "tech", "points": 8 } ] }, "judge_disagreement": 0 }, { "idx": 17, "duration_s": 100, "mean_score": 63.0, "mean_raw": 37.0, "scores_per_judge": [ 63 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "brand", "points": 10 }, { "category": "tech", "points": 8 }, { "category": "tech", "points": 8 }, { "category": "tech", "points": 8 }, { "category": "filler", "points": 3 } ] }, "judge_disagreement": 0 }, { "idx": 18, "duration_s": 125, "mean_score": 43.0, "mean_raw": 57.0, "scores_per_judge": [ 43 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "brand", "points": 10 }, { "category": "brand", "points": 10 }, { "category": "tech", "points": 8 }, { "category": "tech", "points": 8 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 19, "duration_s": 133, "mean_score": 0.0, "mean_raw": 131.0, "scores_per_judge": [ 0 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "tech", "points": 8 }, { "category": "brand", "points": 10 }, { "category": "brand", "points": 10 }, { "category": "tech", "points": 8 }, { "category": "brand", "points": 10 }, { "category": "brand", "points": 10 }, { "category": "brand", "points": 10 }, { "category": "brand", "points": 10 }, { "category": "tech", "points": 8 }, { "category": "brand", "points": 10 }, { "category": "brand", "points": 10 }, { "category": "tech", "points": 8 }, { "category": "brand", "points": 10 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-minor", "points": 2 }, { "category": "filler", "points": 2 } ] }, "judge_disagreement": 0 }, { "idx": 20, "duration_s": 164, "mean_score": 81.0, "mean_raw": 19.0, "scores_per_judge": [ 81 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "culture", "points": 8 }, { "category": "proper-noun", "points": 6 }, { "category": "filler", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 21, "duration_s": 5, "mean_score": 92.0, "mean_raw": 8.0, "scores_per_judge": [ 92 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "tech", "points": 8 } ] }, "judge_disagreement": 0 }, { "idx": 22, "duration_s": 7, "mean_score": 90.0, "mean_raw": 10.0, "scores_per_judge": [ 90 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "brand", "points": 10 } ] }, "judge_disagreement": 0 }, { "idx": 23, "duration_s": 10, "mean_score": 92.0, "mean_raw": 8.0, "scores_per_judge": [ 92 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "culture", "points": 8 } ] }, "judge_disagreement": 0 }, { "idx": 24, "duration_s": 14, "mean_score": 90.0, "mean_raw": 10.0, "scores_per_judge": [ 90 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "tech", "points": 8 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 25, "duration_s": 16, "mean_score": 91.0, "mean_raw": 9.0, "scores_per_judge": [ 91 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "tech", "points": 8 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 26, "duration_s": 23, "mean_score": 91.0, "mean_raw": 9.0, "scores_per_judge": [ 91 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 }, { "category": "mishear-minor", "points": 2 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 27, "duration_s": 25, "mean_score": 93.0, "mean_raw": 7.0, "scores_per_judge": [ 93 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 }, { "category": "mishear-minor", "points": 2 } ] }, "judge_disagreement": 0 }, { "idx": 28, "duration_s": 30, "mean_score": 90.0, "mean_raw": 10.0, "scores_per_judge": [ 90 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "mishear-minor", "points": 2 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 29, "duration_s": 34, "mean_score": 71.0, "mean_raw": 29.0, "scores_per_judge": [ 71 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "brand", "points": 10 }, { "category": "brand", "points": 10 }, { "category": "tech", "points": 8 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 30, "duration_s": 38, "mean_score": 90.0, "mean_raw": 10.0, "scores_per_judge": [ 90 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "tech", "points": 8 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 31, "duration_s": 44, "mean_score": 39.0, "mean_raw": 61.0, "scores_per_judge": [ 39 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "polarity", "points": 35 }, { "category": "brand", "points": 10 }, { "category": "tech", "points": 8 }, { "category": "mishear-minor", "points": 2 }, { "category": "mishear-minor", "points": 2 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 32, "duration_s": 50, "mean_score": 98.0, "mean_raw": 2.0, "scores_per_judge": [ 98 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 33, "duration_s": 62, "mean_score": 34.0, "mean_raw": 66.0, "scores_per_judge": [ 34 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "brand", "points": 10 }, { "category": "brand", "points": 10 }, { "category": "tech", "points": 8 }, { "category": "tech", "points": 8 }, { "category": "tech", "points": 8 }, { "category": "tech", "points": 8 }, { "category": "proper-noun", "points": 6 }, { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 3 } ] }, "judge_disagreement": 0 }, { "idx": 34, "duration_s": 66, "mean_score": 24.0, "mean_raw": 76.0, "scores_per_judge": [ 24 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "polarity", "points": 35 }, { "category": "tech", "points": 8 }, { "category": "tech", "points": 8 }, { "category": "tech", "points": 8 }, { "category": "tech", "points": 8 }, { "category": "culture", "points": 8 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 35, "duration_s": 77, "mean_score": 39.0, "mean_raw": 61.0, "scores_per_judge": [ 39 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "tech", "points": 8 }, { "category": "filler", "points": 1 }, { "category": "tech", "points": 8 }, { "category": "tech", "points": 8 }, { "category": "filler", "points": 1 }, { "category": "tech", "points": 8 }, { "category": "tech", "points": 8 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "tech", "points": 8 }, { "category": "tech", "points": 8 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 36, "duration_s": 94, "mean_score": 60.0, "mean_raw": 40.0, "scores_per_judge": [ 60 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "polarity", "points": 35 }, { "category": "mishear-severe", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 37, "duration_s": 106, "mean_score": 67.0, "mean_raw": 33.0, "scores_per_judge": [ 67 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "tech", "points": 8 }, { "category": "tech", "points": 8 }, { "category": "tech", "points": 8 }, { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 4 } ] }, "judge_disagreement": 0 }, { "idx": 38, "duration_s": 126, "mean_score": 3.0, "mean_raw": 97.0, "scores_per_judge": [ 3 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "culture", "points": 8 }, { "category": "culture", "points": 8 }, { "category": "culture", "points": 8 }, { "category": "culture", "points": 8 }, { "category": "tech", "points": 8 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-minor", "points": 2 }, { "category": "mishear-minor", "points": 2 }, { "category": "mishear-minor", "points": 2 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 39, "duration_s": 158, "mean_score": 44.0, "mean_raw": 56.0, "scores_per_judge": [ 44 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "tech", "points": 8 }, { "category": "tech", "points": 8 }, { "category": "tech", "points": 8 }, { "category": "brand", "points": 10 }, { "category": "proper-noun", "points": 6 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 40, "duration_s": 279, "mean_score": 12.0, "mean_raw": 88.0, "scores_per_judge": [ 12 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 }, { "category": "mishear-severe", "points": 5 }, { "category": "culture", "points": 8 }, { "category": "culture", "points": 8 }, { "category": "paraphrase", "points": 3 }, { "category": "mishear-severe", "points": 5 }, { "category": "culture", "points": 8 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-severe", "points": 5 }, { "category": "tech", "points": 8 }, { "category": "mishear-severe", "points": 5 }, { "category": "culture", "points": 8 }, { "category": "dialect", "points": 5 }, { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 41, "duration_s": 34, "mean_score": 69.0, "mean_raw": 31.0, "scores_per_judge": [ 69 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "culture", "points": 8 }, { "category": "culture", "points": 8 }, { "category": "filler", "points": 1 }, { "category": "culture", "points": 8 }, { "category": "mishear-severe", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 42, "duration_s": 43, "mean_score": 78.0, "mean_raw": 22.0, "scores_per_judge": [ 78 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "dialect", "points": 5 }, { "category": "culture", "points": 8 }, { "category": "culture", "points": 8 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 43, "duration_s": 29, "mean_score": 67.0, "mean_raw": 33.0, "scores_per_judge": [ 67 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "culture", "points": 8 }, { "category": "culture", "points": 8 }, { "category": "culture", "points": 8 }, { "category": "proper-noun", "points": 6 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 44, "duration_s": 71, "mean_score": 50.0, "mean_raw": 50.0, "scores_per_judge": [ 50 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "proper-noun", "points": 6 }, { "category": "proper-noun", "points": 6 }, { "category": "culture", "points": 8 }, { "category": "culture", "points": 8 }, { "category": "culture", "points": 8 }, { "category": "culture", "points": 8 }, { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 45, "duration_s": 46, "mean_score": 27.0, "mean_raw": 73.0, "scores_per_judge": [ 27 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "polarity", "points": 35 }, { "category": "culture", "points": 8 }, { "category": "dialect", "points": 5 }, { "category": "dialect", "points": 5 }, { "category": "dialect", "points": 5 }, { "category": "dialect", "points": 5 }, { "category": "dialect", "points": 5 }, { "category": "dialect", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 46, "duration_s": 62, "mean_score": 85.0, "mean_raw": 15.0, "scores_per_judge": [ 85 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 }, { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 47, "duration_s": 67, "mean_score": 73.0, "mean_raw": 27.0, "scores_per_judge": [ 73 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "brand", "points": 10 }, { "category": "tech", "points": 8 }, { "category": "tech", "points": 8 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 48, "duration_s": 72, "mean_score": 61.0, "mean_raw": 39.0, "scores_per_judge": [ 61 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "brand", "points": 10 }, { "category": "brand", "points": 10 }, { "category": "tech", "points": 8 }, { "category": "tech", "points": 8 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 49, "duration_s": 87, "mean_score": 63.0, "mean_raw": 37.0, "scores_per_judge": [ 63 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "tech", "points": 8 }, { "category": "tech", "points": 8 }, { "category": "tech", "points": 8 }, { "category": "proper-noun", "points": 6 }, { "category": "mishear-minor", "points": 2 }, { "category": "filler", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 50, "duration_s": 106, "mean_score": 63.0, "mean_raw": 37.0, "scores_per_judge": [ 63 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "culture", "points": 8 }, { "category": "culture", "points": 8 }, { "category": "culture", "points": 8 }, { "category": "tech", "points": 8 }, { "category": "mishear-severe", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 51, "duration_s": 33, "mean_score": 73.0, "mean_raw": 27.0, "scores_per_judge": [ 73 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "culture", "points": 8 }, { "category": "proper-noun", "points": 6 }, { "category": "mishear-minor", "points": 2 }, { "category": "filler", "points": 1 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-severe", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 52, "duration_s": 25, "mean_score": 90.0, "mean_raw": 10.0, "scores_per_judge": [ 90 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 }, { "category": "mishear-minor", "points": 2 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 53, "duration_s": 35, "mean_score": 89.0, "mean_raw": 11.0, "scores_per_judge": [ 89 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "proper-noun", "points": 6 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 54, "duration_s": 36, "mean_score": 85.0, "mean_raw": 15.0, "scores_per_judge": [ 85 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "tech", "points": 8 }, { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 55, "duration_s": 33, "mean_score": 84.0, "mean_raw": 16.0, "scores_per_judge": [ 84 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "proper-noun", "points": 6 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 56, "duration_s": 15, "mean_score": 70.0, "mean_raw": 30.0, "scores_per_judge": [ 70 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "brand", "points": 10 }, { "category": "brand", "points": 10 }, { "category": "brand", "points": 10 } ] }, "judge_disagreement": 0 }, { "idx": 57, "duration_s": 38, "mean_score": 52.0, "mean_raw": 48.0, "scores_per_judge": [ 52 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-minor", "points": 2 }, { "category": "tech", "points": 8 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-minor", "points": 2 }, { "category": "mishear-severe", "points": 5 }, { "category": "tech", "points": 8 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 58, "duration_s": 15, "mean_score": 82.0, "mean_raw": 18.0, "scores_per_judge": [ 82 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "brand", "points": 10 }, { "category": "tech", "points": 8 } ] }, "judge_disagreement": 0 }, { "idx": 59, "duration_s": 28, "mean_score": 97.0, "mean_raw": 3.0, "scores_per_judge": [ 97 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-minor", "points": 2 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 60, "duration_s": 18, "mean_score": 51.0, "mean_raw": 49.0, "scores_per_judge": [ 51 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "polarity", "points": 35 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-minor", "points": 2 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 61, "duration_s": 19, "mean_score": 94.0, "mean_raw": 6.0, "scores_per_judge": [ 94 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 62, "duration_s": 17, "mean_score": 100.0, "mean_raw": 0.0, "scores_per_judge": [ 100 ], "deductions_per_judge": { "deepseek_flash": [] }, "judge_disagreement": 0 }, { "idx": 63, "duration_s": 21, "mean_score": 86.0, "mean_raw": 14.0, "scores_per_judge": [ 86 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "paraphrase", "points": 3 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 64, "duration_s": 23, "mean_score": 97.0, "mean_raw": 3.0, "scores_per_judge": [ 97 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 3 } ] }, "judge_disagreement": 0 }, { "idx": 65, "duration_s": 20, "mean_score": 98.0, "mean_raw": 2.0, "scores_per_judge": [ 98 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-minor", "points": 2 } ] }, "judge_disagreement": 0 }, { "idx": 66, "duration_s": 24, "mean_score": 83.0, "mean_raw": 17.0, "scores_per_judge": [ 83 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "tech", "points": 8 }, { "category": "culture", "points": 8 } ] }, "judge_disagreement": 0 }, { "idx": 67, "duration_s": 16, "mean_score": 97.0, "mean_raw": 3.0, "scores_per_judge": [ 97 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-minor", "points": 2 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 68, "duration_s": 18, "mean_score": 89.0, "mean_raw": 11.0, "scores_per_judge": [ 89 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "brand", "points": 10 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 69, "duration_s": 18, "mean_score": 98.0, "mean_raw": 2.0, "scores_per_judge": [ 98 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "format", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 70, "duration_s": 17, "mean_score": 89.0, "mean_raw": 11.0, "scores_per_judge": [ 89 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "mishear-minor", "points": 2 }, { "category": "tech", "points": 8 } ] }, "judge_disagreement": 0 }, { "idx": 71, "duration_s": 51, "mean_score": 59.0, "mean_raw": 41.0, "scores_per_judge": [ 59 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "tech", "points": 8 }, { "category": "brand", "points": 10 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 3 } ] }, "judge_disagreement": 0 }, { "idx": 72, "duration_s": 54, "mean_score": 86.0, "mean_raw": 14.0, "scores_per_judge": [ 86 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "tech", "points": 8 }, { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 73, "duration_s": 56, "mean_score": 49.0, "mean_raw": 51.0, "scores_per_judge": [ 49 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "brand", "points": 10 }, { "category": "brand", "points": 10 }, { "category": "tech", "points": 8 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-minor", "points": 2 }, { "category": "mishear-minor", "points": 2 }, { "category": "paraphrase", "points": 3 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 74, "duration_s": 45, "mean_score": 89.0, "mean_raw": 11.0, "scores_per_judge": [ 89 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "tech", "points": 8 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 75, "duration_s": 74, "mean_score": 73.0, "mean_raw": 27.0, "scores_per_judge": [ 73 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "tech", "points": 8 }, { "category": "tech", "points": 8 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-minor", "points": 2 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 76, "duration_s": 58, "mean_score": 78.0, "mean_raw": 22.0, "scores_per_judge": [ 78 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "proper-noun", "points": 6 }, { "category": "culture", "points": 8 }, { "category": "culture", "points": 8 } ] }, "judge_disagreement": 0 }, { "idx": 77, "duration_s": 130, "mean_score": 75.0, "mean_raw": 25.0, "scores_per_judge": [ 75 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "brand", "points": 10 }, { "category": "tech", "points": 8 }, { "category": "mishear-minor", "points": 2 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 2 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 78, "duration_s": 61, "mean_score": 98.0, "mean_raw": 2.0, "scores_per_judge": [ 98 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 2 } ] }, "judge_disagreement": 0 }, { "idx": 79, "duration_s": 47, "mean_score": 84.0, "mean_raw": 16.0, "scores_per_judge": [ 84 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "tech", "points": 8 }, { "category": "paraphrase", "points": 3 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 80, "duration_s": 42, "mean_score": 95.0, "mean_raw": 5.0, "scores_per_judge": [ 95 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-minor", "points": 2 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 81, "duration_s": 38, "mean_score": 80.0, "mean_raw": 20.0, "scores_per_judge": [ 80 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "proper-noun", "points": 6 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-minor", "points": 2 }, { "category": "paraphrase", "points": 3 }, { "category": "filler", "points": 4 } ] }, "judge_disagreement": 0 }, { "idx": 82, "duration_s": 38, "mean_score": 83.0, "mean_raw": 17.0, "scores_per_judge": [ 83 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 }, { "category": "tech", "points": 8 }, { "category": "mishear-minor", "points": 2 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 83, "duration_s": 37, "mean_score": 90.0, "mean_raw": 10.0, "scores_per_judge": [ 90 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 }, { "category": "mishear-minor", "points": 2 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 84, "duration_s": 36, "mean_score": 60.0, "mean_raw": 40.0, "scores_per_judge": [ 60 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "polarity", "points": 35 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 85, "duration_s": 36, "mean_score": 82.0, "mean_raw": 18.0, "scores_per_judge": [ 82 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-minor", "points": 2 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 86, "duration_s": 35, "mean_score": 73.0, "mean_raw": 27.0, "scores_per_judge": [ 73 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "tech", "points": 8 }, { "category": "tech", "points": 8 }, { "category": "tech", "points": 8 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 87, "duration_s": 34, "mean_score": 80.0, "mean_raw": 20.0, "scores_per_judge": [ 80 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "mishear-minor", "points": 2 }, { "category": "numeric", "points": 10 }, { "category": "filler", "points": 1 }, { "category": "mishear-severe", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 88, "duration_s": 34, "mean_score": 92.0, "mean_raw": 8.0, "scores_per_judge": [ 92 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "tech", "points": 8 } ] }, "judge_disagreement": 0 }, { "idx": 89, "duration_s": 34, "mean_score": 95.0, "mean_raw": 5.0, "scores_per_judge": [ 95 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 90, "duration_s": 33, "mean_score": 93.0, "mean_raw": 7.0, "scores_per_judge": [ 93 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 2 } ] }, "judge_disagreement": 0 }, { "idx": 91, "duration_s": 33, "mean_score": 83.0, "mean_raw": 17.0, "scores_per_judge": [ 83 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 92, "duration_s": 32, "mean_score": 100.0, "mean_raw": 0.0, "scores_per_judge": [ 100 ], "deductions_per_judge": { "deepseek_flash": [] }, "judge_disagreement": 0 }, { "idx": 93, "duration_s": 32, "mean_score": 77.0, "mean_raw": 23.0, "scores_per_judge": [ 77 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "tech", "points": 8 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-minor", "points": 2 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 94, "duration_s": 32, "mean_score": 93.0, "mean_raw": 7.0, "scores_per_judge": [ 93 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 95, "duration_s": 32, "mean_score": 96.0, "mean_raw": 4.0, "scores_per_judge": [ 96 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 96, "duration_s": 31, "mean_score": 100.0, "mean_raw": 0.0, "scores_per_judge": [ 100 ], "deductions_per_judge": { "deepseek_flash": [] }, "judge_disagreement": 0 }, { "idx": 97, "duration_s": 30, "mean_score": 96.0, "mean_raw": 4.0, "scores_per_judge": [ 96 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 98, "duration_s": 30, "mean_score": 94.0, "mean_raw": 6.0, "scores_per_judge": [ 94 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 99, "duration_s": 30, "mean_score": 100.0, "mean_raw": 0.0, "scores_per_judge": [ 100 ], "deductions_per_judge": { "deepseek_flash": [] }, "judge_disagreement": 0 }, { "idx": 100, "duration_s": 30, "mean_score": 93.0, "mean_raw": 7.0, "scores_per_judge": [ 93 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 } ], "total_deductions": 2328, "avg_deductions_per_sample": 23.28, "mean_clamped_score": 77.0, "n_judge_sample_pairs": 100, "gt_chars": 22625, "ded_per_100char": 10.29, "fidelity_score": 48.6, "overall_score": 77.0, "overall_raw_deductions": 23.28, "score_per_judge": { "deepseek_flash": 77.0 }, "per_group_deductions": { "meaning-flip": { "deepseek_flash": 220 }, "entity": { "deepseek_flash": 1082 }, "cultural": { "deepseek_flash": 296 }, "fidelity": { "deepseek_flash": 730 } }, "per_bucket": { "general": { "n_samples": 40, "mean_raw": 30.4, "mean_score": 70.4, "total_raw": 1214.0, "idxs": [ 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40 ], "worst_idx": 19 }, "idiom-test": { "n_samples": 5, "mean_raw": 41.8, "mean_score": 58.2, "total_raw": 209.0, "idxs": [ 41, 42, 43, 44, 45 ], "worst_idx": 45 }, "long-mixed": { "n_samples": 5, "mean_raw": 31.0, "mean_score": 69.0, "total_raw": 155.0, "idxs": [ 46, 47, 48, 49, 50 ], "worst_idx": 48 } }, "per_group_per_100char": { "meaning-flip": 0.972, "entity": 4.782, "cultural": 1.308, "fidelity": 3.227 }, "per_group_fidelity": { "meaning-flip": 76.7, "entity": 15.2, "cultural": 20.9, "fidelity": 62.2 } }, "doubao": { "scores_by_judge": { "deepseek_flash": [ 100, 100, 100, 100, 97, 84, 99, 100, 98, 99, 79, 98, 80, 88, 100, 77, 71, 81, 84, 96, 100, 90, 100, 90, 92, 99, 100, 39, 80, 64, 89, 100, 81, 98, 82, 97, 99, 86, 99, 90, 100, 80, 99, 92, 98, 95, 90, 79, 89, 92, 99, 97, 97, 99, 90, 79, 92, 98, 100, 100, 85, 99, 99, 96, 100, 84, 100, 90, 100, 90, 98, 94, 93, 95, 87, 78, 60, 100, 94, 97, 96, 81, 98, 65, 96, 84, 81, 99, 100, 93, 99, 99, 96, 98, 99, 100, 100, 99, 100, 95 ] }, "per_category_deductions": { "polarity": { "deepseek_flash": 105 }, "semantic-inversion": { "deepseek_flash": 80 }, "hallucination": { "deepseek_flash": 0 }, "numeric": { "deepseek_flash": 30 }, "brand": { "deepseek_flash": 150 }, "tech": { "deepseek_flash": 128 }, "culture": { "deepseek_flash": 24 }, "proper-noun": { "deepseek_flash": 12 }, "dialect": { "deepseek_flash": 0 }, "mishear-severe": { "deepseek_flash": 110 }, "content-drop": { "deepseek_flash": 6 }, "paraphrase": { "deepseek_flash": 9 }, "mishear-minor": { "deepseek_flash": 8 }, "filler": { "deepseek_flash": 148 }, "format": { "deepseek_flash": 2 }, "unscoreable": { "deepseek_flash": 0 } }, "per_tier_deductions": { "tier1": { "deepseek_flash": 215 }, "tier2": { "deepseek_flash": 314 }, "tier3": { "deepseek_flash": 133 }, "tier4": { "deepseek_flash": 150 }, "unscoreable": { "deepseek_flash": 0 } }, "raw_deductions_by_judge": { "deepseek_flash": [ 0, 0, 0, 0, 3, 16, 1, 0, 2, 1, 21, 2, 20, 12, 0, 23, 29, 19, 16, 4, 0, 10, 0, 10, 8, 1, 0, 61, 20, 36, 11, 0, 19, 2, 18, 3, 1, 14, 1, 10, 0, 20, 1, 8, 2, 5, 10, 21, 11, 8, 1, 3, 3, 1, 10, 21, 8, 2, 0, 0, 15, 1, 1, 4, 0, 16, 0, 10, 0, 10, 2, 6, 7, 5, 13, 22, 40, 0, 6, 3, 4, 19, 2, 35, 4, 16, 19, 1, 0, 7, 1, 1, 4, 2, 1, 0, 0, 1, 0, 5 ] }, "sample_details": [ { "idx": 1, "duration_s": 5, "mean_score": 100.0, "mean_raw": 0.0, "scores_per_judge": [ 100 ], "deductions_per_judge": { "deepseek_flash": [] }, "judge_disagreement": 0 }, { "idx": 2, "duration_s": 7, "mean_score": 100.0, "mean_raw": 0.0, "scores_per_judge": [ 100 ], "deductions_per_judge": { "deepseek_flash": [] }, "judge_disagreement": 0 }, { "idx": 3, "duration_s": 8, "mean_score": 100.0, "mean_raw": 0.0, "scores_per_judge": [ 100 ], "deductions_per_judge": { "deepseek_flash": [] }, "judge_disagreement": 0 }, { "idx": 4, "duration_s": 11, "mean_score": 100.0, "mean_raw": 0.0, "scores_per_judge": [ 100 ], "deductions_per_judge": { "deepseek_flash": [] }, "judge_disagreement": 0 }, { "idx": 5, "duration_s": 16, "mean_score": 97.0, "mean_raw": 3.0, "scores_per_judge": [ 97 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 6, "duration_s": 17, "mean_score": 84.0, "mean_raw": 16.0, "scores_per_judge": [ 84 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "tech", "points": 8 }, { "category": "mishear-severe", "points": 5 }, { "category": "paraphrase", "points": 3 } ] }, "judge_disagreement": 0 }, { "idx": 7, "duration_s": 20, "mean_score": 99.0, "mean_raw": 1.0, "scores_per_judge": [ 99 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 8, "duration_s": 30, "mean_score": 100.0, "mean_raw": 0.0, "scores_per_judge": [ 100 ], "deductions_per_judge": { "deepseek_flash": [] }, "judge_disagreement": 0 }, { "idx": 9, "duration_s": 32, "mean_score": 98.0, "mean_raw": 2.0, "scores_per_judge": [ 98 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 10, "duration_s": 35, "mean_score": 99.0, "mean_raw": 1.0, "scores_per_judge": [ 99 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 11, "duration_s": 40, "mean_score": 79.0, "mean_raw": 21.0, "scores_per_judge": [ 79 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "tech", "points": 8 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 3 } ] }, "judge_disagreement": 0 }, { "idx": 12, "duration_s": 46, "mean_score": 98.0, "mean_raw": 2.0, "scores_per_judge": [ 98 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 13, "duration_s": 60, "mean_score": 80.0, "mean_raw": 20.0, "scores_per_judge": [ 80 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "brand", "points": 10 }, { "category": "brand", "points": 10 } ] }, "judge_disagreement": 0 }, { "idx": 14, "duration_s": 64, "mean_score": 88.0, "mean_raw": 12.0, "scores_per_judge": [ 88 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 }, { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 15, "duration_s": 69, "mean_score": 100.0, "mean_raw": 0.0, "scores_per_judge": [ 100 ], "deductions_per_judge": { "deepseek_flash": [] }, "judge_disagreement": 0 }, { "idx": 16, "duration_s": 81, "mean_score": 77.0, "mean_raw": 23.0, "scores_per_judge": [ 77 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "proper-noun", "points": 6 }, { "category": "mishear-severe", "points": 5 }, { "category": "numeric", "points": 10 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 17, "duration_s": 100, "mean_score": 71.0, "mean_raw": 29.0, "scores_per_judge": [ 71 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "brand", "points": 10 }, { "category": "tech", "points": 8 }, { "category": "tech", "points": 8 }, { "category": "filler", "points": 3 } ] }, "judge_disagreement": 0 }, { "idx": 18, "duration_s": 125, "mean_score": 81.0, "mean_raw": 19.0, "scores_per_judge": [ 81 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "brand", "points": 10 }, { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 19, "duration_s": 133, "mean_score": 84.0, "mean_raw": 16.0, "scores_per_judge": [ 84 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 }, { "category": "mishear-severe", "points": 5 }, { "category": "content-drop", "points": 3 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 20, "duration_s": 164, "mean_score": 96.0, "mean_raw": 4.0, "scores_per_judge": [ 96 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "format", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 21, "duration_s": 5, "mean_score": 100.0, "mean_raw": 0.0, "scores_per_judge": [ 100 ], "deductions_per_judge": { "deepseek_flash": [] }, "judge_disagreement": 0 }, { "idx": 22, "duration_s": 7, "mean_score": 90.0, "mean_raw": 10.0, "scores_per_judge": [ 90 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "brand", "points": 10 } ] }, "judge_disagreement": 0 }, { "idx": 23, "duration_s": 10, "mean_score": 100.0, "mean_raw": 0.0, "scores_per_judge": [ 100 ], "deductions_per_judge": { "deepseek_flash": [] }, "judge_disagreement": 0 }, { "idx": 24, "duration_s": 14, "mean_score": 90.0, "mean_raw": 10.0, "scores_per_judge": [ 90 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "tech", "points": 8 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 25, "duration_s": 16, "mean_score": 92.0, "mean_raw": 8.0, "scores_per_judge": [ 92 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "tech", "points": 8 } ] }, "judge_disagreement": 0 }, { "idx": 26, "duration_s": 23, "mean_score": 99.0, "mean_raw": 1.0, "scores_per_judge": [ 99 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 27, "duration_s": 25, "mean_score": 100.0, "mean_raw": 0.0, "scores_per_judge": [ 100 ], "deductions_per_judge": { "deepseek_flash": [] }, "judge_disagreement": 0 }, { "idx": 28, "duration_s": 30, "mean_score": 39.0, "mean_raw": 61.0, "scores_per_judge": [ 39 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "polarity", "points": 35 }, { "category": "semantic-inversion", "points": 20 }, { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 29, "duration_s": 34, "mean_score": 80.0, "mean_raw": 20.0, "scores_per_judge": [ 80 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "brand", "points": 10 }, { "category": "brand", "points": 10 } ] }, "judge_disagreement": 0 }, { "idx": 30, "duration_s": 38, "mean_score": 64.0, "mean_raw": 36.0, "scores_per_judge": [ 64 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "polarity", "points": 35 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 31, "duration_s": 44, "mean_score": 89.0, "mean_raw": 11.0, "scores_per_judge": [ 89 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "tech", "points": 8 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 32, "duration_s": 50, "mean_score": 100.0, "mean_raw": 0.0, "scores_per_judge": [ 100 ], "deductions_per_judge": { "deepseek_flash": [] }, "judge_disagreement": 0 }, { "idx": 33, "duration_s": 62, "mean_score": 81.0, "mean_raw": 19.0, "scores_per_judge": [ 81 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "brand", "points": 10 }, { "category": "proper-noun", "points": 6 }, { "category": "mishear-minor", "points": 2 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 34, "duration_s": 66, "mean_score": 98.0, "mean_raw": 2.0, "scores_per_judge": [ 98 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 35, "duration_s": 77, "mean_score": 82.0, "mean_raw": 18.0, "scores_per_judge": [ 82 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 5 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-severe", "points": 5 }, { "category": "content-drop", "points": 3 } ] }, "judge_disagreement": 0 }, { "idx": 36, "duration_s": 94, "mean_score": 97.0, "mean_raw": 3.0, "scores_per_judge": [ 97 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "paraphrase", "points": 3 } ] }, "judge_disagreement": 0 }, { "idx": 37, "duration_s": 106, "mean_score": 99.0, "mean_raw": 1.0, "scores_per_judge": [ 99 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 38, "duration_s": 126, "mean_score": 86.0, "mean_raw": 14.0, "scores_per_judge": [ 86 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "tech", "points": 8 }, { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 39, "duration_s": 158, "mean_score": 99.0, "mean_raw": 1.0, "scores_per_judge": [ 99 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 40, "duration_s": 279, "mean_score": 90.0, "mean_raw": 10.0, "scores_per_judge": [ 90 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 41, "duration_s": 34, "mean_score": 100.0, "mean_raw": 0.0, "scores_per_judge": [ 100 ], "deductions_per_judge": { "deepseek_flash": [] }, "judge_disagreement": 0 }, { "idx": 42, "duration_s": 43, "mean_score": 80.0, "mean_raw": 20.0, "scores_per_judge": [ 80 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "semantic-inversion", "points": 20 } ] }, "judge_disagreement": 0 }, { "idx": 43, "duration_s": 29, "mean_score": 99.0, "mean_raw": 1.0, "scores_per_judge": [ 99 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 44, "duration_s": 71, "mean_score": 92.0, "mean_raw": 8.0, "scores_per_judge": [ 92 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "culture", "points": 8 } ] }, "judge_disagreement": 0 }, { "idx": 45, "duration_s": 46, "mean_score": 98.0, "mean_raw": 2.0, "scores_per_judge": [ 98 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-minor", "points": 2 } ] }, "judge_disagreement": 0 }, { "idx": 46, "duration_s": 62, "mean_score": 95.0, "mean_raw": 5.0, "scores_per_judge": [ 95 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 47, "duration_s": 67, "mean_score": 90.0, "mean_raw": 10.0, "scores_per_judge": [ 90 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "brand", "points": 10 } ] }, "judge_disagreement": 0 }, { "idx": 48, "duration_s": 72, "mean_score": 79.0, "mean_raw": 21.0, "scores_per_judge": [ 79 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "tech", "points": 8 }, { "category": "brand", "points": 10 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 49, "duration_s": 87, "mean_score": 89.0, "mean_raw": 11.0, "scores_per_judge": [ 89 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "tech", "points": 8 }, { "category": "filler", "points": 3 } ] }, "judge_disagreement": 0 }, { "idx": 50, "duration_s": 106, "mean_score": 92.0, "mean_raw": 8.0, "scores_per_judge": [ 92 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "culture", "points": 8 } ] }, "judge_disagreement": 0 }, { "idx": 51, "duration_s": 33, "mean_score": 99.0, "mean_raw": 1.0, "scores_per_judge": [ 99 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 52, "duration_s": 25, "mean_score": 97.0, "mean_raw": 3.0, "scores_per_judge": [ 97 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 53, "duration_s": 35, "mean_score": 97.0, "mean_raw": 3.0, "scores_per_judge": [ 97 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 3 } ] }, "judge_disagreement": 0 }, { "idx": 54, "duration_s": 36, "mean_score": 99.0, "mean_raw": 1.0, "scores_per_judge": [ 99 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 55, "duration_s": 33, "mean_score": 90.0, "mean_raw": 10.0, "scores_per_judge": [ 90 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 56, "duration_s": 15, "mean_score": 79.0, "mean_raw": 21.0, "scores_per_judge": [ 79 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "brand", "points": 10 }, { "category": "brand", "points": 10 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 57, "duration_s": 38, "mean_score": 92.0, "mean_raw": 8.0, "scores_per_judge": [ 92 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 58, "duration_s": 15, "mean_score": 98.0, "mean_raw": 2.0, "scores_per_judge": [ 98 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 59, "duration_s": 28, "mean_score": 100.0, "mean_raw": 0.0, "scores_per_judge": [ 100 ], "deductions_per_judge": { "deepseek_flash": [] }, "judge_disagreement": 0 }, { "idx": 60, "duration_s": 18, "mean_score": 100.0, "mean_raw": 0.0, "scores_per_judge": [ 100 ], "deductions_per_judge": { "deepseek_flash": [] }, "judge_disagreement": 0 }, { "idx": 61, "duration_s": 19, "mean_score": 85.0, "mean_raw": 15.0, "scores_per_judge": [ 85 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 }, { "category": "brand", "points": 10 } ] }, "judge_disagreement": 0 }, { "idx": 62, "duration_s": 17, "mean_score": 99.0, "mean_raw": 1.0, "scores_per_judge": [ 99 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "format", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 63, "duration_s": 21, "mean_score": 99.0, "mean_raw": 1.0, "scores_per_judge": [ 99 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 64, "duration_s": 23, "mean_score": 96.0, "mean_raw": 4.0, "scores_per_judge": [ 96 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 65, "duration_s": 20, "mean_score": 100.0, "mean_raw": 0.0, "scores_per_judge": [ 100 ], "deductions_per_judge": { "deepseek_flash": [] }, "judge_disagreement": 0 }, { "idx": 66, "duration_s": 24, "mean_score": 84.0, "mean_raw": 16.0, "scores_per_judge": [ 84 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "tech", "points": 8 }, { "category": "culture", "points": 8 } ] }, "judge_disagreement": 0 }, { "idx": 67, "duration_s": 16, "mean_score": 100.0, "mean_raw": 0.0, "scores_per_judge": [ 100 ], "deductions_per_judge": { "deepseek_flash": [] }, "judge_disagreement": 0 }, { "idx": 68, "duration_s": 18, "mean_score": 90.0, "mean_raw": 10.0, "scores_per_judge": [ 90 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "brand", "points": 10 } ] }, "judge_disagreement": 0 }, { "idx": 69, "duration_s": 18, "mean_score": 100.0, "mean_raw": 0.0, "scores_per_judge": [ 100 ], "deductions_per_judge": { "deepseek_flash": [] }, "judge_disagreement": 0 }, { "idx": 70, "duration_s": 17, "mean_score": 90.0, "mean_raw": 10.0, "scores_per_judge": [ 90 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "tech", "points": 8 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 71, "duration_s": 51, "mean_score": 98.0, "mean_raw": 2.0, "scores_per_judge": [ 98 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 72, "duration_s": 54, "mean_score": 94.0, "mean_raw": 6.0, "scores_per_judge": [ 94 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 73, "duration_s": 56, "mean_score": 93.0, "mean_raw": 7.0, "scores_per_judge": [ 93 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 74, "duration_s": 45, "mean_score": 95.0, "mean_raw": 5.0, "scores_per_judge": [ 95 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 75, "duration_s": 74, "mean_score": 87.0, "mean_raw": 13.0, "scores_per_judge": [ 87 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "tech", "points": 8 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 76, "duration_s": 58, "mean_score": 78.0, "mean_raw": 22.0, "scores_per_judge": [ 78 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "semantic-inversion", "points": 20 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 77, "duration_s": 130, "mean_score": 60.0, "mean_raw": 40.0, "scores_per_judge": [ 60 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "semantic-inversion", "points": 20 }, { "category": "brand", "points": 10 }, { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 78, "duration_s": 61, "mean_score": 100.0, "mean_raw": 0.0, "scores_per_judge": [ 100 ], "deductions_per_judge": { "deepseek_flash": [] }, "judge_disagreement": 0 }, { "idx": 79, "duration_s": 47, "mean_score": 94.0, "mean_raw": 6.0, "scores_per_judge": [ 94 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "paraphrase", "points": 3 }, { "category": "filler", "points": 3 } ] }, "judge_disagreement": 0 }, { "idx": 80, "duration_s": 42, "mean_score": 97.0, "mean_raw": 3.0, "scores_per_judge": [ 97 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 81, "duration_s": 38, "mean_score": 96.0, "mean_raw": 4.0, "scores_per_judge": [ 96 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-minor", "points": 2 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 82, "duration_s": 38, "mean_score": 81.0, "mean_raw": 19.0, "scores_per_judge": [ 81 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "numeric", "points": 10 }, { "category": "tech", "points": 8 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 83, "duration_s": 37, "mean_score": 98.0, "mean_raw": 2.0, "scores_per_judge": [ 98 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 84, "duration_s": 36, "mean_score": 65.0, "mean_raw": 35.0, "scores_per_judge": [ 65 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "polarity", "points": 35 } ] }, "judge_disagreement": 0 }, { "idx": 85, "duration_s": 36, "mean_score": 96.0, "mean_raw": 4.0, "scores_per_judge": [ 96 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 86, "duration_s": 35, "mean_score": 84.0, "mean_raw": 16.0, "scores_per_judge": [ 84 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "tech", "points": 8 }, { "category": "tech", "points": 8 } ] }, "judge_disagreement": 0 }, { "idx": 87, "duration_s": 34, "mean_score": 81.0, "mean_raw": 19.0, "scores_per_judge": [ 81 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "numeric", "points": 10 }, { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 4 } ] }, "judge_disagreement": 0 }, { "idx": 88, "duration_s": 34, "mean_score": 99.0, "mean_raw": 1.0, "scores_per_judge": [ 99 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 89, "duration_s": 34, "mean_score": 100.0, "mean_raw": 0.0, "scores_per_judge": [ 100 ], "deductions_per_judge": { "deepseek_flash": [] }, "judge_disagreement": 0 }, { "idx": 90, "duration_s": 33, "mean_score": 93.0, "mean_raw": 7.0, "scores_per_judge": [ 93 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 91, "duration_s": 33, "mean_score": 99.0, "mean_raw": 1.0, "scores_per_judge": [ 99 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 92, "duration_s": 32, "mean_score": 99.0, "mean_raw": 1.0, "scores_per_judge": [ 99 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 93, "duration_s": 32, "mean_score": 96.0, "mean_raw": 4.0, "scores_per_judge": [ 96 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-minor", "points": 2 }, { "category": "filler", "points": 2 } ] }, "judge_disagreement": 0 }, { "idx": 94, "duration_s": 32, "mean_score": 98.0, "mean_raw": 2.0, "scores_per_judge": [ 98 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 95, "duration_s": 32, "mean_score": 99.0, "mean_raw": 1.0, "scores_per_judge": [ 99 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 96, "duration_s": 31, "mean_score": 100.0, "mean_raw": 0.0, "scores_per_judge": [ 100 ], "deductions_per_judge": { "deepseek_flash": [] }, "judge_disagreement": 0 }, { "idx": 97, "duration_s": 30, "mean_score": 100.0, "mean_raw": 0.0, "scores_per_judge": [ 100 ], "deductions_per_judge": { "deepseek_flash": [] }, "judge_disagreement": 0 }, { "idx": 98, "duration_s": 30, "mean_score": 99.0, "mean_raw": 1.0, "scores_per_judge": [ 99 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 99, "duration_s": 30, "mean_score": 100.0, "mean_raw": 0.0, "scores_per_judge": [ 100 ], "deductions_per_judge": { "deepseek_flash": [] }, "judge_disagreement": 0 }, { "idx": 100, "duration_s": 30, "mean_score": 95.0, "mean_raw": 5.0, "scores_per_judge": [ 95 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 } ], "total_deductions": 812, "avg_deductions_per_sample": 8.12, "mean_clamped_score": 91.9, "n_judge_sample_pairs": 100, "gt_chars": 22625, "ded_per_100char": 3.59, "fidelity_score": 82.1, "overall_score": 91.9, "overall_raw_deductions": 8.12, "score_per_judge": { "deepseek_flash": 91.9 }, "per_group_deductions": { "meaning-flip": { "deepseek_flash": 215 }, "entity": { "deepseek_flash": 290 }, "cultural": { "deepseek_flash": 24 }, "fidelity": { "deepseek_flash": 283 } }, "per_bucket": { "general": { "n_samples": 40, "mean_raw": 9.8, "mean_score": 90.2, "total_raw": 394.0, "idxs": [ 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40 ], "worst_idx": 28 }, "idiom-test": { "n_samples": 5, "mean_raw": 6.2, "mean_score": 93.8, "total_raw": 31.0, "idxs": [ 41, 42, 43, 44, 45 ], "worst_idx": 42 }, "long-mixed": { "n_samples": 5, "mean_raw": 11.0, "mean_score": 89.0, "total_raw": 55.0, "idxs": [ 46, 47, 48, 49, 50 ], "worst_idx": 48 } }, "per_group_per_100char": { "meaning-flip": 0.95, "entity": 1.282, "cultural": 0.106, "fidelity": 1.251 }, "per_group_fidelity": { "meaning-flip": 77.2, "entity": 77.3, "cultural": 93.6, "fidelity": 85.3 } }, "elevenlabs": { "scores_by_judge": { "deepseek_flash": [ 100, 100, 100, 92, 98, 99, 98, 100, 96, 99, 91, 92, 68, 99, 97, 84, 98, 100, 90, 98, 100, 94, 100, 91, 100, 99, 97, 97, 90, 99, 100, 97, 79, 98, 97, 96, 100, 86, 85, 70, 87, 87, 87, 76, 67, 87, 87, 77, 83, 95, 94, 98, 99, 98, 94, 98, 74, 97, 93, 64, 100, 100, 99, 95, 99, 91, 94, 100, 99, 59, 92, 99, 90, 99, 95, 76, 68, 98, 98, 88, 92, 85, 93, 94, 95, 92, 97, 100, 100, 96, 99, 99, 93, 97, 95, 100, 97, 97, 100, 97 ] }, "per_category_deductions": { "polarity": { "deepseek_flash": 70 }, "semantic-inversion": { "deepseek_flash": 60 }, "hallucination": { "deepseek_flash": 15 }, "numeric": { "deepseek_flash": 0 }, "brand": { "deepseek_flash": 90 }, "tech": { "deepseek_flash": 64 }, "culture": { "deepseek_flash": 80 }, "proper-noun": { "deepseek_flash": 30 }, "dialect": { "deepseek_flash": 10 }, "mishear-severe": { "deepseek_flash": 85 }, "content-drop": { "deepseek_flash": 3 }, "paraphrase": { "deepseek_flash": 18 }, "mishear-minor": { "deepseek_flash": 14 }, "filler": { "deepseek_flash": 188 }, "format": { "deepseek_flash": 0 }, "unscoreable": { "deepseek_flash": 0 } }, "per_tier_deductions": { "tier1": { "deepseek_flash": 145 }, "tier2": { "deepseek_flash": 274 }, "tier3": { "deepseek_flash": 120 }, "tier4": { "deepseek_flash": 188 }, "unscoreable": { "deepseek_flash": 0 } }, "raw_deductions_by_judge": { "deepseek_flash": [ 0, 0, 0, 8, 2, 1, 2, 0, 4, 1, 9, 8, 32, 1, 3, 16, 2, 0, 10, 2, 0, 6, 0, 9, 0, 1, 3, 3, 10, 1, 0, 3, 21, 2, 3, 4, 0, 14, 15, 30, 13, 13, 13, 24, 33, 13, 13, 23, 17, 5, 6, 2, 1, 2, 6, 2, 26, 3, 7, 36, 0, 0, 1, 5, 1, 9, 6, 0, 1, 41, 8, 1, 10, 1, 5, 24, 32, 2, 2, 12, 8, 15, 7, 6, 5, 8, 3, 0, 0, 4, 1, 1, 7, 3, 5, 0, 3, 3, 0, 3 ] }, "sample_details": [ { "idx": 1, "duration_s": 5, "mean_score": 100.0, "mean_raw": 0.0, "scores_per_judge": [ 100 ], "deductions_per_judge": { "deepseek_flash": [] }, "judge_disagreement": 0 }, { "idx": 2, "duration_s": 7, "mean_score": 100.0, "mean_raw": 0.0, "scores_per_judge": [ 100 ], "deductions_per_judge": { "deepseek_flash": [] }, "judge_disagreement": 0 }, { "idx": 3, "duration_s": 8, "mean_score": 100.0, "mean_raw": 0.0, "scores_per_judge": [ 100 ], "deductions_per_judge": { "deepseek_flash": [] }, "judge_disagreement": 0 }, { "idx": 4, "duration_s": 11, "mean_score": 92.0, "mean_raw": 8.0, "scores_per_judge": [ 92 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "culture", "points": 8 } ] }, "judge_disagreement": 0 }, { "idx": 5, "duration_s": 16, "mean_score": 98.0, "mean_raw": 2.0, "scores_per_judge": [ 98 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 6, "duration_s": 17, "mean_score": 99.0, "mean_raw": 1.0, "scores_per_judge": [ 99 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 7, "duration_s": 20, "mean_score": 98.0, "mean_raw": 2.0, "scores_per_judge": [ 98 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 8, "duration_s": 30, "mean_score": 100.0, "mean_raw": 0.0, "scores_per_judge": [ 100 ], "deductions_per_judge": { "deepseek_flash": [] }, "judge_disagreement": 0 }, { "idx": 9, "duration_s": 32, "mean_score": 96.0, "mean_raw": 4.0, "scores_per_judge": [ 96 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 10, "duration_s": 35, "mean_score": 99.0, "mean_raw": 1.0, "scores_per_judge": [ 99 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 11, "duration_s": 40, "mean_score": 91.0, "mean_raw": 9.0, "scores_per_judge": [ 91 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 }, { "category": "mishear-minor", "points": 2 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 12, "duration_s": 46, "mean_score": 92.0, "mean_raw": 8.0, "scores_per_judge": [ 92 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "culture", "points": 8 } ] }, "judge_disagreement": 0 }, { "idx": 13, "duration_s": 60, "mean_score": 68.0, "mean_raw": 32.0, "scores_per_judge": [ 68 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "brand", "points": 10 }, { "category": "brand", "points": 10 }, { "category": "tech", "points": 8 }, { "category": "mishear-minor", "points": 2 } ] }, "judge_disagreement": 0 }, { "idx": 14, "duration_s": 64, "mean_score": 99.0, "mean_raw": 1.0, "scores_per_judge": [ 99 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 15, "duration_s": 69, "mean_score": 97.0, "mean_raw": 3.0, "scores_per_judge": [ 97 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 16, "duration_s": 81, "mean_score": 84.0, "mean_raw": 16.0, "scores_per_judge": [ 84 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "proper-noun", "points": 6 }, { "category": "proper-noun", "points": 6 }, { "category": "filler", "points": 2 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 17, "duration_s": 100, "mean_score": 98.0, "mean_raw": 2.0, "scores_per_judge": [ 98 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 18, "duration_s": 125, "mean_score": 100.0, "mean_raw": 0.0, "scores_per_judge": [ 100 ], "deductions_per_judge": { "deepseek_flash": [] }, "judge_disagreement": 0 }, { "idx": 19, "duration_s": 133, "mean_score": 90.0, "mean_raw": 10.0, "scores_per_judge": [ 90 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 20, "duration_s": 164, "mean_score": 98.0, "mean_raw": 2.0, "scores_per_judge": [ 98 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 21, "duration_s": 5, "mean_score": 100.0, "mean_raw": 0.0, "scores_per_judge": [ 100 ], "deductions_per_judge": { "deepseek_flash": [] }, "judge_disagreement": 0 }, { "idx": 22, "duration_s": 7, "mean_score": 94.0, "mean_raw": 6.0, "scores_per_judge": [ 94 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "proper-noun", "points": 6 } ] }, "judge_disagreement": 0 }, { "idx": 23, "duration_s": 10, "mean_score": 100.0, "mean_raw": 0.0, "scores_per_judge": [ 100 ], "deductions_per_judge": { "deepseek_flash": [] }, "judge_disagreement": 0 }, { "idx": 24, "duration_s": 14, "mean_score": 91.0, "mean_raw": 9.0, "scores_per_judge": [ 91 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "tech", "points": 8 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 25, "duration_s": 16, "mean_score": 100.0, "mean_raw": 0.0, "scores_per_judge": [ 100 ], "deductions_per_judge": { "deepseek_flash": [] }, "judge_disagreement": 0 }, { "idx": 26, "duration_s": 23, "mean_score": 99.0, "mean_raw": 1.0, "scores_per_judge": [ 99 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 27, "duration_s": 25, "mean_score": 97.0, "mean_raw": 3.0, "scores_per_judge": [ 97 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "paraphrase", "points": 3 } ] }, "judge_disagreement": 0 }, { "idx": 28, "duration_s": 30, "mean_score": 97.0, "mean_raw": 3.0, "scores_per_judge": [ 97 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 29, "duration_s": 34, "mean_score": 90.0, "mean_raw": 10.0, "scores_per_judge": [ 90 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "brand", "points": 10 } ] }, "judge_disagreement": 0 }, { "idx": 30, "duration_s": 38, "mean_score": 99.0, "mean_raw": 1.0, "scores_per_judge": [ 99 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 31, "duration_s": 44, "mean_score": 100.0, "mean_raw": 0.0, "scores_per_judge": [ 100 ], "deductions_per_judge": { "deepseek_flash": [] }, "judge_disagreement": 0 }, { "idx": 32, "duration_s": 50, "mean_score": 97.0, "mean_raw": 3.0, "scores_per_judge": [ 97 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 33, "duration_s": 62, "mean_score": 79.0, "mean_raw": 21.0, "scores_per_judge": [ 79 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "brand", "points": 10 }, { "category": "proper-noun", "points": 6 }, { "category": "mishear-severe", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 34, "duration_s": 66, "mean_score": 98.0, "mean_raw": 2.0, "scores_per_judge": [ 98 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 35, "duration_s": 77, "mean_score": 97.0, "mean_raw": 3.0, "scores_per_judge": [ 97 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 36, "duration_s": 94, "mean_score": 96.0, "mean_raw": 4.0, "scores_per_judge": [ 96 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "paraphrase", "points": 3 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 37, "duration_s": 106, "mean_score": 100.0, "mean_raw": 0.0, "scores_per_judge": [ 100 ], "deductions_per_judge": { "deepseek_flash": [] }, "judge_disagreement": 0 }, { "idx": 38, "duration_s": 126, "mean_score": 86.0, "mean_raw": 14.0, "scores_per_judge": [ 86 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "tech", "points": 8 }, { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 39, "duration_s": 158, "mean_score": 85.0, "mean_raw": 15.0, "scores_per_judge": [ 85 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "brand", "points": 10 }, { "category": "mishear-severe", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 40, "duration_s": 279, "mean_score": 70.0, "mean_raw": 30.0, "scores_per_judge": [ 70 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "culture", "points": 8 }, { "category": "culture", "points": 8 }, { "category": "dialect", "points": 5 }, { "category": "paraphrase", "points": 3 }, { "category": "paraphrase", "points": 3 }, { "category": "filler", "points": 3 } ] }, "judge_disagreement": 0 }, { "idx": 41, "duration_s": 34, "mean_score": 87.0, "mean_raw": 13.0, "scores_per_judge": [ 87 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "culture", "points": 8 }, { "category": "filler", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 42, "duration_s": 43, "mean_score": 87.0, "mean_raw": 13.0, "scores_per_judge": [ 87 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "culture", "points": 8 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 43, "duration_s": 29, "mean_score": 87.0, "mean_raw": 13.0, "scores_per_judge": [ 87 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "culture", "points": 8 }, { "category": "filler", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 44, "duration_s": 71, "mean_score": 76.0, "mean_raw": 24.0, "scores_per_judge": [ 76 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "proper-noun", "points": 6 }, { "category": "culture", "points": 8 }, { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 45, "duration_s": 46, "mean_score": 67.0, "mean_raw": 33.0, "scores_per_judge": [ 67 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "semantic-inversion", "points": 20 }, { "category": "dialect", "points": 5 }, { "category": "paraphrase", "points": 3 }, { "category": "filler", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 46, "duration_s": 62, "mean_score": 87.0, "mean_raw": 13.0, "scores_per_judge": [ 87 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "culture", "points": 8 }, { "category": "filler", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 47, "duration_s": 67, "mean_score": 87.0, "mean_raw": 13.0, "scores_per_judge": [ 87 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 5 }, { "category": "tech", "points": 8 } ] }, "judge_disagreement": 0 }, { "idx": 48, "duration_s": 72, "mean_score": 77.0, "mean_raw": 23.0, "scores_per_judge": [ 77 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "tech", "points": 8 }, { "category": "brand", "points": 10 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 49, "duration_s": 87, "mean_score": 83.0, "mean_raw": 17.0, "scores_per_judge": [ 83 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "brand", "points": 10 }, { "category": "mishear-minor", "points": 2 }, { "category": "filler", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 50, "duration_s": 106, "mean_score": 95.0, "mean_raw": 5.0, "scores_per_judge": [ 95 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 51, "duration_s": 33, "mean_score": 94.0, "mean_raw": 6.0, "scores_per_judge": [ 94 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 52, "duration_s": 25, "mean_score": 98.0, "mean_raw": 2.0, "scores_per_judge": [ 98 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 53, "duration_s": 35, "mean_score": 99.0, "mean_raw": 1.0, "scores_per_judge": [ 99 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 54, "duration_s": 36, "mean_score": 98.0, "mean_raw": 2.0, "scores_per_judge": [ 98 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 55, "duration_s": 33, "mean_score": 94.0, "mean_raw": 6.0, "scores_per_judge": [ 94 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 56, "duration_s": 15, "mean_score": 98.0, "mean_raw": 2.0, "scores_per_judge": [ 98 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 57, "duration_s": 38, "mean_score": 74.0, "mean_raw": 26.0, "scores_per_judge": [ 74 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "hallucination", "points": 15 }, { "category": "tech", "points": 8 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 58, "duration_s": 15, "mean_score": 97.0, "mean_raw": 3.0, "scores_per_judge": [ 97 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 59, "duration_s": 28, "mean_score": 93.0, "mean_raw": 7.0, "scores_per_judge": [ 93 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 2 } ] }, "judge_disagreement": 0 }, { "idx": 60, "duration_s": 18, "mean_score": 64.0, "mean_raw": 36.0, "scores_per_judge": [ 64 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "polarity", "points": 35 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 61, "duration_s": 19, "mean_score": 100.0, "mean_raw": 0.0, "scores_per_judge": [ 100 ], "deductions_per_judge": { "deepseek_flash": [] }, "judge_disagreement": 0 }, { "idx": 62, "duration_s": 17, "mean_score": 100.0, "mean_raw": 0.0, "scores_per_judge": [ 100 ], "deductions_per_judge": { "deepseek_flash": [] }, "judge_disagreement": 0 }, { "idx": 63, "duration_s": 21, "mean_score": 99.0, "mean_raw": 1.0, "scores_per_judge": [ 99 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 64, "duration_s": 23, "mean_score": 95.0, "mean_raw": 5.0, "scores_per_judge": [ 95 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 65, "duration_s": 20, "mean_score": 99.0, "mean_raw": 1.0, "scores_per_judge": [ 99 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 66, "duration_s": 24, "mean_score": 91.0, "mean_raw": 9.0, "scores_per_judge": [ 91 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "culture", "points": 8 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 67, "duration_s": 16, "mean_score": 94.0, "mean_raw": 6.0, "scores_per_judge": [ 94 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-minor", "points": 2 }, { "category": "mishear-minor", "points": 2 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 68, "duration_s": 18, "mean_score": 100.0, "mean_raw": 0.0, "scores_per_judge": [ 100 ], "deductions_per_judge": { "deepseek_flash": [] }, "judge_disagreement": 0 }, { "idx": 69, "duration_s": 18, "mean_score": 99.0, "mean_raw": 1.0, "scores_per_judge": [ 99 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 70, "duration_s": 17, "mean_score": 59.0, "mean_raw": 41.0, "scores_per_judge": [ 59 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "polarity", "points": 35 }, { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 71, "duration_s": 51, "mean_score": 92.0, "mean_raw": 8.0, "scores_per_judge": [ 92 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 3 } ] }, "judge_disagreement": 0 }, { "idx": 72, "duration_s": 54, "mean_score": 99.0, "mean_raw": 1.0, "scores_per_judge": [ 99 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 73, "duration_s": 56, "mean_score": 90.0, "mean_raw": 10.0, "scores_per_judge": [ 90 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "brand", "points": 10 } ] }, "judge_disagreement": 0 }, { "idx": 74, "duration_s": 45, "mean_score": 99.0, "mean_raw": 1.0, "scores_per_judge": [ 99 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 75, "duration_s": 74, "mean_score": 95.0, "mean_raw": 5.0, "scores_per_judge": [ 95 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 76, "duration_s": 58, "mean_score": 76.0, "mean_raw": 24.0, "scores_per_judge": [ 76 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "semantic-inversion", "points": 20 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 77, "duration_s": 130, "mean_score": 68.0, "mean_raw": 32.0, "scores_per_judge": [ 68 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "semantic-inversion", "points": 20 }, { "category": "brand", "points": 10 }, { "category": "filler", "points": 2 } ] }, "judge_disagreement": 0 }, { "idx": 78, "duration_s": 61, "mean_score": 98.0, "mean_raw": 2.0, "scores_per_judge": [ 98 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 79, "duration_s": 47, "mean_score": 98.0, "mean_raw": 2.0, "scores_per_judge": [ 98 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 80, "duration_s": 42, "mean_score": 88.0, "mean_raw": 12.0, "scores_per_judge": [ 88 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 }, { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 81, "duration_s": 38, "mean_score": 92.0, "mean_raw": 8.0, "scores_per_judge": [ 92 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 82, "duration_s": 38, "mean_score": 85.0, "mean_raw": 15.0, "scores_per_judge": [ 85 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "tech", "points": 8 }, { "category": "mishear-severe", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 83, "duration_s": 37, "mean_score": 93.0, "mean_raw": 7.0, "scores_per_judge": [ 93 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-minor", "points": 2 }, { "category": "filler", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 84, "duration_s": 36, "mean_score": 94.0, "mean_raw": 6.0, "scores_per_judge": [ 94 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-minor", "points": 2 }, { "category": "filler", "points": 4 } ] }, "judge_disagreement": 0 }, { "idx": 85, "duration_s": 36, "mean_score": 95.0, "mean_raw": 5.0, "scores_per_judge": [ 95 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 86, "duration_s": 35, "mean_score": 92.0, "mean_raw": 8.0, "scores_per_judge": [ 92 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "tech", "points": 8 } ] }, "judge_disagreement": 0 }, { "idx": 87, "duration_s": 34, "mean_score": 97.0, "mean_raw": 3.0, "scores_per_judge": [ 97 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 88, "duration_s": 34, "mean_score": 100.0, "mean_raw": 0.0, "scores_per_judge": [ 100 ], "deductions_per_judge": { "deepseek_flash": [] }, "judge_disagreement": 0 }, { "idx": 89, "duration_s": 34, "mean_score": 100.0, "mean_raw": 0.0, "scores_per_judge": [ 100 ], "deductions_per_judge": { "deepseek_flash": [] }, "judge_disagreement": 0 }, { "idx": 90, "duration_s": 33, "mean_score": 96.0, "mean_raw": 4.0, "scores_per_judge": [ 96 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "content-drop", "points": 3 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 91, "duration_s": 33, "mean_score": 99.0, "mean_raw": 1.0, "scores_per_judge": [ 99 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 92, "duration_s": 32, "mean_score": 99.0, "mean_raw": 1.0, "scores_per_judge": [ 99 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 93, "duration_s": 32, "mean_score": 93.0, "mean_raw": 7.0, "scores_per_judge": [ 93 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 94, "duration_s": 32, "mean_score": 97.0, "mean_raw": 3.0, "scores_per_judge": [ 97 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 95, "duration_s": 32, "mean_score": 95.0, "mean_raw": 5.0, "scores_per_judge": [ 95 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "paraphrase", "points": 3 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 96, "duration_s": 31, "mean_score": 100.0, "mean_raw": 0.0, "scores_per_judge": [ 100 ], "deductions_per_judge": { "deepseek_flash": [] }, "judge_disagreement": 0 }, { "idx": 97, "duration_s": 30, "mean_score": 97.0, "mean_raw": 3.0, "scores_per_judge": [ 97 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 98, "duration_s": 30, "mean_score": 97.0, "mean_raw": 3.0, "scores_per_judge": [ 97 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 99, "duration_s": 30, "mean_score": 100.0, "mean_raw": 0.0, "scores_per_judge": [ 100 ], "deductions_per_judge": { "deepseek_flash": [] }, "judge_disagreement": 0 }, { "idx": 100, "duration_s": 30, "mean_score": 97.0, "mean_raw": 3.0, "scores_per_judge": [ 97 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 } ], "total_deductions": 727, "avg_deductions_per_sample": 7.27, "mean_clamped_score": 92.7, "n_judge_sample_pairs": 100, "gt_chars": 22625, "ded_per_100char": 3.21, "fidelity_score": 83.9, "overall_score": 92.7, "overall_raw_deductions": 7.27, "score_per_judge": { "deepseek_flash": 92.7 }, "per_group_deductions": { "meaning-flip": { "deepseek_flash": 145 }, "entity": { "deepseek_flash": 184 }, "cultural": { "deepseek_flash": 90 }, "fidelity": { "deepseek_flash": 308 } }, "per_bucket": { "general": { "n_samples": 40, "mean_raw": 5.7, "mean_score": 94.3, "total_raw": 226.0, "idxs": [ 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40 ], "worst_idx": 13 }, "idiom-test": { "n_samples": 5, "mean_raw": 19.2, "mean_score": 80.8, "total_raw": 96.0, "idxs": [ 41, 42, 43, 44, 45 ], "worst_idx": 45 }, "long-mixed": { "n_samples": 5, "mean_raw": 14.2, "mean_score": 85.8, "total_raw": 71.0, "idxs": [ 46, 47, 48, 49, 50 ], "worst_idx": 48 } }, "per_group_per_100char": { "meaning-flip": 0.641, "entity": 0.813, "cultural": 0.398, "fidelity": 1.361 }, "per_group_fidelity": { "meaning-flip": 84.6, "entity": 85.6, "cultural": 75.9, "fidelity": 84.1 } }, "firered": { "scores_by_judge": { "deepseek_flash": [ 100, 95, 95, 92, 97, 86, 77, 92, 90, 87, 73, 57, 10, 63, 60, 50, 19, 54, 0, 65, 91, 94, 92, 90, 88, 84, 96, 84, 59, 62, 59, 60, 28, 23, 37, 63, 76, 10, 0, null, 71, 52, 68, 28, 73, 88, 70, 47, 39, 60, 67, 74, 81, 86, 84, 68, 42, 79, 79, 40, 67, 100, 91, 92, 91, 65, 93, 87, 89, 56, 52, 79, 58, 80, 74, 51, 12, 95, 97, 93, 92, 85, 96, 95, 85, 75, 85, 97, 94, 97, 96, 100, 75, 90, 95, 100, 93, 97, 98, 84 ] }, "per_category_deductions": { "polarity": { "deepseek_flash": 140 }, "semantic-inversion": { "deepseek_flash": 40 }, "hallucination": { "deepseek_flash": 0 }, "numeric": { "deepseek_flash": 10 }, "brand": { "deepseek_flash": 420 }, "tech": { "deepseek_flash": 816 }, "culture": { "deepseek_flash": 288 }, "proper-noun": { "deepseek_flash": 126 }, "dialect": { "deepseek_flash": 5 }, "mishear-severe": { "deepseek_flash": 520 }, "content-drop": { "deepseek_flash": 36 }, "paraphrase": { "deepseek_flash": 3 }, "mishear-minor": { "deepseek_flash": 102 }, "filler": { "deepseek_flash": 245 }, "format": { "deepseek_flash": 1 }, "unscoreable": { "deepseek_flash": 0 } }, "per_tier_deductions": { "tier1": { "deepseek_flash": 190 }, "tier2": { "deepseek_flash": 1655 }, "tier3": { "deepseek_flash": 661 }, "tier4": { "deepseek_flash": 246 }, "unscoreable": { "deepseek_flash": 0 } }, "raw_deductions_by_judge": { "deepseek_flash": [ 0, 5, 5, 8, 3, 14, 23, 8, 10, 13, 27, 43, 90, 37, 40, 50, 81, 46, 156, 35, 9, 6, 8, 10, 12, 16, 4, 16, 41, 38, 41, 40, 72, 77, 63, 37, 24, 90, 101, null, 29, 48, 32, 72, 27, 12, 30, 53, 61, 40, 33, 26, 19, 14, 16, 32, 58, 21, 21, 60, 33, 0, 9, 8, 9, 35, 7, 13, 11, 44, 48, 21, 42, 20, 26, 49, 88, 5, 3, 7, 8, 15, 4, 5, 15, 25, 15, 3, 6, 3, 4, 0, 25, 10, 5, 0, 7, 3, 2, 16 ] }, "sample_details": [ { "idx": 1, "duration_s": 5, "mean_score": 100.0, "mean_raw": 0.0, "scores_per_judge": [ 100 ], "deductions_per_judge": { "deepseek_flash": [] }, "judge_disagreement": 0 }, { "idx": 2, "duration_s": 7, "mean_score": 95.0, "mean_raw": 5.0, "scores_per_judge": [ 95 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 3, "duration_s": 8, "mean_score": 95.0, "mean_raw": 5.0, "scores_per_judge": [ 95 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 4, "duration_s": 11, "mean_score": 92.0, "mean_raw": 8.0, "scores_per_judge": [ 92 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "culture", "points": 8 } ] }, "judge_disagreement": 0 }, { "idx": 5, "duration_s": 16, "mean_score": 97.0, "mean_raw": 3.0, "scores_per_judge": [ 97 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 6, "duration_s": 17, "mean_score": 86.0, "mean_raw": 14.0, "scores_per_judge": [ 86 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "tech", "points": 8 }, { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 7, "duration_s": 20, "mean_score": 77.0, "mean_raw": 23.0, "scores_per_judge": [ 77 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "brand", "points": 10 }, { "category": "tech", "points": 8 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 8, "duration_s": 30, "mean_score": 92.0, "mean_raw": 8.0, "scores_per_judge": [ 92 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "tech", "points": 8 } ] }, "judge_disagreement": 0 }, { "idx": 9, "duration_s": 32, "mean_score": 90.0, "mean_raw": 10.0, "scores_per_judge": [ 90 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 }, { "category": "mishear-severe", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 10, "duration_s": 35, "mean_score": 87.0, "mean_raw": 13.0, "scores_per_judge": [ 87 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "culture", "points": 8 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 11, "duration_s": 40, "mean_score": 73.0, "mean_raw": 27.0, "scores_per_judge": [ 73 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "tech", "points": 8 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 12, "duration_s": 46, "mean_score": 57.0, "mean_raw": 43.0, "scores_per_judge": [ 57 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "brand", "points": 10 }, { "category": "culture", "points": 8 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 13, "duration_s": 60, "mean_score": 10.0, "mean_raw": 90.0, "scores_per_judge": [ 10 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "brand", "points": 10 }, { "category": "brand", "points": 10 }, { "category": "brand", "points": 10 }, { "category": "tech", "points": 8 }, { "category": "brand", "points": 10 }, { "category": "content-drop", "points": 3 }, { "category": "brand", "points": 10 }, { "category": "tech", "points": 8 }, { "category": "tech", "points": 8 }, { "category": "brand", "points": 10 }, { "category": "filler", "points": 3 } ] }, "judge_disagreement": 0 }, { "idx": 14, "duration_s": 64, "mean_score": 63.0, "mean_raw": 37.0, "scores_per_judge": [ 63 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-severe", "points": 5 }, { "category": "proper-noun", "points": 6 }, { "category": "content-drop", "points": 3 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 15, "duration_s": 69, "mean_score": 60.0, "mean_raw": 40.0, "scores_per_judge": [ 60 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "culture", "points": 8 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-minor", "points": 2 }, { "category": "mishear-minor", "points": 2 }, { "category": "filler", "points": 3 } ] }, "judge_disagreement": 0 }, { "idx": 16, "duration_s": 81, "mean_score": 50.0, "mean_raw": 50.0, "scores_per_judge": [ 50 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "proper-noun", "points": 6 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-severe", "points": 5 }, { "category": "content-drop", "points": 3 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-severe", "points": 5 }, { "category": "proper-noun", "points": 6 }, { "category": "tech", "points": 8 }, { "category": "mishear-minor", "points": 2 }, { "category": "filler", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 17, "duration_s": 100, "mean_score": 19.0, "mean_raw": 81.0, "scores_per_judge": [ 19 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "brand", "points": 10 }, { "category": "brand", "points": 10 }, { "category": "brand", "points": 10 }, { "category": "tech", "points": 8 }, { "category": "tech", "points": 8 }, { "category": "tech", "points": 8 }, { "category": "tech", "points": 8 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 4 } ] }, "judge_disagreement": 0 }, { "idx": 18, "duration_s": 125, "mean_score": 54.0, "mean_raw": 46.0, "scores_per_judge": [ 54 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "brand", "points": 10 }, { "category": "brand", "points": 10 }, { "category": "tech", "points": 8 }, { "category": "tech", "points": 8 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-minor", "points": 2 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 19, "duration_s": 133, "mean_score": 0.0, "mean_raw": 156.0, "scores_per_judge": [ 0 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "tech", "points": 8 }, { "category": "tech", "points": 8 }, { "category": "tech", "points": 8 }, { "category": "brand", "points": 10 }, { "category": "brand", "points": 10 }, { "category": "tech", "points": 8 }, { "category": "brand", "points": 10 }, { "category": "tech", "points": 8 }, { "category": "tech", "points": 8 }, { "category": "brand", "points": 10 }, { "category": "tech", "points": 8 }, { "category": "tech", "points": 8 }, { "category": "tech", "points": 8 }, { "category": "brand", "points": 10 }, { "category": "brand", "points": 10 }, { "category": "tech", "points": 8 }, { "category": "tech", "points": 8 }, { "category": "mishear-minor", "points": 2 }, { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 20, "duration_s": 164, "mean_score": 65.0, "mean_raw": 35.0, "scores_per_judge": [ 65 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 }, { "category": "culture", "points": 8 }, { "category": "mishear-minor", "points": 2 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 21, "duration_s": 5, "mean_score": 91.0, "mean_raw": 9.0, "scores_per_judge": [ 91 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "tech", "points": 8 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 22, "duration_s": 7, "mean_score": 94.0, "mean_raw": 6.0, "scores_per_judge": [ 94 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "proper-noun", "points": 6 } ] }, "judge_disagreement": 0 }, { "idx": 23, "duration_s": 10, "mean_score": 92.0, "mean_raw": 8.0, "scores_per_judge": [ 92 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "culture", "points": 8 } ] }, "judge_disagreement": 0 }, { "idx": 24, "duration_s": 14, "mean_score": 90.0, "mean_raw": 10.0, "scores_per_judge": [ 90 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "tech", "points": 8 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 25, "duration_s": 16, "mean_score": 88.0, "mean_raw": 12.0, "scores_per_judge": [ 88 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "tech", "points": 8 }, { "category": "content-drop", "points": 3 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 26, "duration_s": 23, "mean_score": 84.0, "mean_raw": 16.0, "scores_per_judge": [ 84 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-severe", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 27, "duration_s": 25, "mean_score": 96.0, "mean_raw": 4.0, "scores_per_judge": [ 96 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-minor", "points": 2 }, { "category": "mishear-minor", "points": 2 } ] }, "judge_disagreement": 0 }, { "idx": 28, "duration_s": 30, "mean_score": 84.0, "mean_raw": 16.0, "scores_per_judge": [ 84 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "tech", "points": 8 }, { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 29, "duration_s": 34, "mean_score": 59.0, "mean_raw": 41.0, "scores_per_judge": [ 59 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "brand", "points": 10 }, { "category": "brand", "points": 10 }, { "category": "brand", "points": 10 }, { "category": "tech", "points": 8 }, { "category": "paraphrase", "points": 3 } ] }, "judge_disagreement": 0 }, { "idx": 30, "duration_s": 38, "mean_score": 62.0, "mean_raw": 38.0, "scores_per_judge": [ 62 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "culture", "points": 8 }, { "category": "tech", "points": 8 }, { "category": "tech", "points": 8 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-minor", "points": 2 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 31, "duration_s": 44, "mean_score": 59.0, "mean_raw": 41.0, "scores_per_judge": [ 59 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "brand", "points": 10 }, { "category": "brand", "points": 10 }, { "category": "tech", "points": 8 }, { "category": "tech", "points": 8 }, { "category": "filler", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 32, "duration_s": 50, "mean_score": 60.0, "mean_raw": 40.0, "scores_per_judge": [ 60 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "tech", "points": 8 }, { "category": "tech", "points": 8 }, { "category": "culture", "points": 8 }, { "category": "tech", "points": 8 }, { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 33, "duration_s": 62, "mean_score": 28.0, "mean_raw": 72.0, "scores_per_judge": [ 28 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "tech", "points": 8 }, { "category": "tech", "points": 8 }, { "category": "brand", "points": 10 }, { "category": "brand", "points": 10 }, { "category": "tech", "points": 8 }, { "category": "tech", "points": 8 }, { "category": "proper-noun", "points": 6 }, { "category": "tech", "points": 8 }, { "category": "content-drop", "points": 3 }, { "category": "filler", "points": 3 } ] }, "judge_disagreement": 0 }, { "idx": 34, "duration_s": 66, "mean_score": 23.0, "mean_raw": 77.0, "scores_per_judge": [ 23 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "polarity", "points": 35 }, { "category": "tech", "points": 8 }, { "category": "tech", "points": 8 }, { "category": "tech", "points": 8 }, { "category": "tech", "points": 8 }, { "category": "mishear-severe", "points": 5 }, { "category": "content-drop", "points": 3 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 35, "duration_s": 77, "mean_score": 37.0, "mean_raw": 63.0, "scores_per_judge": [ 37 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "tech", "points": 8 }, { "category": "tech", "points": 8 }, { "category": "tech", "points": 8 }, { "category": "tech", "points": 8 }, { "category": "tech", "points": 8 }, { "category": "tech", "points": 8 }, { "category": "tech", "points": 8 }, { "category": "mishear-minor", "points": 2 }, { "category": "filler", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 36, "duration_s": 94, "mean_score": 63.0, "mean_raw": 37.0, "scores_per_judge": [ 63 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "tech", "points": 8 }, { "category": "tech", "points": 8 }, { "category": "tech", "points": 8 }, { "category": "tech", "points": 8 }, { "category": "filler", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 37, "duration_s": 106, "mean_score": 76.0, "mean_raw": 24.0, "scores_per_judge": [ 76 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "proper-noun", "points": 6 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 3 } ] }, "judge_disagreement": 0 }, { "idx": 38, "duration_s": 126, "mean_score": 10.0, "mean_raw": 90.0, "scores_per_judge": [ 10 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "culture", "points": 8 }, { "category": "culture", "points": 8 }, { "category": "culture", "points": 8 }, { "category": "culture", "points": 8 }, { "category": "tech", "points": 8 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-minor", "points": 2 }, { "category": "mishear-minor", "points": 2 }, { "category": "mishear-minor", "points": 2 }, { "category": "mishear-minor", "points": 2 }, { "category": "mishear-minor", "points": 2 }, { "category": "mishear-minor", "points": 2 }, { "category": "mishear-minor", "points": 2 }, { "category": "mishear-minor", "points": 2 }, { "category": "mishear-minor", "points": 2 }, { "category": "mishear-minor", "points": 2 }, { "category": "filler", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 39, "duration_s": 158, "mean_score": 0.0, "mean_raw": 101.0, "scores_per_judge": [ 0 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "brand", "points": 10 }, { "category": "brand", "points": 10 }, { "category": "brand", "points": 10 }, { "category": "tech", "points": 8 }, { "category": "tech", "points": 8 }, { "category": "tech", "points": 8 }, { "category": "culture", "points": 8 }, { "category": "proper-noun", "points": 6 }, { "category": "proper-noun", "points": 6 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-minor", "points": 2 }, { "category": "mishear-minor", "points": 2 }, { "category": "mishear-minor", "points": 2 }, { "category": "content-drop", "points": 3 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 41, "duration_s": 34, "mean_score": 71.0, "mean_raw": 29.0, "scores_per_judge": [ 71 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "culture", "points": 8 }, { "category": "culture", "points": 8 }, { "category": "culture", "points": 8 }, { "category": "mishear-minor", "points": 2 }, { "category": "filler", "points": 3 } ] }, "judge_disagreement": 0 }, { "idx": 42, "duration_s": 43, "mean_score": 52.0, "mean_raw": 48.0, "scores_per_judge": [ 52 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "semantic-inversion", "points": 20 }, { "category": "culture", "points": 8 }, { "category": "culture", "points": 8 }, { "category": "mishear-severe", "points": 5 }, { "category": "content-drop", "points": 3 }, { "category": "filler", "points": 4 } ] }, "judge_disagreement": 0 }, { "idx": 43, "duration_s": 29, "mean_score": 68.0, "mean_raw": 32.0, "scores_per_judge": [ 68 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "culture", "points": 8 }, { "category": "culture", "points": 8 }, { "category": "culture", "points": 8 }, { "category": "mishear-minor", "points": 2 }, { "category": "mishear-minor", "points": 2 }, { "category": "filler", "points": 4 } ] }, "judge_disagreement": 0 }, { "idx": 44, "duration_s": 71, "mean_score": 28.0, "mean_raw": 72.0, "scores_per_judge": [ 28 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "culture", "points": 8 }, { "category": "proper-noun", "points": 6 }, { "category": "proper-noun", "points": 6 }, { "category": "mishear-severe", "points": 5 }, { "category": "culture", "points": 8 }, { "category": "mishear-severe", "points": 5 }, { "category": "culture", "points": 8 }, { "category": "culture", "points": 8 }, { "category": "culture", "points": 8 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-minor", "points": 2 }, { "category": "filler", "points": 2 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 45, "duration_s": 46, "mean_score": 73.0, "mean_raw": 27.0, "scores_per_judge": [ 73 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "dialect", "points": 5 }, { "category": "culture", "points": 8 }, { "category": "culture", "points": 8 }, { "category": "mishear-minor", "points": 2 }, { "category": "mishear-minor", "points": 2 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 46, "duration_s": 62, "mean_score": 88.0, "mean_raw": 12.0, "scores_per_judge": [ 88 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "culture", "points": 8 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 47, "duration_s": 67, "mean_score": 70.0, "mean_raw": 30.0, "scores_per_judge": [ 70 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "brand", "points": 10 }, { "category": "tech", "points": 8 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 48, "duration_s": 72, "mean_score": 47.0, "mean_raw": 53.0, "scores_per_judge": [ 47 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "tech", "points": 8 }, { "category": "tech", "points": 8 }, { "category": "tech", "points": 8 }, { "category": "tech", "points": 8 }, { "category": "brand", "points": 10 }, { "category": "mishear-severe", "points": 5 }, { "category": "content-drop", "points": 3 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 49, "duration_s": 87, "mean_score": 39.0, "mean_raw": 61.0, "scores_per_judge": [ 39 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "tech", "points": 8 }, { "category": "tech", "points": 8 }, { "category": "proper-noun", "points": 6 }, { "category": "proper-noun", "points": 6 }, { "category": "tech", "points": 8 }, { "category": "brand", "points": 10 }, { "category": "tech", "points": 8 }, { "category": "mishear-minor", "points": 2 }, { "category": "filler", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 50, "duration_s": 106, "mean_score": 60.0, "mean_raw": 40.0, "scores_per_judge": [ 60 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "culture", "points": 8 }, { "category": "culture", "points": 8 }, { "category": "culture", "points": 8 }, { "category": "tech", "points": 8 }, { "category": "tech", "points": 8 } ] }, "judge_disagreement": 0 }, { "idx": 51, "duration_s": 33, "mean_score": 67.0, "mean_raw": 33.0, "scores_per_judge": [ 67 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 }, { "category": "culture", "points": 8 }, { "category": "mishear-severe", "points": 5 }, { "category": "proper-noun", "points": 6 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-minor", "points": 2 }, { "category": "mishear-minor", "points": 2 } ] }, "judge_disagreement": 0 }, { "idx": 52, "duration_s": 25, "mean_score": 74.0, "mean_raw": 26.0, "scores_per_judge": [ 74 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "proper-noun", "points": 6 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 53, "duration_s": 35, "mean_score": 81.0, "mean_raw": 19.0, "scores_per_judge": [ 81 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-minor", "points": 2 }, { "category": "mishear-minor", "points": 2 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 54, "duration_s": 36, "mean_score": 86.0, "mean_raw": 14.0, "scores_per_judge": [ 86 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "tech", "points": 8 }, { "category": "mishear-minor", "points": 2 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 55, "duration_s": 33, "mean_score": 84.0, "mean_raw": 16.0, "scores_per_judge": [ 84 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "proper-noun", "points": 6 }, { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 56, "duration_s": 15, "mean_score": 68.0, "mean_raw": 32.0, "scores_per_judge": [ 68 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "brand", "points": 10 }, { "category": "brand", "points": 10 }, { "category": "brand", "points": 10 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 57, "duration_s": 38, "mean_score": 42.0, "mean_raw": 58.0, "scores_per_judge": [ 42 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "tech", "points": 8 }, { "category": "tech", "points": 8 }, { "category": "tech", "points": 8 }, { "category": "tech", "points": 8 }, { "category": "tech", "points": 8 }, { "category": "tech", "points": 8 }, { "category": "proper-noun", "points": 6 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 58, "duration_s": 15, "mean_score": 79.0, "mean_raw": 21.0, "scores_per_judge": [ 79 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "brand", "points": 10 }, { "category": "tech", "points": 8 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 59, "duration_s": 28, "mean_score": 79.0, "mean_raw": 21.0, "scores_per_judge": [ 79 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "tech", "points": 8 }, { "category": "tech", "points": 8 }, { "category": "content-drop", "points": 3 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 60, "duration_s": 18, "mean_score": 40.0, "mean_raw": 60.0, "scores_per_judge": [ 40 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "polarity", "points": 35 }, { "category": "tech", "points": 8 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 61, "duration_s": 19, "mean_score": 67.0, "mean_raw": 33.0, "scores_per_judge": [ 67 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-severe", "points": 5 }, { "category": "brand", "points": 10 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 62, "duration_s": 17, "mean_score": 100.0, "mean_raw": 0.0, "scores_per_judge": [ 100 ], "deductions_per_judge": { "deepseek_flash": [] }, "judge_disagreement": 0 }, { "idx": 63, "duration_s": 21, "mean_score": 91.0, "mean_raw": 9.0, "scores_per_judge": [ 91 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "tech", "points": 8 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 64, "duration_s": 23, "mean_score": 92.0, "mean_raw": 8.0, "scores_per_judge": [ 92 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 65, "duration_s": 20, "mean_score": 91.0, "mean_raw": 9.0, "scores_per_judge": [ 91 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "tech", "points": 8 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 66, "duration_s": 24, "mean_score": 65.0, "mean_raw": 35.0, "scores_per_judge": [ 65 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "tech", "points": 8 }, { "category": "tech", "points": 8 }, { "category": "culture", "points": 8 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-severe", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 67, "duration_s": 16, "mean_score": 93.0, "mean_raw": 7.0, "scores_per_judge": [ 93 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 }, { "category": "mishear-minor", "points": 2 } ] }, "judge_disagreement": 0 }, { "idx": 68, "duration_s": 18, "mean_score": 87.0, "mean_raw": 13.0, "scores_per_judge": [ 87 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "tech", "points": 8 }, { "category": "mishear-severe", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 69, "duration_s": 18, "mean_score": 89.0, "mean_raw": 11.0, "scores_per_judge": [ 89 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 }, { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 70, "duration_s": 17, "mean_score": 56.0, "mean_raw": 44.0, "scores_per_judge": [ 56 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "polarity", "points": 35 }, { "category": "tech", "points": 8 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 71, "duration_s": 51, "mean_score": 52.0, "mean_raw": 48.0, "scores_per_judge": [ 52 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 }, { "category": "tech", "points": 8 }, { "category": "brand", "points": 10 }, { "category": "brand", "points": 10 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-severe", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 72, "duration_s": 54, "mean_score": 79.0, "mean_raw": 21.0, "scores_per_judge": [ 79 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "tech", "points": 8 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 73, "duration_s": 56, "mean_score": 58.0, "mean_raw": 42.0, "scores_per_judge": [ 58 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 }, { "category": "mishear-minor", "points": 2 }, { "category": "mishear-minor", "points": 2 }, { "category": "brand", "points": 10 }, { "category": "brand", "points": 10 }, { "category": "mishear-minor", "points": 2 }, { "category": "content-drop", "points": 3 }, { "category": "mishear-minor", "points": 2 }, { "category": "proper-noun", "points": 6 } ] }, "judge_disagreement": 0 }, { "idx": 74, "duration_s": 45, "mean_score": 80.0, "mean_raw": 20.0, "scores_per_judge": [ 80 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "tech", "points": 8 }, { "category": "tech", "points": 8 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 75, "duration_s": 74, "mean_score": 74.0, "mean_raw": 26.0, "scores_per_judge": [ 74 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "tech", "points": 8 }, { "category": "tech", "points": 8 }, { "category": "content-drop", "points": 3 }, { "category": "mishear-minor", "points": 2 }, { "category": "filler", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 76, "duration_s": 58, "mean_score": 51.0, "mean_raw": 49.0, "scores_per_judge": [ 51 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "proper-noun", "points": 6 }, { "category": "proper-noun", "points": 6 }, { "category": "proper-noun", "points": 6 }, { "category": "proper-noun", "points": 6 }, { "category": "culture", "points": 8 }, { "category": "culture", "points": 8 }, { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 4 } ] }, "judge_disagreement": 0 }, { "idx": 77, "duration_s": 130, "mean_score": 12.0, "mean_raw": 88.0, "scores_per_judge": [ 12 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "polarity", "points": 35 }, { "category": "semantic-inversion", "points": 20 }, { "category": "brand", "points": 10 }, { "category": "tech", "points": 8 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-minor", "points": 2 }, { "category": "mishear-minor", "points": 2 }, { "category": "mishear-minor", "points": 2 }, { "category": "filler", "points": 4 } ] }, "judge_disagreement": 0 }, { "idx": 78, "duration_s": 61, "mean_score": 95.0, "mean_raw": 5.0, "scores_per_judge": [ 95 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 79, "duration_s": 47, "mean_score": 97.0, "mean_raw": 3.0, "scores_per_judge": [ 97 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 80, "duration_s": 42, "mean_score": 93.0, "mean_raw": 7.0, "scores_per_judge": [ 93 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 81, "duration_s": 38, "mean_score": 92.0, "mean_raw": 8.0, "scores_per_judge": [ 92 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 }, { "category": "mishear-minor", "points": 2 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 82, "duration_s": 38, "mean_score": 85.0, "mean_raw": 15.0, "scores_per_judge": [ 85 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "tech", "points": 8 }, { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 83, "duration_s": 37, "mean_score": 96.0, "mean_raw": 4.0, "scores_per_judge": [ 96 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 84, "duration_s": 36, "mean_score": 95.0, "mean_raw": 5.0, "scores_per_judge": [ 95 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "mishear-minor", "points": 2 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 85, "duration_s": 36, "mean_score": 85.0, "mean_raw": 15.0, "scores_per_judge": [ 85 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-minor", "points": 2 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 86, "duration_s": 35, "mean_score": 75.0, "mean_raw": 25.0, "scores_per_judge": [ 75 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "tech", "points": 8 }, { "category": "tech", "points": 8 }, { "category": "tech", "points": 8 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 87, "duration_s": 34, "mean_score": 85.0, "mean_raw": 15.0, "scores_per_judge": [ 85 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "numeric", "points": 10 }, { "category": "mishear-minor", "points": 2 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 88, "duration_s": 34, "mean_score": 97.0, "mean_raw": 3.0, "scores_per_judge": [ 97 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 89, "duration_s": 34, "mean_score": 94.0, "mean_raw": 6.0, "scores_per_judge": [ 94 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 90, "duration_s": 33, "mean_score": 97.0, "mean_raw": 3.0, "scores_per_judge": [ 97 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-minor", "points": 2 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 91, "duration_s": 33, "mean_score": 96.0, "mean_raw": 4.0, "scores_per_judge": [ 96 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 4 } ] }, "judge_disagreement": 0 }, { "idx": 92, "duration_s": 32, "mean_score": 100.0, "mean_raw": 0.0, "scores_per_judge": [ 100 ], "deductions_per_judge": { "deepseek_flash": [] }, "judge_disagreement": 0 }, { "idx": 93, "duration_s": 32, "mean_score": 75.0, "mean_raw": 25.0, "scores_per_judge": [ 75 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "tech", "points": 8 }, { "category": "tech", "points": 8 }, { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 1 }, { "category": "format", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 94, "duration_s": 32, "mean_score": 90.0, "mean_raw": 10.0, "scores_per_judge": [ 90 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 95, "duration_s": 32, "mean_score": 95.0, "mean_raw": 5.0, "scores_per_judge": [ 95 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 96, "duration_s": 31, "mean_score": 100.0, "mean_raw": 0.0, "scores_per_judge": [ 100 ], "deductions_per_judge": { "deepseek_flash": [] }, "judge_disagreement": 0 }, { "idx": 97, "duration_s": 30, "mean_score": 93.0, "mean_raw": 7.0, "scores_per_judge": [ 93 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 98, "duration_s": 30, "mean_score": 97.0, "mean_raw": 3.0, "scores_per_judge": [ 97 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-minor", "points": 2 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 99, "duration_s": 30, "mean_score": 98.0, "mean_raw": 2.0, "scores_per_judge": [ 98 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 100, "duration_s": 30, "mean_score": 84.0, "mean_raw": 16.0, "scores_per_judge": [ 84 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-minor", "points": 2 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 } ], "total_deductions": 2752, "avg_deductions_per_sample": 27.8, "mean_clamped_score": 72.8, "n_judge_sample_pairs": 99, "gt_chars": 21388, "ded_per_100char": 12.87, "fidelity_score": 35.7, "overall_score": 72.8, "overall_raw_deductions": 27.8, "score_per_judge": { "deepseek_flash": 72.8 }, "per_group_deductions": { "meaning-flip": { "deepseek_flash": 190 }, "entity": { "deepseek_flash": 1362 }, "cultural": { "deepseek_flash": 293 }, "fidelity": { "deepseek_flash": 907 } }, "per_bucket": { "general": { "n_samples": 39, "mean_raw": 35.9, "mean_score": 65.6, "total_raw": 1399.0, "idxs": [ 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39 ], "worst_idx": 19 }, "idiom-test": { "n_samples": 5, "mean_raw": 41.6, "mean_score": 58.4, "total_raw": 208.0, "idxs": [ 41, 42, 43, 44, 45 ], "worst_idx": 44 }, "long-mixed": { "n_samples": 5, "mean_raw": 39.2, "mean_score": 60.8, "total_raw": 196.0, "idxs": [ 46, 47, 48, 49, 50 ], "worst_idx": 49 } }, "per_group_per_100char": { "meaning-flip": 0.888, "entity": 6.368, "cultural": 1.37, "fidelity": 4.241 }, "per_group_fidelity": { "meaning-flip": 78.7, "entity": 0.0, "cultural": 17.2, "fidelity": 50.3 } }, "firered_aed": { "scores_by_judge": { "deepseek_flash": [ 100, 100, 100, 100, 98, 99, 81, 100, 73, 98, 82, 96, 51, 67, 66, 74, 48, 82, 1, 93, 99, 94, 100, 90, 91, 99, 100, 74, 67, 55, 97, 95, 61, 77, 66, 97, 98, 79, 51, 65, 89, 92, 82, 85, 96, 95, 79, 89, 74, 67, 85, 93, 96, 98, 89, 70, 66, 89, 79, 62, 100, 100, 96, 94, 100, 92, 98, 90, 94, 90, 84, 91, 90, 97, 83, 64, 72, 89, 94, 98, 89, 94, 96, 64, 97, 84, 81, 98, 100, 95, 99, 98, 90, 99, 99, 99, 96, 99, 100, 78 ] }, "per_category_deductions": { "polarity": { "deepseek_flash": 105 }, "semantic-inversion": { "deepseek_flash": 20 }, "hallucination": { "deepseek_flash": 60 }, "numeric": { "deepseek_flash": 10 }, "brand": { "deepseek_flash": 270 }, "tech": { "deepseek_flash": 312 }, "culture": { "deepseek_flash": 96 }, "proper-noun": { "deepseek_flash": 60 }, "dialect": { "deepseek_flash": 0 }, "mishear-severe": { "deepseek_flash": 185 }, "content-drop": { "deepseek_flash": 39 }, "paraphrase": { "deepseek_flash": 12 }, "mishear-minor": { "deepseek_flash": 46 }, "filler": { "deepseek_flash": 175 }, "format": { "deepseek_flash": 0 }, "unscoreable": { "deepseek_flash": 0 } }, "per_tier_deductions": { "tier1": { "deepseek_flash": 195 }, "tier2": { "deepseek_flash": 738 }, "tier3": { "deepseek_flash": 282 }, "tier4": { "deepseek_flash": 175 }, "unscoreable": { "deepseek_flash": 0 } }, "raw_deductions_by_judge": { "deepseek_flash": [ 0, 0, 0, 0, 2, 1, 19, 0, 27, 2, 18, 4, 49, 33, 34, 26, 52, 18, 99, 7, 1, 6, 0, 10, 9, 1, 0, 26, 33, 45, 3, 5, 39, 23, 34, 3, 2, 21, 49, 35, 11, 8, 18, 15, 4, 5, 21, 11, 26, 33, 15, 7, 4, 2, 11, 30, 34, 11, 21, 38, 0, 0, 4, 6, 0, 8, 2, 10, 6, 10, 16, 9, 10, 3, 17, 36, 28, 11, 6, 2, 11, 6, 4, 36, 3, 16, 19, 2, 0, 5, 1, 2, 10, 1, 1, 1, 4, 1, 0, 22 ] }, "sample_details": [ { "idx": 1, "duration_s": 5, "mean_score": 100.0, "mean_raw": 0.0, "scores_per_judge": [ 100 ], "deductions_per_judge": { "deepseek_flash": [] }, "judge_disagreement": 0 }, { "idx": 2, "duration_s": 7, "mean_score": 100.0, "mean_raw": 0.0, "scores_per_judge": [ 100 ], "deductions_per_judge": { "deepseek_flash": [] }, "judge_disagreement": 0 }, { "idx": 3, "duration_s": 8, "mean_score": 100.0, "mean_raw": 0.0, "scores_per_judge": [ 100 ], "deductions_per_judge": { "deepseek_flash": [] }, "judge_disagreement": 0 }, { "idx": 4, "duration_s": 11, "mean_score": 100.0, "mean_raw": 0.0, "scores_per_judge": [ 100 ], "deductions_per_judge": { "deepseek_flash": [] }, "judge_disagreement": 0 }, { "idx": 5, "duration_s": 16, "mean_score": 98.0, "mean_raw": 2.0, "scores_per_judge": [ 98 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 6, "duration_s": 17, "mean_score": 99.0, "mean_raw": 1.0, "scores_per_judge": [ 99 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 7, "duration_s": 20, "mean_score": 81.0, "mean_raw": 19.0, "scores_per_judge": [ 81 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "brand", "points": 10 }, { "category": "tech", "points": 8 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 8, "duration_s": 30, "mean_score": 100.0, "mean_raw": 0.0, "scores_per_judge": [ 100 ], "deductions_per_judge": { "deepseek_flash": [] }, "judge_disagreement": 0 }, { "idx": 9, "duration_s": 32, "mean_score": 73.0, "mean_raw": 27.0, "scores_per_judge": [ 73 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "hallucination", "points": 15 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 2 } ] }, "judge_disagreement": 0 }, { "idx": 10, "duration_s": 35, "mean_score": 98.0, "mean_raw": 2.0, "scores_per_judge": [ 98 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 11, "duration_s": 40, "mean_score": 82.0, "mean_raw": 18.0, "scores_per_judge": [ 82 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "tech", "points": 8 }, { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 12, "duration_s": 46, "mean_score": 96.0, "mean_raw": 4.0, "scores_per_judge": [ 96 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 4 } ] }, "judge_disagreement": 0 }, { "idx": 13, "duration_s": 60, "mean_score": 51.0, "mean_raw": 49.0, "scores_per_judge": [ 51 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "brand", "points": 10 }, { "category": "brand", "points": 10 }, { "category": "brand", "points": 10 }, { "category": "tech", "points": 8 }, { "category": "tech", "points": 8 }, { "category": "mishear-minor", "points": 2 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 14, "duration_s": 64, "mean_score": 67.0, "mean_raw": 33.0, "scores_per_judge": [ 67 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "proper-noun", "points": 6 }, { "category": "proper-noun", "points": 6 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 15, "duration_s": 69, "mean_score": 66.0, "mean_raw": 34.0, "scores_per_judge": [ 66 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "hallucination", "points": 15 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-minor", "points": 2 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 16, "duration_s": 81, "mean_score": 74.0, "mean_raw": 26.0, "scores_per_judge": [ 74 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 }, { "category": "proper-noun", "points": 6 }, { "category": "tech", "points": 8 }, { "category": "content-drop", "points": 3 }, { "category": "mishear-minor", "points": 2 }, { "category": "filler", "points": 2 } ] }, "judge_disagreement": 0 }, { "idx": 17, "duration_s": 100, "mean_score": 48.0, "mean_raw": 52.0, "scores_per_judge": [ 48 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "brand", "points": 10 }, { "category": "tech", "points": 8 }, { "category": "tech", "points": 8 }, { "category": "tech", "points": 8 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-severe", "points": 5 }, { "category": "content-drop", "points": 3 }, { "category": "content-drop", "points": 3 }, { "category": "filler", "points": 2 } ] }, "judge_disagreement": 0 }, { "idx": 18, "duration_s": 125, "mean_score": 82.0, "mean_raw": 18.0, "scores_per_judge": [ 82 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "brand", "points": 10 }, { "category": "tech", "points": 8 } ] }, "judge_disagreement": 0 }, { "idx": 19, "duration_s": 133, "mean_score": 1.0, "mean_raw": 99.0, "scores_per_judge": [ 1 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "brand", "points": 10 }, { "category": "brand", "points": 10 }, { "category": "brand", "points": 10 }, { "category": "brand", "points": 10 }, { "category": "brand", "points": 10 }, { "category": "brand", "points": 10 }, { "category": "tech", "points": 8 }, { "category": "content-drop", "points": 3 }, { "category": "content-drop", "points": 3 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 20, "duration_s": 164, "mean_score": 93.0, "mean_raw": 7.0, "scores_per_judge": [ 93 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 }, { "category": "mishear-minor", "points": 2 } ] }, "judge_disagreement": 0 }, { "idx": 21, "duration_s": 5, "mean_score": 99.0, "mean_raw": 1.0, "scores_per_judge": [ 99 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 22, "duration_s": 7, "mean_score": 94.0, "mean_raw": 6.0, "scores_per_judge": [ 94 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "proper-noun", "points": 6 } ] }, "judge_disagreement": 0 }, { "idx": 23, "duration_s": 10, "mean_score": 100.0, "mean_raw": 0.0, "scores_per_judge": [ 100 ], "deductions_per_judge": { "deepseek_flash": [] }, "judge_disagreement": 0 }, { "idx": 24, "duration_s": 14, "mean_score": 90.0, "mean_raw": 10.0, "scores_per_judge": [ 90 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "tech", "points": 8 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 25, "duration_s": 16, "mean_score": 91.0, "mean_raw": 9.0, "scores_per_judge": [ 91 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "tech", "points": 8 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 26, "duration_s": 23, "mean_score": 99.0, "mean_raw": 1.0, "scores_per_judge": [ 99 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 27, "duration_s": 25, "mean_score": 100.0, "mean_raw": 0.0, "scores_per_judge": [ 100 ], "deductions_per_judge": { "deepseek_flash": [] }, "judge_disagreement": 0 }, { "idx": 28, "duration_s": 30, "mean_score": 74.0, "mean_raw": 26.0, "scores_per_judge": [ 74 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "hallucination", "points": 15 }, { "category": "tech", "points": 8 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 29, "duration_s": 34, "mean_score": 67.0, "mean_raw": 33.0, "scores_per_judge": [ 67 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "brand", "points": 10 }, { "category": "brand", "points": 10 }, { "category": "tech", "points": 8 }, { "category": "content-drop", "points": 3 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 30, "duration_s": 38, "mean_score": 55.0, "mean_raw": 45.0, "scores_per_judge": [ 55 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "polarity", "points": 35 }, { "category": "culture", "points": 8 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 31, "duration_s": 44, "mean_score": 97.0, "mean_raw": 3.0, "scores_per_judge": [ 97 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-minor", "points": 2 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 32, "duration_s": 50, "mean_score": 95.0, "mean_raw": 5.0, "scores_per_judge": [ 95 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "paraphrase", "points": 3 } ] }, "judge_disagreement": 0 }, { "idx": 33, "duration_s": 62, "mean_score": 61.0, "mean_raw": 39.0, "scores_per_judge": [ 61 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-minor", "points": 2 }, { "category": "tech", "points": 8 }, { "category": "brand", "points": 10 }, { "category": "content-drop", "points": 3 }, { "category": "tech", "points": 8 }, { "category": "proper-noun", "points": 6 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 34, "duration_s": 66, "mean_score": 77.0, "mean_raw": 23.0, "scores_per_judge": [ 77 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "tech", "points": 8 }, { "category": "culture", "points": 8 }, { "category": "content-drop", "points": 3 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 35, "duration_s": 77, "mean_score": 66.0, "mean_raw": 34.0, "scores_per_judge": [ 66 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "tech", "points": 8 }, { "category": "tech", "points": 8 }, { "category": "tech", "points": 8 }, { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 36, "duration_s": 94, "mean_score": 97.0, "mean_raw": 3.0, "scores_per_judge": [ 97 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "content-drop", "points": 3 } ] }, "judge_disagreement": 0 }, { "idx": 37, "duration_s": 106, "mean_score": 98.0, "mean_raw": 2.0, "scores_per_judge": [ 98 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-minor", "points": 2 } ] }, "judge_disagreement": 0 }, { "idx": 38, "duration_s": 126, "mean_score": 79.0, "mean_raw": 21.0, "scores_per_judge": [ 79 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "tech", "points": 8 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 39, "duration_s": 158, "mean_score": 51.0, "mean_raw": 49.0, "scores_per_judge": [ 51 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "tech", "points": 8 }, { "category": "tech", "points": 8 }, { "category": "brand", "points": 10 }, { "category": "brand", "points": 10 }, { "category": "proper-noun", "points": 6 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-minor", "points": 2 } ] }, "judge_disagreement": 0 }, { "idx": 40, "duration_s": 279, "mean_score": 65.0, "mean_raw": 35.0, "scores_per_judge": [ 65 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "culture", "points": 8 }, { "category": "culture", "points": 8 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-minor", "points": 2 }, { "category": "paraphrase", "points": 3 }, { "category": "filler", "points": 4 } ] }, "judge_disagreement": 0 }, { "idx": 41, "duration_s": 34, "mean_score": 89.0, "mean_raw": 11.0, "scores_per_judge": [ 89 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "culture", "points": 8 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 42, "duration_s": 43, "mean_score": 92.0, "mean_raw": 8.0, "scores_per_judge": [ 92 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "culture", "points": 8 } ] }, "judge_disagreement": 0 }, { "idx": 43, "duration_s": 29, "mean_score": 82.0, "mean_raw": 18.0, "scores_per_judge": [ 82 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "culture", "points": 8 }, { "category": "proper-noun", "points": 6 }, { "category": "filler", "points": 4 } ] }, "judge_disagreement": 0 }, { "idx": 44, "duration_s": 71, "mean_score": 85.0, "mean_raw": 15.0, "scores_per_judge": [ 85 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 }, { "category": "culture", "points": 8 }, { "category": "filler", "points": 2 } ] }, "judge_disagreement": 0 }, { "idx": 45, "duration_s": 46, "mean_score": 96.0, "mean_raw": 4.0, "scores_per_judge": [ 96 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-minor", "points": 2 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 46, "duration_s": 62, "mean_score": 95.0, "mean_raw": 5.0, "scores_per_judge": [ 95 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 4 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 47, "duration_s": 67, "mean_score": 79.0, "mean_raw": 21.0, "scores_per_judge": [ 79 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "brand", "points": 10 }, { "category": "tech", "points": 8 }, { "category": "mishear-minor", "points": 2 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 48, "duration_s": 72, "mean_score": 89.0, "mean_raw": 11.0, "scores_per_judge": [ 89 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "tech", "points": 8 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 49, "duration_s": 87, "mean_score": 74.0, "mean_raw": 26.0, "scores_per_judge": [ 74 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "tech", "points": 8 }, { "category": "brand", "points": 10 }, { "category": "tech", "points": 8 } ] }, "judge_disagreement": 0 }, { "idx": 50, "duration_s": 106, "mean_score": 67.0, "mean_raw": 33.0, "scores_per_judge": [ 67 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "culture", "points": 8 }, { "category": "culture", "points": 8 }, { "category": "tech", "points": 8 }, { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 4 } ] }, "judge_disagreement": 0 }, { "idx": 51, "duration_s": 33, "mean_score": 85.0, "mean_raw": 15.0, "scores_per_judge": [ 85 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "culture", "points": 8 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-minor", "points": 2 } ] }, "judge_disagreement": 0 }, { "idx": 52, "duration_s": 25, "mean_score": 93.0, "mean_raw": 7.0, "scores_per_judge": [ 93 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 53, "duration_s": 35, "mean_score": 96.0, "mean_raw": 4.0, "scores_per_judge": [ 96 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "mishear-minor", "points": 2 } ] }, "judge_disagreement": 0 }, { "idx": 54, "duration_s": 36, "mean_score": 98.0, "mean_raw": 2.0, "scores_per_judge": [ 98 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 55, "duration_s": 33, "mean_score": 89.0, "mean_raw": 11.0, "scores_per_judge": [ 89 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "proper-noun", "points": 6 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 56, "duration_s": 15, "mean_score": 70.0, "mean_raw": 30.0, "scores_per_judge": [ 70 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "brand", "points": 10 }, { "category": "brand", "points": 10 }, { "category": "brand", "points": 10 } ] }, "judge_disagreement": 0 }, { "idx": 57, "duration_s": 38, "mean_score": 66.0, "mean_raw": 34.0, "scores_per_judge": [ 66 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "tech", "points": 8 }, { "category": "tech", "points": 8 }, { "category": "tech", "points": 8 }, { "category": "tech", "points": 8 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 58, "duration_s": 15, "mean_score": 89.0, "mean_raw": 11.0, "scores_per_judge": [ 89 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "brand", "points": 10 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 59, "duration_s": 28, "mean_score": 79.0, "mean_raw": 21.0, "scores_per_judge": [ 79 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "tech", "points": 8 }, { "category": "tech", "points": 8 }, { "category": "mishear-minor", "points": 2 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 60, "duration_s": 18, "mean_score": 62.0, "mean_raw": 38.0, "scores_per_judge": [ 62 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "polarity", "points": 35 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 61, "duration_s": 19, "mean_score": 100.0, "mean_raw": 0.0, "scores_per_judge": [ 100 ], "deductions_per_judge": { "deepseek_flash": [] }, "judge_disagreement": 0 }, { "idx": 62, "duration_s": 17, "mean_score": 100.0, "mean_raw": 0.0, "scores_per_judge": [ 100 ], "deductions_per_judge": { "deepseek_flash": [] }, "judge_disagreement": 0 }, { "idx": 63, "duration_s": 21, "mean_score": 96.0, "mean_raw": 4.0, "scores_per_judge": [ 96 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-minor", "points": 2 }, { "category": "filler", "points": 2 } ] }, "judge_disagreement": 0 }, { "idx": 64, "duration_s": 23, "mean_score": 94.0, "mean_raw": 6.0, "scores_per_judge": [ 94 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 3 }, { "category": "paraphrase", "points": 3 } ] }, "judge_disagreement": 0 }, { "idx": 65, "duration_s": 20, "mean_score": 100.0, "mean_raw": 0.0, "scores_per_judge": [ 100 ], "deductions_per_judge": { "deepseek_flash": [] }, "judge_disagreement": 0 }, { "idx": 66, "duration_s": 24, "mean_score": 92.0, "mean_raw": 8.0, "scores_per_judge": [ 92 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "culture", "points": 8 } ] }, "judge_disagreement": 0 }, { "idx": 67, "duration_s": 16, "mean_score": 98.0, "mean_raw": 2.0, "scores_per_judge": [ 98 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-minor", "points": 2 } ] }, "judge_disagreement": 0 }, { "idx": 68, "duration_s": 18, "mean_score": 90.0, "mean_raw": 10.0, "scores_per_judge": [ 90 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "brand", "points": 10 } ] }, "judge_disagreement": 0 }, { "idx": 69, "duration_s": 18, "mean_score": 94.0, "mean_raw": 6.0, "scores_per_judge": [ 94 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 70, "duration_s": 17, "mean_score": 90.0, "mean_raw": 10.0, "scores_per_judge": [ 90 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "tech", "points": 8 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 71, "duration_s": 51, "mean_score": 84.0, "mean_raw": 16.0, "scores_per_judge": [ 84 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "brand", "points": 10 }, { "category": "mishear-minor", "points": 2 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 72, "duration_s": 54, "mean_score": 91.0, "mean_raw": 9.0, "scores_per_judge": [ 91 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 }, { "category": "mishear-minor", "points": 2 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 73, "duration_s": 56, "mean_score": 90.0, "mean_raw": 10.0, "scores_per_judge": [ 90 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "brand", "points": 10 } ] }, "judge_disagreement": 0 }, { "idx": 74, "duration_s": 45, "mean_score": 97.0, "mean_raw": 3.0, "scores_per_judge": [ 97 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 75, "duration_s": 74, "mean_score": 83.0, "mean_raw": 17.0, "scores_per_judge": [ 83 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "tech", "points": 8 }, { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 2 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 76, "duration_s": 58, "mean_score": 64.0, "mean_raw": 36.0, "scores_per_judge": [ 64 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "semantic-inversion", "points": 20 }, { "category": "proper-noun", "points": 6 }, { "category": "proper-noun", "points": 6 }, { "category": "filler", "points": 4 } ] }, "judge_disagreement": 0 }, { "idx": 77, "duration_s": 130, "mean_score": 72.0, "mean_raw": 28.0, "scores_per_judge": [ 72 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "brand", "points": 10 }, { "category": "tech", "points": 8 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-minor", "points": 2 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 78, "duration_s": 61, "mean_score": 89.0, "mean_raw": 11.0, "scores_per_judge": [ 89 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 }, { "category": "mishear-minor", "points": 2 }, { "category": "filler", "points": 4 } ] }, "judge_disagreement": 0 }, { "idx": 79, "duration_s": 47, "mean_score": 94.0, "mean_raw": 6.0, "scores_per_judge": [ 94 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "paraphrase", "points": 3 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 80, "duration_s": 42, "mean_score": 98.0, "mean_raw": 2.0, "scores_per_judge": [ 98 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 81, "duration_s": 38, "mean_score": 89.0, "mean_raw": 11.0, "scores_per_judge": [ 89 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 }, { "category": "mishear-minor", "points": 2 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 82, "duration_s": 38, "mean_score": 94.0, "mean_raw": 6.0, "scores_per_judge": [ 94 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "content-drop", "points": 3 } ] }, "judge_disagreement": 0 }, { "idx": 83, "duration_s": 37, "mean_score": 96.0, "mean_raw": 4.0, "scores_per_judge": [ 96 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 4 } ] }, "judge_disagreement": 0 }, { "idx": 84, "duration_s": 36, "mean_score": 64.0, "mean_raw": 36.0, "scores_per_judge": [ 64 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "polarity", "points": 35 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 85, "duration_s": 36, "mean_score": 97.0, "mean_raw": 3.0, "scores_per_judge": [ 97 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "content-drop", "points": 3 } ] }, "judge_disagreement": 0 }, { "idx": 86, "duration_s": 35, "mean_score": 84.0, "mean_raw": 16.0, "scores_per_judge": [ 84 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "tech", "points": 8 }, { "category": "tech", "points": 8 } ] }, "judge_disagreement": 0 }, { "idx": 87, "duration_s": 34, "mean_score": 81.0, "mean_raw": 19.0, "scores_per_judge": [ 81 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "numeric", "points": 10 }, { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 88, "duration_s": 34, "mean_score": 98.0, "mean_raw": 2.0, "scores_per_judge": [ 98 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 89, "duration_s": 34, "mean_score": 100.0, "mean_raw": 0.0, "scores_per_judge": [ 100 ], "deductions_per_judge": { "deepseek_flash": [] }, "judge_disagreement": 0 }, { "idx": 90, "duration_s": 33, "mean_score": 95.0, "mean_raw": 5.0, "scores_per_judge": [ 95 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "content-drop", "points": 3 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 91, "duration_s": 33, "mean_score": 99.0, "mean_raw": 1.0, "scores_per_judge": [ 99 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 92, "duration_s": 32, "mean_score": 98.0, "mean_raw": 2.0, "scores_per_judge": [ 98 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-minor", "points": 2 } ] }, "judge_disagreement": 0 }, { "idx": 93, "duration_s": 32, "mean_score": 90.0, "mean_raw": 10.0, "scores_per_judge": [ 90 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 }, { "category": "content-drop", "points": 3 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 94, "duration_s": 32, "mean_score": 99.0, "mean_raw": 1.0, "scores_per_judge": [ 99 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 95, "duration_s": 32, "mean_score": 99.0, "mean_raw": 1.0, "scores_per_judge": [ 99 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 96, "duration_s": 31, "mean_score": 99.0, "mean_raw": 1.0, "scores_per_judge": [ 99 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 97, "duration_s": 30, "mean_score": 96.0, "mean_raw": 4.0, "scores_per_judge": [ 96 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 98, "duration_s": 30, "mean_score": 99.0, "mean_raw": 1.0, "scores_per_judge": [ 99 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 99, "duration_s": 30, "mean_score": 100.0, "mean_raw": 0.0, "scores_per_judge": [ 100 ], "deductions_per_judge": { "deepseek_flash": [] }, "judge_disagreement": 0 }, { "idx": 100, "duration_s": 30, "mean_score": 78.0, "mean_raw": 22.0, "scores_per_judge": [ 78 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "hallucination", "points": 15 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-minor", "points": 2 } ] }, "judge_disagreement": 0 } ], "total_deductions": 1390, "avg_deductions_per_sample": 13.9, "mean_clamped_score": 86.1, "n_judge_sample_pairs": 100, "gt_chars": 22625, "ded_per_100char": 6.14, "fidelity_score": 69.3, "overall_score": 86.1, "overall_raw_deductions": 13.9, "score_per_judge": { "deepseek_flash": 86.1 }, "per_group_deductions": { "meaning-flip": { "deepseek_flash": 195 }, "entity": { "deepseek_flash": 642 }, "cultural": { "deepseek_flash": 96 }, "fidelity": { "deepseek_flash": 457 } }, "per_bucket": { "general": { "n_samples": 40, "mean_raw": 18.4, "mean_score": 81.6, "total_raw": 736.0, "idxs": [ 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40 ], "worst_idx": 19 }, "idiom-test": { "n_samples": 5, "mean_raw": 11.2, "mean_score": 88.8, "total_raw": 56.0, "idxs": [ 41, 42, 43, 44, 45 ], "worst_idx": 43 }, "long-mixed": { "n_samples": 5, "mean_raw": 19.2, "mean_score": 80.8, "total_raw": 96.0, "idxs": [ 46, 47, 48, 49, 50 ], "worst_idx": 50 } }, "per_group_per_100char": { "meaning-flip": 0.862, "entity": 2.838, "cultural": 0.424, "fidelity": 2.02 }, "per_group_fidelity": { "meaning-flip": 79.3, "entity": 49.7, "cultural": 74.3, "fidelity": 76.3 } }, "flash": { "scores_by_judge": { "deepseek_flash": [ 100, 100, 100, 100, 98, 97, 97, 100, 98, 99, 77, 98, 70, 82, 97, 92, 80, 91, 75, 90, 99, 90, 100, 90, 95, 100, 100, 98, 79, 99, 99, 100, 81, 64, 96, 98, 99, 87, 74, 74, 100, 100, 90, 90, 98, 97, 89, 100, 88, 91, 97, 98, 96, 97, 91, 89, 97, 98, 98, 100, 100, 100, 99, 95, 99, 89, 98, 85, 100, 64, 96, 100, 97, 97, 95, 79, 60, 97, 96, 97, 90, 97, 97, 62, 95, 91, 81, 100, 100, 100, 98, 100, 98, 98, 96, 100, 96, 99, 100, 92 ] }, "per_category_deductions": { "polarity": { "deepseek_flash": 105 }, "semantic-inversion": { "deepseek_flash": 40 }, "hallucination": { "deepseek_flash": 15 }, "numeric": { "deepseek_flash": 0 }, "brand": { "deepseek_flash": 140 }, "tech": { "deepseek_flash": 56 }, "culture": { "deepseek_flash": 40 }, "proper-noun": { "deepseek_flash": 24 }, "dialect": { "deepseek_flash": 0 }, "mishear-severe": { "deepseek_flash": 75 }, "content-drop": { "deepseek_flash": 0 }, "paraphrase": { "deepseek_flash": 15 }, "mishear-minor": { "deepseek_flash": 12 }, "filler": { "deepseek_flash": 175 }, "format": { "deepseek_flash": 3 }, "unscoreable": { "deepseek_flash": 0 } }, "per_tier_deductions": { "tier1": { "deepseek_flash": 160 }, "tier2": { "deepseek_flash": 260 }, "tier3": { "deepseek_flash": 102 }, "tier4": { "deepseek_flash": 178 }, "unscoreable": { "deepseek_flash": 0 } }, "raw_deductions_by_judge": { "deepseek_flash": [ 0, 0, 0, 0, 2, 3, 3, 0, 2, 1, 23, 2, 30, 18, 3, 8, 20, 9, 25, 10, 1, 10, 0, 10, 5, 0, 0, 2, 21, 1, 1, 0, 19, 36, 4, 2, 1, 13, 26, 26, 0, 0, 10, 10, 2, 3, 11, 0, 12, 9, 3, 2, 4, 3, 9, 11, 3, 2, 2, 0, 0, 0, 1, 5, 1, 11, 2, 15, 0, 36, 4, 0, 3, 3, 5, 21, 40, 3, 4, 3, 10, 3, 3, 38, 5, 9, 19, 0, 0, 0, 2, 0, 2, 2, 4, 0, 4, 1, 0, 8 ] }, "sample_details": [ { "idx": 1, "duration_s": 5, "mean_score": 100.0, "mean_raw": 0.0, "scores_per_judge": [ 100 ], "deductions_per_judge": { "deepseek_flash": [] }, "judge_disagreement": 0 }, { "idx": 2, "duration_s": 7, "mean_score": 100.0, "mean_raw": 0.0, "scores_per_judge": [ 100 ], "deductions_per_judge": { "deepseek_flash": [] }, "judge_disagreement": 0 }, { "idx": 3, "duration_s": 8, "mean_score": 100.0, "mean_raw": 0.0, "scores_per_judge": [ 100 ], "deductions_per_judge": { "deepseek_flash": [] }, "judge_disagreement": 0 }, { "idx": 4, "duration_s": 11, "mean_score": 100.0, "mean_raw": 0.0, "scores_per_judge": [ 100 ], "deductions_per_judge": { "deepseek_flash": [] }, "judge_disagreement": 0 }, { "idx": 5, "duration_s": 16, "mean_score": 98.0, "mean_raw": 2.0, "scores_per_judge": [ 98 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 6, "duration_s": 17, "mean_score": 97.0, "mean_raw": 3.0, "scores_per_judge": [ 97 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "paraphrase", "points": 3 } ] }, "judge_disagreement": 0 }, { "idx": 7, "duration_s": 20, "mean_score": 97.0, "mean_raw": 3.0, "scores_per_judge": [ 97 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 8, "duration_s": 30, "mean_score": 100.0, "mean_raw": 0.0, "scores_per_judge": [ 100 ], "deductions_per_judge": { "deepseek_flash": [] }, "judge_disagreement": 0 }, { "idx": 9, "duration_s": 32, "mean_score": 98.0, "mean_raw": 2.0, "scores_per_judge": [ 98 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 10, "duration_s": 35, "mean_score": 99.0, "mean_raw": 1.0, "scores_per_judge": [ 99 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 11, "duration_s": 40, "mean_score": 77.0, "mean_raw": 23.0, "scores_per_judge": [ 77 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "tech", "points": 8 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 12, "duration_s": 46, "mean_score": 98.0, "mean_raw": 2.0, "scores_per_judge": [ 98 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 13, "duration_s": 60, "mean_score": 70.0, "mean_raw": 30.0, "scores_per_judge": [ 70 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "brand", "points": 10 }, { "category": "brand", "points": 10 }, { "category": "tech", "points": 8 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 14, "duration_s": 64, "mean_score": 82.0, "mean_raw": 18.0, "scores_per_judge": [ 82 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 }, { "category": "proper-noun", "points": 6 }, { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 2 } ] }, "judge_disagreement": 0 }, { "idx": 15, "duration_s": 69, "mean_score": 97.0, "mean_raw": 3.0, "scores_per_judge": [ 97 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 16, "duration_s": 81, "mean_score": 92.0, "mean_raw": 8.0, "scores_per_judge": [ 92 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "proper-noun", "points": 6 }, { "category": "filler", "points": 1 }, { "category": "format", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 17, "duration_s": 100, "mean_score": 80.0, "mean_raw": 20.0, "scores_per_judge": [ 80 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "brand", "points": 10 }, { "category": "tech", "points": 8 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 18, "duration_s": 125, "mean_score": 91.0, "mean_raw": 9.0, "scores_per_judge": [ 91 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 5 }, { "category": "mishear-minor", "points": 2 }, { "category": "mishear-minor", "points": 2 } ] }, "judge_disagreement": 0 }, { "idx": 19, "duration_s": 133, "mean_score": 75.0, "mean_raw": 25.0, "scores_per_judge": [ 75 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "brand", "points": 10 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 20, "duration_s": 164, "mean_score": 90.0, "mean_raw": 10.0, "scores_per_judge": [ 90 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "format", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 21, "duration_s": 5, "mean_score": 99.0, "mean_raw": 1.0, "scores_per_judge": [ 99 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 22, "duration_s": 7, "mean_score": 90.0, "mean_raw": 10.0, "scores_per_judge": [ 90 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "brand", "points": 10 } ] }, "judge_disagreement": 0 }, { "idx": 23, "duration_s": 10, "mean_score": 100.0, "mean_raw": 0.0, "scores_per_judge": [ 100 ], "deductions_per_judge": { "deepseek_flash": [] }, "judge_disagreement": 0 }, { "idx": 24, "duration_s": 14, "mean_score": 90.0, "mean_raw": 10.0, "scores_per_judge": [ 90 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "tech", "points": 8 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 25, "duration_s": 16, "mean_score": 95.0, "mean_raw": 5.0, "scores_per_judge": [ 95 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 26, "duration_s": 23, "mean_score": 100.0, "mean_raw": 0.0, "scores_per_judge": [ 100 ], "deductions_per_judge": { "deepseek_flash": [] }, "judge_disagreement": 0 }, { "idx": 27, "duration_s": 25, "mean_score": 100.0, "mean_raw": 0.0, "scores_per_judge": [ 100 ], "deductions_per_judge": { "deepseek_flash": [] }, "judge_disagreement": 0 }, { "idx": 28, "duration_s": 30, "mean_score": 98.0, "mean_raw": 2.0, "scores_per_judge": [ 98 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 29, "duration_s": 34, "mean_score": 79.0, "mean_raw": 21.0, "scores_per_judge": [ 79 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "brand", "points": 10 }, { "category": "brand", "points": 10 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 30, "duration_s": 38, "mean_score": 99.0, "mean_raw": 1.0, "scores_per_judge": [ 99 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 31, "duration_s": 44, "mean_score": 99.0, "mean_raw": 1.0, "scores_per_judge": [ 99 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 32, "duration_s": 50, "mean_score": 100.0, "mean_raw": 0.0, "scores_per_judge": [ 100 ], "deductions_per_judge": { "deepseek_flash": [] }, "judge_disagreement": 0 }, { "idx": 33, "duration_s": 62, "mean_score": 81.0, "mean_raw": 19.0, "scores_per_judge": [ 81 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "brand", "points": 10 }, { "category": "proper-noun", "points": 6 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 34, "duration_s": 66, "mean_score": 64.0, "mean_raw": 36.0, "scores_per_judge": [ 64 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "polarity", "points": 35 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 35, "duration_s": 77, "mean_score": 96.0, "mean_raw": 4.0, "scores_per_judge": [ 96 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 36, "duration_s": 94, "mean_score": 98.0, "mean_raw": 2.0, "scores_per_judge": [ 98 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 37, "duration_s": 106, "mean_score": 99.0, "mean_raw": 1.0, "scores_per_judge": [ 99 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 38, "duration_s": 126, "mean_score": 87.0, "mean_raw": 13.0, "scores_per_judge": [ 87 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "tech", "points": 8 }, { "category": "mishear-severe", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 39, "duration_s": 158, "mean_score": 74.0, "mean_raw": 26.0, "scores_per_judge": [ 74 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "brand", "points": 10 }, { "category": "proper-noun", "points": 6 }, { "category": "tech", "points": 8 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 40, "duration_s": 279, "mean_score": 74.0, "mean_raw": 26.0, "scores_per_judge": [ 74 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 }, { "category": "mishear-severe", "points": 5 }, { "category": "culture", "points": 8 }, { "category": "paraphrase", "points": 3 }, { "category": "filler", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 41, "duration_s": 34, "mean_score": 100.0, "mean_raw": 0.0, "scores_per_judge": [ 100 ], "deductions_per_judge": { "deepseek_flash": [] }, "judge_disagreement": 0 }, { "idx": 42, "duration_s": 43, "mean_score": 100.0, "mean_raw": 0.0, "scores_per_judge": [ 100 ], "deductions_per_judge": { "deepseek_flash": [] }, "judge_disagreement": 0 }, { "idx": 43, "duration_s": 29, "mean_score": 90.0, "mean_raw": 10.0, "scores_per_judge": [ 90 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "culture", "points": 8 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 44, "duration_s": 71, "mean_score": 90.0, "mean_raw": 10.0, "scores_per_judge": [ 90 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "culture", "points": 8 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 45, "duration_s": 46, "mean_score": 98.0, "mean_raw": 2.0, "scores_per_judge": [ 98 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-minor", "points": 2 } ] }, "judge_disagreement": 0 }, { "idx": 46, "duration_s": 62, "mean_score": 97.0, "mean_raw": 3.0, "scores_per_judge": [ 97 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 47, "duration_s": 67, "mean_score": 89.0, "mean_raw": 11.0, "scores_per_judge": [ 89 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "brand", "points": 10 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 48, "duration_s": 72, "mean_score": 100.0, "mean_raw": 0.0, "scores_per_judge": [ 100 ], "deductions_per_judge": { "deepseek_flash": [] }, "judge_disagreement": 0 }, { "idx": 49, "duration_s": 87, "mean_score": 88.0, "mean_raw": 12.0, "scores_per_judge": [ 88 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "brand", "points": 10 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 50, "duration_s": 106, "mean_score": 91.0, "mean_raw": 9.0, "scores_per_judge": [ 91 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "culture", "points": 8 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 51, "duration_s": 33, "mean_score": 97.0, "mean_raw": 3.0, "scores_per_judge": [ 97 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-minor", "points": 2 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 52, "duration_s": 25, "mean_score": 98.0, "mean_raw": 2.0, "scores_per_judge": [ 98 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 53, "duration_s": 35, "mean_score": 96.0, "mean_raw": 4.0, "scores_per_judge": [ 96 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 54, "duration_s": 36, "mean_score": 97.0, "mean_raw": 3.0, "scores_per_judge": [ 97 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 55, "duration_s": 33, "mean_score": 91.0, "mean_raw": 9.0, "scores_per_judge": [ 91 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 56, "duration_s": 15, "mean_score": 89.0, "mean_raw": 11.0, "scores_per_judge": [ 89 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "brand", "points": 10 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 57, "duration_s": 38, "mean_score": 97.0, "mean_raw": 3.0, "scores_per_judge": [ 97 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 58, "duration_s": 15, "mean_score": 98.0, "mean_raw": 2.0, "scores_per_judge": [ 98 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 59, "duration_s": 28, "mean_score": 98.0, "mean_raw": 2.0, "scores_per_judge": [ 98 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 60, "duration_s": 18, "mean_score": 100.0, "mean_raw": 0.0, "scores_per_judge": [ 100 ], "deductions_per_judge": { "deepseek_flash": [] }, "judge_disagreement": 0 }, { "idx": 61, "duration_s": 19, "mean_score": 100.0, "mean_raw": 0.0, "scores_per_judge": [ 100 ], "deductions_per_judge": { "deepseek_flash": [] }, "judge_disagreement": 0 }, { "idx": 62, "duration_s": 17, "mean_score": 100.0, "mean_raw": 0.0, "scores_per_judge": [ 100 ], "deductions_per_judge": { "deepseek_flash": [] }, "judge_disagreement": 0 }, { "idx": 63, "duration_s": 21, "mean_score": 99.0, "mean_raw": 1.0, "scores_per_judge": [ 99 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 64, "duration_s": 23, "mean_score": 95.0, "mean_raw": 5.0, "scores_per_judge": [ 95 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 65, "duration_s": 20, "mean_score": 99.0, "mean_raw": 1.0, "scores_per_judge": [ 99 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 66, "duration_s": 24, "mean_score": 89.0, "mean_raw": 11.0, "scores_per_judge": [ 89 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "culture", "points": 8 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 67, "duration_s": 16, "mean_score": 98.0, "mean_raw": 2.0, "scores_per_judge": [ 98 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-minor", "points": 2 } ] }, "judge_disagreement": 0 }, { "idx": 68, "duration_s": 18, "mean_score": 85.0, "mean_raw": 15.0, "scores_per_judge": [ 85 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "brand", "points": 10 }, { "category": "mishear-severe", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 69, "duration_s": 18, "mean_score": 100.0, "mean_raw": 0.0, "scores_per_judge": [ 100 ], "deductions_per_judge": { "deepseek_flash": [] }, "judge_disagreement": 0 }, { "idx": 70, "duration_s": 17, "mean_score": 64.0, "mean_raw": 36.0, "scores_per_judge": [ 64 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "polarity", "points": 35 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 71, "duration_s": 51, "mean_score": 96.0, "mean_raw": 4.0, "scores_per_judge": [ 96 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 72, "duration_s": 54, "mean_score": 100.0, "mean_raw": 0.0, "scores_per_judge": [ 100 ], "deductions_per_judge": { "deepseek_flash": [] }, "judge_disagreement": 0 }, { "idx": 73, "duration_s": 56, "mean_score": 97.0, "mean_raw": 3.0, "scores_per_judge": [ 97 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "paraphrase", "points": 3 } ] }, "judge_disagreement": 0 }, { "idx": 74, "duration_s": 45, "mean_score": 97.0, "mean_raw": 3.0, "scores_per_judge": [ 97 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 75, "duration_s": 74, "mean_score": 95.0, "mean_raw": 5.0, "scores_per_judge": [ 95 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 76, "duration_s": 58, "mean_score": 79.0, "mean_raw": 21.0, "scores_per_judge": [ 79 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "semantic-inversion", "points": 20 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 77, "duration_s": 130, "mean_score": 60.0, "mean_raw": 40.0, "scores_per_judge": [ 60 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "semantic-inversion", "points": 20 }, { "category": "brand", "points": 10 }, { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 78, "duration_s": 61, "mean_score": 97.0, "mean_raw": 3.0, "scores_per_judge": [ 97 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 79, "duration_s": 47, "mean_score": 96.0, "mean_raw": 4.0, "scores_per_judge": [ 96 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 4 } ] }, "judge_disagreement": 0 }, { "idx": 80, "duration_s": 42, "mean_score": 97.0, "mean_raw": 3.0, "scores_per_judge": [ 97 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 81, "duration_s": 38, "mean_score": 90.0, "mean_raw": 10.0, "scores_per_judge": [ 90 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "paraphrase", "points": 3 }, { "category": "mishear-minor", "points": 2 } ] }, "judge_disagreement": 0 }, { "idx": 82, "duration_s": 38, "mean_score": 97.0, "mean_raw": 3.0, "scores_per_judge": [ 97 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "format", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 83, "duration_s": 37, "mean_score": 97.0, "mean_raw": 3.0, "scores_per_judge": [ 97 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 84, "duration_s": 36, "mean_score": 62.0, "mean_raw": 38.0, "scores_per_judge": [ 62 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "polarity", "points": 35 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 85, "duration_s": 36, "mean_score": 95.0, "mean_raw": 5.0, "scores_per_judge": [ 95 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "paraphrase", "points": 3 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 86, "duration_s": 35, "mean_score": 91.0, "mean_raw": 9.0, "scores_per_judge": [ 91 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "tech", "points": 8 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 87, "duration_s": 34, "mean_score": 81.0, "mean_raw": 19.0, "scores_per_judge": [ 81 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "hallucination", "points": 15 }, { "category": "filler", "points": 4 } ] }, "judge_disagreement": 0 }, { "idx": 88, "duration_s": 34, "mean_score": 100.0, "mean_raw": 0.0, "scores_per_judge": [ 100 ], "deductions_per_judge": { "deepseek_flash": [] }, "judge_disagreement": 0 }, { "idx": 89, "duration_s": 34, "mean_score": 100.0, "mean_raw": 0.0, "scores_per_judge": [ 100 ], "deductions_per_judge": { "deepseek_flash": [] }, "judge_disagreement": 0 }, { "idx": 90, "duration_s": 33, "mean_score": 100.0, "mean_raw": 0.0, "scores_per_judge": [ 100 ], "deductions_per_judge": { "deepseek_flash": [] }, "judge_disagreement": 0 }, { "idx": 91, "duration_s": 33, "mean_score": 98.0, "mean_raw": 2.0, "scores_per_judge": [ 98 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 92, "duration_s": 32, "mean_score": 100.0, "mean_raw": 0.0, "scores_per_judge": [ 100 ], "deductions_per_judge": { "deepseek_flash": [] }, "judge_disagreement": 0 }, { "idx": 93, "duration_s": 32, "mean_score": 98.0, "mean_raw": 2.0, "scores_per_judge": [ 98 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 94, "duration_s": 32, "mean_score": 98.0, "mean_raw": 2.0, "scores_per_judge": [ 98 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 95, "duration_s": 32, "mean_score": 96.0, "mean_raw": 4.0, "scores_per_judge": [ 96 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 96, "duration_s": 31, "mean_score": 100.0, "mean_raw": 0.0, "scores_per_judge": [ 100 ], "deductions_per_judge": { "deepseek_flash": [] }, "judge_disagreement": 0 }, { "idx": 97, "duration_s": 30, "mean_score": 96.0, "mean_raw": 4.0, "scores_per_judge": [ 96 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 2 } ] }, "judge_disagreement": 0 }, { "idx": 98, "duration_s": 30, "mean_score": 99.0, "mean_raw": 1.0, "scores_per_judge": [ 99 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 99, "duration_s": 30, "mean_score": 100.0, "mean_raw": 0.0, "scores_per_judge": [ 100 ], "deductions_per_judge": { "deepseek_flash": [] }, "judge_disagreement": 0 }, { "idx": 100, "duration_s": 30, "mean_score": 92.0, "mean_raw": 8.0, "scores_per_judge": [ 92 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 3 } ] }, "judge_disagreement": 0 } ], "total_deductions": 700, "avg_deductions_per_sample": 7.0, "mean_clamped_score": 93.0, "n_judge_sample_pairs": 100, "gt_chars": 22625, "ded_per_100char": 3.09, "fidelity_score": 84.5, "overall_score": 93.0, "overall_raw_deductions": 7.0, "score_per_judge": { "deepseek_flash": 93.0 }, "per_group_deductions": { "meaning-flip": { "deepseek_flash": 160 }, "entity": { "deepseek_flash": 220 }, "cultural": { "deepseek_flash": 40 }, "fidelity": { "deepseek_flash": 280 } }, "per_bucket": { "general": { "n_samples": 40, "mean_raw": 8.4, "mean_score": 91.6, "total_raw": 337.0, "idxs": [ 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40 ], "worst_idx": 34 }, "idiom-test": { "n_samples": 5, "mean_raw": 4.4, "mean_score": 95.6, "total_raw": 22.0, "idxs": [ 41, 42, 43, 44, 45 ], "worst_idx": 43 }, "long-mixed": { "n_samples": 5, "mean_raw": 7.0, "mean_score": 93.0, "total_raw": 35.0, "idxs": [ 46, 47, 48, 49, 50 ], "worst_idx": 49 } }, "per_group_per_100char": { "meaning-flip": 0.707, "entity": 0.972, "cultural": 0.177, "fidelity": 1.238 }, "per_group_fidelity": { "meaning-flip": 83.0, "entity": 82.8, "cultural": 89.3, "fidelity": 85.5 } }, "funasr_nano": { "scores_by_judge": { "deepseek_flash": [ 100, 100, 100, 100, 97, 0, 90, 99, 98, 99, 73, 97, 70, 86, null, 65, 36, 66, 0, 0, 99, 90, 100, 90, 92, 100, 100, 94, 90, 62, 55, 99, 82, 58, 74, 89, 72, 64, 0, 0, 97, 99, 89, 76, 56, 100, 81, 78, 89, 69, 90, 96, 98, 99, 92, 79, 90, 98, 88, 63, 100, 100, 98, 96, 100, 91, 99, 89, 100, 55, 88, 92, 86, 97, 87, 72, 62, 90, 95, 98, 94, 81, 93, 62, 91, 91, 91, 100, 100, 98, 99, 100, 97, 92, 95, 100, 96, 99, 100, 89 ] }, "per_category_deductions": { "polarity": { "deepseek_flash": 245 }, "semantic-inversion": { "deepseek_flash": 40 }, "hallucination": { "deepseek_flash": 75 }, "numeric": { "deepseek_flash": 10 }, "brand": { "deepseek_flash": 130 }, "tech": { "deepseek_flash": 160 }, "culture": { "deepseek_flash": 48 }, "proper-noun": { "deepseek_flash": 48 }, "dialect": { "deepseek_flash": 0 }, "mishear-severe": { "deepseek_flash": 125 }, "content-drop": { "deepseek_flash": 60 }, "paraphrase": { "deepseek_flash": 18 }, "mishear-minor": { "deepseek_flash": 14 }, "filler": { "deepseek_flash": 150 }, "format": { "deepseek_flash": 1 }, "unscoreable": { "deepseek_flash": 500 } }, "per_tier_deductions": { "tier1": { "deepseek_flash": 370 }, "tier2": { "deepseek_flash": 386 }, "tier3": { "deepseek_flash": 217 }, "tier4": { "deepseek_flash": 151 }, "unscoreable": { "deepseek_flash": 500 } }, "raw_deductions_by_judge": { "deepseek_flash": [ 0, 0, 0, 0, 3, 100, 10, 1, 2, 1, 27, 3, 30, 14, null, 35, 64, 34, 100, 100, 1, 10, 0, 10, 8, 0, 0, 6, 10, 38, 45, 1, 18, 42, 26, 11, 28, 36, 100, 100, 3, 1, 11, 24, 44, 0, 19, 22, 11, 31, 10, 4, 2, 1, 8, 21, 10, 2, 12, 37, 0, 0, 2, 4, 0, 9, 1, 11, 0, 45, 12, 8, 14, 3, 13, 28, 38, 10, 5, 2, 6, 19, 7, 38, 9, 9, 9, 0, 0, 2, 1, 0, 3, 8, 5, 0, 4, 1, 0, 11 ] }, "sample_details": [ { "idx": 1, "duration_s": 5, "mean_score": 100.0, "mean_raw": 0.0, "scores_per_judge": [ 100 ], "deductions_per_judge": { "deepseek_flash": [] }, "judge_disagreement": 0 }, { "idx": 2, "duration_s": 7, "mean_score": 100.0, "mean_raw": 0.0, "scores_per_judge": [ 100 ], "deductions_per_judge": { "deepseek_flash": [] }, "judge_disagreement": 0 }, { "idx": 3, "duration_s": 8, "mean_score": 100.0, "mean_raw": 0.0, "scores_per_judge": [ 100 ], "deductions_per_judge": { "deepseek_flash": [] }, "judge_disagreement": 0 }, { "idx": 4, "duration_s": 11, "mean_score": 100.0, "mean_raw": 0.0, "scores_per_judge": [ 100 ], "deductions_per_judge": { "deepseek_flash": [] }, "judge_disagreement": 0 }, { "idx": 5, "duration_s": 16, "mean_score": 97.0, "mean_raw": 3.0, "scores_per_judge": [ 97 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 6, "duration_s": 17, "mean_score": 0.0, "mean_raw": 100.0, "scores_per_judge": [ 0 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "unscoreable", "points": 100 } ] }, "judge_disagreement": 0 }, { "idx": 7, "duration_s": 20, "mean_score": 90.0, "mean_raw": 10.0, "scores_per_judge": [ 90 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "tech", "points": 8 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 8, "duration_s": 30, "mean_score": 99.0, "mean_raw": 1.0, "scores_per_judge": [ 99 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 9, "duration_s": 32, "mean_score": 98.0, "mean_raw": 2.0, "scores_per_judge": [ 98 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 10, "duration_s": 35, "mean_score": 99.0, "mean_raw": 1.0, "scores_per_judge": [ 99 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 11, "duration_s": 40, "mean_score": 73.0, "mean_raw": 27.0, "scores_per_judge": [ 73 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "tech", "points": 8 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 4 } ] }, "judge_disagreement": 0 }, { "idx": 12, "duration_s": 46, "mean_score": 97.0, "mean_raw": 3.0, "scores_per_judge": [ 97 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 13, "duration_s": 60, "mean_score": 70.0, "mean_raw": 30.0, "scores_per_judge": [ 70 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "brand", "points": 10 }, { "category": "brand", "points": 10 }, { "category": "tech", "points": 8 }, { "category": "filler", "points": 2 } ] }, "judge_disagreement": 0 }, { "idx": 14, "duration_s": 64, "mean_score": 86.0, "mean_raw": 14.0, "scores_per_judge": [ 86 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "proper-noun", "points": 6 }, { "category": "proper-noun", "points": 6 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 16, "duration_s": 81, "mean_score": 65.0, "mean_raw": 35.0, "scores_per_judge": [ 65 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "tech", "points": 8 }, { "category": "proper-noun", "points": 6 }, { "category": "tech", "points": 8 }, { "category": "tech", "points": 8 }, { "category": "content-drop", "points": 3 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 17, "duration_s": 100, "mean_score": 36.0, "mean_raw": 64.0, "scores_per_judge": [ 36 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "brand", "points": 10 }, { "category": "hallucination", "points": 15 }, { "category": "content-drop", "points": 3 }, { "category": "content-drop", "points": 3 }, { "category": "semantic-inversion", "points": 20 }, { "category": "tech", "points": 8 }, { "category": "filler", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 18, "duration_s": 125, "mean_score": 66.0, "mean_raw": 34.0, "scores_per_judge": [ 66 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "hallucination", "points": 15 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-severe", "points": 5 }, { "category": "content-drop", "points": 3 }, { "category": "content-drop", "points": 3 }, { "category": "paraphrase", "points": 3 } ] }, "judge_disagreement": 0 }, { "idx": 19, "duration_s": 133, "mean_score": 0.0, "mean_raw": 100.0, "scores_per_judge": [ 0 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "unscoreable", "points": 100 } ] }, "judge_disagreement": 0 }, { "idx": 20, "duration_s": 164, "mean_score": 0.0, "mean_raw": 100.0, "scores_per_judge": [ 0 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "unscoreable", "points": 100 } ] }, "judge_disagreement": 0 }, { "idx": 21, "duration_s": 5, "mean_score": 99.0, "mean_raw": 1.0, "scores_per_judge": [ 99 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "format", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 22, "duration_s": 7, "mean_score": 90.0, "mean_raw": 10.0, "scores_per_judge": [ 90 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "brand", "points": 10 } ] }, "judge_disagreement": 0 }, { "idx": 23, "duration_s": 10, "mean_score": 100.0, "mean_raw": 0.0, "scores_per_judge": [ 100 ], "deductions_per_judge": { "deepseek_flash": [] }, "judge_disagreement": 0 }, { "idx": 24, "duration_s": 14, "mean_score": 90.0, "mean_raw": 10.0, "scores_per_judge": [ 90 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "tech", "points": 8 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 25, "duration_s": 16, "mean_score": 92.0, "mean_raw": 8.0, "scores_per_judge": [ 92 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "tech", "points": 8 } ] }, "judge_disagreement": 0 }, { "idx": 26, "duration_s": 23, "mean_score": 100.0, "mean_raw": 0.0, "scores_per_judge": [ 100 ], "deductions_per_judge": { "deepseek_flash": [] }, "judge_disagreement": 0 }, { "idx": 27, "duration_s": 25, "mean_score": 100.0, "mean_raw": 0.0, "scores_per_judge": [ 100 ], "deductions_per_judge": { "deepseek_flash": [] }, "judge_disagreement": 0 }, { "idx": 28, "duration_s": 30, "mean_score": 94.0, "mean_raw": 6.0, "scores_per_judge": [ 94 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 29, "duration_s": 34, "mean_score": 90.0, "mean_raw": 10.0, "scores_per_judge": [ 90 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "brand", "points": 10 } ] }, "judge_disagreement": 0 }, { "idx": 30, "duration_s": 38, "mean_score": 62.0, "mean_raw": 38.0, "scores_per_judge": [ 62 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "polarity", "points": 35 }, { "category": "mishear-minor", "points": 2 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 31, "duration_s": 44, "mean_score": 55.0, "mean_raw": 45.0, "scores_per_judge": [ 55 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "polarity", "points": 35 }, { "category": "tech", "points": 8 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 32, "duration_s": 50, "mean_score": 99.0, "mean_raw": 1.0, "scores_per_judge": [ 99 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 33, "duration_s": 62, "mean_score": 82.0, "mean_raw": 18.0, "scores_per_judge": [ 82 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "brand", "points": 10 }, { "category": "proper-noun", "points": 6 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 34, "duration_s": 66, "mean_score": 58.0, "mean_raw": 42.0, "scores_per_judge": [ 58 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "polarity", "points": 35 }, { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 2 } ] }, "judge_disagreement": 0 }, { "idx": 35, "duration_s": 77, "mean_score": 74.0, "mean_raw": 26.0, "scores_per_judge": [ 74 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "tech", "points": 8 }, { "category": "tech", "points": 8 }, { "category": "content-drop", "points": 3 }, { "category": "mishear-minor", "points": 2 }, { "category": "filler", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 36, "duration_s": 94, "mean_score": 89.0, "mean_raw": 11.0, "scores_per_judge": [ 89 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "content-drop", "points": 3 }, { "category": "content-drop", "points": 3 }, { "category": "content-drop", "points": 3 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 37, "duration_s": 106, "mean_score": 72.0, "mean_raw": 28.0, "scores_per_judge": [ 72 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "hallucination", "points": 15 }, { "category": "content-drop", "points": 3 }, { "category": "content-drop", "points": 3 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-minor", "points": 2 } ] }, "judge_disagreement": 0 }, { "idx": 38, "duration_s": 126, "mean_score": 64.0, "mean_raw": 36.0, "scores_per_judge": [ 64 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "hallucination", "points": 15 }, { "category": "tech", "points": 8 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-severe", "points": 5 }, { "category": "content-drop", "points": 3 } ] }, "judge_disagreement": 0 }, { "idx": 39, "duration_s": 158, "mean_score": 0.0, "mean_raw": 100.0, "scores_per_judge": [ 0 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "unscoreable", "points": 100 } ] }, "judge_disagreement": 0 }, { "idx": 40, "duration_s": 279, "mean_score": 0.0, "mean_raw": 100.0, "scores_per_judge": [ 0 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "unscoreable", "points": 100 } ] }, "judge_disagreement": 0 }, { "idx": 41, "duration_s": 34, "mean_score": 97.0, "mean_raw": 3.0, "scores_per_judge": [ 97 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 42, "duration_s": 43, "mean_score": 99.0, "mean_raw": 1.0, "scores_per_judge": [ 99 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 43, "duration_s": 29, "mean_score": 89.0, "mean_raw": 11.0, "scores_per_judge": [ 89 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "culture", "points": 8 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 44, "duration_s": 71, "mean_score": 76.0, "mean_raw": 24.0, "scores_per_judge": [ 76 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "proper-noun", "points": 6 }, { "category": "mishear-severe", "points": 5 }, { "category": "culture", "points": 8 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 45, "duration_s": 46, "mean_score": 56.0, "mean_raw": 44.0, "scores_per_judge": [ 56 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "polarity", "points": 35 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-minor", "points": 2 }, { "category": "mishear-minor", "points": 2 } ] }, "judge_disagreement": 0 }, { "idx": 46, "duration_s": 62, "mean_score": 100.0, "mean_raw": 0.0, "scores_per_judge": [ 100 ], "deductions_per_judge": { "deepseek_flash": [] }, "judge_disagreement": 0 }, { "idx": 47, "duration_s": 67, "mean_score": 81.0, "mean_raw": 19.0, "scores_per_judge": [ 81 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "brand", "points": 10 }, { "category": "tech", "points": 8 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 48, "duration_s": 72, "mean_score": 78.0, "mean_raw": 22.0, "scores_per_judge": [ 78 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "brand", "points": 10 }, { "category": "tech", "points": 8 }, { "category": "content-drop", "points": 3 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 49, "duration_s": 87, "mean_score": 89.0, "mean_raw": 11.0, "scores_per_judge": [ 89 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "tech", "points": 8 }, { "category": "content-drop", "points": 3 } ] }, "judge_disagreement": 0 }, { "idx": 50, "duration_s": 106, "mean_score": 69.0, "mean_raw": 31.0, "scores_per_judge": [ 69 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "culture", "points": 8 }, { "category": "culture", "points": 8 }, { "category": "content-drop", "points": 3 }, { "category": "content-drop", "points": 3 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-minor", "points": 2 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 51, "duration_s": 33, "mean_score": 90.0, "mean_raw": 10.0, "scores_per_judge": [ 90 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "culture", "points": 8 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 52, "duration_s": 25, "mean_score": 96.0, "mean_raw": 4.0, "scores_per_judge": [ 96 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 53, "duration_s": 35, "mean_score": 98.0, "mean_raw": 2.0, "scores_per_judge": [ 98 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 54, "duration_s": 36, "mean_score": 99.0, "mean_raw": 1.0, "scores_per_judge": [ 99 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 55, "duration_s": 33, "mean_score": 92.0, "mean_raw": 8.0, "scores_per_judge": [ 92 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "proper-noun", "points": 6 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 56, "duration_s": 15, "mean_score": 79.0, "mean_raw": 21.0, "scores_per_judge": [ 79 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "brand", "points": 10 }, { "category": "brand", "points": 10 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 57, "duration_s": 38, "mean_score": 90.0, "mean_raw": 10.0, "scores_per_judge": [ 90 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "tech", "points": 8 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 58, "duration_s": 15, "mean_score": 98.0, "mean_raw": 2.0, "scores_per_judge": [ 98 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 59, "duration_s": 28, "mean_score": 88.0, "mean_raw": 12.0, "scores_per_judge": [ 88 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 }, { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 60, "duration_s": 18, "mean_score": 63.0, "mean_raw": 37.0, "scores_per_judge": [ 63 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "polarity", "points": 35 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 61, "duration_s": 19, "mean_score": 100.0, "mean_raw": 0.0, "scores_per_judge": [ 100 ], "deductions_per_judge": { "deepseek_flash": [] }, "judge_disagreement": 0 }, { "idx": 62, "duration_s": 17, "mean_score": 100.0, "mean_raw": 0.0, "scores_per_judge": [ 100 ], "deductions_per_judge": { "deepseek_flash": [] }, "judge_disagreement": 0 }, { "idx": 63, "duration_s": 21, "mean_score": 98.0, "mean_raw": 2.0, "scores_per_judge": [ 98 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 64, "duration_s": 23, "mean_score": 96.0, "mean_raw": 4.0, "scores_per_judge": [ 96 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 4 } ] }, "judge_disagreement": 0 }, { "idx": 65, "duration_s": 20, "mean_score": 100.0, "mean_raw": 0.0, "scores_per_judge": [ 100 ], "deductions_per_judge": { "deepseek_flash": [] }, "judge_disagreement": 0 }, { "idx": 66, "duration_s": 24, "mean_score": 91.0, "mean_raw": 9.0, "scores_per_judge": [ 91 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "culture", "points": 8 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 67, "duration_s": 16, "mean_score": 99.0, "mean_raw": 1.0, "scores_per_judge": [ 99 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 68, "duration_s": 18, "mean_score": 89.0, "mean_raw": 11.0, "scores_per_judge": [ 89 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "brand", "points": 10 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 69, "duration_s": 18, "mean_score": 100.0, "mean_raw": 0.0, "scores_per_judge": [ 100 ], "deductions_per_judge": { "deepseek_flash": [] }, "judge_disagreement": 0 }, { "idx": 70, "duration_s": 17, "mean_score": 55.0, "mean_raw": 45.0, "scores_per_judge": [ 55 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "polarity", "points": 35 }, { "category": "tech", "points": 8 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 71, "duration_s": 51, "mean_score": 88.0, "mean_raw": 12.0, "scores_per_judge": [ 88 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "brand", "points": 10 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 72, "duration_s": 54, "mean_score": 92.0, "mean_raw": 8.0, "scores_per_judge": [ 92 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "mishear-severe", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 73, "duration_s": 56, "mean_score": 86.0, "mean_raw": 14.0, "scores_per_judge": [ 86 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 }, { "category": "paraphrase", "points": 3 }, { "category": "proper-noun", "points": 6 } ] }, "judge_disagreement": 0 }, { "idx": 74, "duration_s": 45, "mean_score": 97.0, "mean_raw": 3.0, "scores_per_judge": [ 97 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 75, "duration_s": 74, "mean_score": 87.0, "mean_raw": 13.0, "scores_per_judge": [ 87 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 }, { "category": "paraphrase", "points": 3 }, { "category": "filler", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 76, "duration_s": 58, "mean_score": 72.0, "mean_raw": 28.0, "scores_per_judge": [ 72 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "semantic-inversion", "points": 20 }, { "category": "proper-noun", "points": 6 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 77, "duration_s": 130, "mean_score": 62.0, "mean_raw": 38.0, "scores_per_judge": [ 62 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "brand", "points": 10 }, { "category": "hallucination", "points": 15 }, { "category": "content-drop", "points": 3 }, { "category": "content-drop", "points": 3 }, { "category": "content-drop", "points": 3 }, { "category": "content-drop", "points": 3 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 78, "duration_s": 61, "mean_score": 90.0, "mean_raw": 10.0, "scores_per_judge": [ 90 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "mishear-severe", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 79, "duration_s": 47, "mean_score": 95.0, "mean_raw": 5.0, "scores_per_judge": [ 95 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "paraphrase", "points": 3 } ] }, "judge_disagreement": 0 }, { "idx": 80, "duration_s": 42, "mean_score": 98.0, "mean_raw": 2.0, "scores_per_judge": [ 98 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 81, "duration_s": 38, "mean_score": 94.0, "mean_raw": 6.0, "scores_per_judge": [ 94 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "paraphrase", "points": 3 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 82, "duration_s": 38, "mean_score": 81.0, "mean_raw": 19.0, "scores_per_judge": [ 81 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "numeric", "points": 10 }, { "category": "tech", "points": 8 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 83, "duration_s": 37, "mean_score": 93.0, "mean_raw": 7.0, "scores_per_judge": [ 93 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 84, "duration_s": 36, "mean_score": 62.0, "mean_raw": 38.0, "scores_per_judge": [ 62 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "polarity", "points": 35 }, { "category": "filler", "points": 3 } ] }, "judge_disagreement": 0 }, { "idx": 85, "duration_s": 36, "mean_score": 91.0, "mean_raw": 9.0, "scores_per_judge": [ 91 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 }, { "category": "mishear-minor", "points": 2 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 86, "duration_s": 35, "mean_score": 91.0, "mean_raw": 9.0, "scores_per_judge": [ 91 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "tech", "points": 8 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 87, "duration_s": 34, "mean_score": 91.0, "mean_raw": 9.0, "scores_per_judge": [ 91 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "mishear-severe", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 88, "duration_s": 34, "mean_score": 100.0, "mean_raw": 0.0, "scores_per_judge": [ 100 ], "deductions_per_judge": { "deepseek_flash": [] }, "judge_disagreement": 0 }, { "idx": 89, "duration_s": 34, "mean_score": 100.0, "mean_raw": 0.0, "scores_per_judge": [ 100 ], "deductions_per_judge": { "deepseek_flash": [] }, "judge_disagreement": 0 }, { "idx": 90, "duration_s": 33, "mean_score": 98.0, "mean_raw": 2.0, "scores_per_judge": [ 98 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 2 } ] }, "judge_disagreement": 0 }, { "idx": 91, "duration_s": 33, "mean_score": 99.0, "mean_raw": 1.0, "scores_per_judge": [ 99 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 92, "duration_s": 32, "mean_score": 100.0, "mean_raw": 0.0, "scores_per_judge": [ 100 ], "deductions_per_judge": { "deepseek_flash": [] }, "judge_disagreement": 0 }, { "idx": 93, "duration_s": 32, "mean_score": 97.0, "mean_raw": 3.0, "scores_per_judge": [ 97 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 3 } ] }, "judge_disagreement": 0 }, { "idx": 94, "duration_s": 32, "mean_score": 92.0, "mean_raw": 8.0, "scores_per_judge": [ 92 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 95, "duration_s": 32, "mean_score": 95.0, "mean_raw": 5.0, "scores_per_judge": [ 95 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "paraphrase", "points": 3 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 96, "duration_s": 31, "mean_score": 100.0, "mean_raw": 0.0, "scores_per_judge": [ 100 ], "deductions_per_judge": { "deepseek_flash": [] }, "judge_disagreement": 0 }, { "idx": 97, "duration_s": 30, "mean_score": 96.0, "mean_raw": 4.0, "scores_per_judge": [ 96 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 98, "duration_s": 30, "mean_score": 99.0, "mean_raw": 1.0, "scores_per_judge": [ 99 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 99, "duration_s": 30, "mean_score": 100.0, "mean_raw": 0.0, "scores_per_judge": [ 100 ], "deductions_per_judge": { "deepseek_flash": [] }, "judge_disagreement": 0 }, { "idx": 100, "duration_s": 30, "mean_score": 89.0, "mean_raw": 11.0, "scores_per_judge": [ 89 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 }, { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 } ], "total_deductions": 1624, "avg_deductions_per_sample": 16.4, "mean_clamped_score": 83.6, "n_judge_sample_pairs": 99, "gt_chars": 22264, "ded_per_100char": 7.29, "fidelity_score": 63.5, "overall_score": 83.6, "overall_raw_deductions": 16.4, "score_per_judge": { "deepseek_flash": 83.6 }, "per_group_deductions": { "meaning-flip": { "deepseek_flash": 370 }, "entity": { "deepseek_flash": 338 }, "cultural": { "deepseek_flash": 48 }, "fidelity": { "deepseek_flash": 368 } }, "per_bucket": { "general": { "n_samples": 39, "mean_raw": 26.0, "mean_score": 74.0, "total_raw": 1014.0, "idxs": [ 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40 ], "worst_idx": 6 }, "idiom-test": { "n_samples": 5, "mean_raw": 16.6, "mean_score": 83.4, "total_raw": 83.0, "idxs": [ 41, 42, 43, 44, 45 ], "worst_idx": 45 }, "long-mixed": { "n_samples": 5, "mean_raw": 16.6, "mean_score": 83.4, "total_raw": 83.0, "idxs": [ 46, 47, 48, 49, 50 ], "worst_idx": 50 } }, "per_group_per_100char": { "meaning-flip": 1.662, "entity": 1.518, "cultural": 0.216, "fidelity": 1.653 }, "per_group_fidelity": { "meaning-flip": 60.2, "entity": 73.1, "cultural": 87.0, "fidelity": 80.6 } }, "gemini_3_5_flash_lite_stt": { "scores_by_judge": { "deepseek_flash": [ 85, 99, 95, 91, 97, 91, 91, 92, 95, 91, 66, 87, 60, 62, 95, 78, 80, 55, 39, 0, 100, 82, 99, 77, 87, 99, 93, 82, 84, 87, 87, 56, 61, 45, 79, 55, 78, 56, 35, 0, 78, 61, 61, 54, 0, 79, 84, 41, 78, 46, 39, 47, 91, 93, 80, 89, 56, 77, 85, 52, 84, 100, 96, 88, 99, 91, 94, 86, 100, 50, 49, 88, 61, 90, 79, 66, 49, 90, 91, 72, 80, 95, 90, 96, 89, 70, 85, 97, 88, 77, 80, 94, 96, 91, 59, 99, 97, 97, 99, 92 ] }, "per_category_deductions": { "polarity": { "deepseek_flash": 315 }, "semantic-inversion": { "deepseek_flash": 100 }, "hallucination": { "deepseek_flash": 30 }, "numeric": { "deepseek_flash": 50 }, "brand": { "deepseek_flash": 170 }, "tech": { "deepseek_flash": 216 }, "culture": { "deepseek_flash": 192 }, "proper-noun": { "deepseek_flash": 66 }, "dialect": { "deepseek_flash": 25 }, "mishear-severe": { "deepseek_flash": 715 }, "content-drop": { "deepseek_flash": 36 }, "paraphrase": { "deepseek_flash": 42 }, "mishear-minor": { "deepseek_flash": 108 }, "filler": { "deepseek_flash": 295 }, "format": { "deepseek_flash": 0 }, "unscoreable": { "deepseek_flash": 0 } }, "per_tier_deductions": { "tier1": { "deepseek_flash": 495 }, "tier2": { "deepseek_flash": 669 }, "tier3": { "deepseek_flash": 901 }, "tier4": { "deepseek_flash": 295 }, "unscoreable": { "deepseek_flash": 0 } }, "raw_deductions_by_judge": { "deepseek_flash": [ 15, 1, 5, 9, 3, 9, 9, 8, 5, 9, 34, 13, 40, 38, 5, 22, 20, 45, 61, 124, 0, 18, 1, 23, 13, 1, 7, 18, 16, 13, 13, 44, 39, 55, 21, 45, 22, 44, 65, 112, 22, 39, 39, 46, 100, 21, 16, 59, 22, 54, 61, 53, 9, 7, 20, 11, 44, 23, 15, 48, 16, 0, 4, 12, 1, 9, 6, 14, 0, 50, 51, 12, 39, 10, 21, 34, 51, 10, 9, 28, 20, 5, 10, 4, 11, 30, 15, 3, 12, 23, 20, 6, 4, 9, 41, 1, 3, 3, 1, 8 ] }, "sample_details": [ { "idx": 1, "duration_s": 5, "mean_score": 85.0, "mean_raw": 15.0, "scores_per_judge": [ 85 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "hallucination", "points": 15 } ] }, "judge_disagreement": 0 }, { "idx": 2, "duration_s": 7, "mean_score": 99.0, "mean_raw": 1.0, "scores_per_judge": [ 99 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 3, "duration_s": 8, "mean_score": 95.0, "mean_raw": 5.0, "scores_per_judge": [ 95 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 4, "duration_s": 11, "mean_score": 91.0, "mean_raw": 9.0, "scores_per_judge": [ 91 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "culture", "points": 8 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 5, "duration_s": 16, "mean_score": 97.0, "mean_raw": 3.0, "scores_per_judge": [ 97 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 6, "duration_s": 17, "mean_score": 91.0, "mean_raw": 9.0, "scores_per_judge": [ 91 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-minor", "points": 2 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-minor", "points": 2 } ] }, "judge_disagreement": 0 }, { "idx": 7, "duration_s": 20, "mean_score": 91.0, "mean_raw": 9.0, "scores_per_judge": [ 91 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 }, { "category": "content-drop", "points": 3 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 8, "duration_s": 30, "mean_score": 92.0, "mean_raw": 8.0, "scores_per_judge": [ 92 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "tech", "points": 8 } ] }, "judge_disagreement": 0 }, { "idx": 9, "duration_s": 32, "mean_score": 95.0, "mean_raw": 5.0, "scores_per_judge": [ 95 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-minor", "points": 2 }, { "category": "mishear-minor", "points": 2 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 10, "duration_s": 35, "mean_score": 91.0, "mean_raw": 9.0, "scores_per_judge": [ 91 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 11, "duration_s": 40, "mean_score": 66.0, "mean_raw": 34.0, "scores_per_judge": [ 66 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "tech", "points": 8 }, { "category": "tech", "points": 8 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-severe", "points": 5 }, { "category": "paraphrase", "points": 3 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 12, "duration_s": 46, "mean_score": 87.0, "mean_raw": 13.0, "scores_per_judge": [ 87 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 }, { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 13, "duration_s": 60, "mean_score": 60.0, "mean_raw": 40.0, "scores_per_judge": [ 60 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "brand", "points": 10 }, { "category": "tech", "points": 8 }, { "category": "tech", "points": 8 }, { "category": "tech", "points": 8 }, { "category": "mishear-minor", "points": 2 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 14, "duration_s": 64, "mean_score": 62.0, "mean_raw": 38.0, "scores_per_judge": [ 62 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 1 }, { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 1 }, { "category": "culture", "points": 8 }, { "category": "filler", "points": 1 }, { "category": "mishear-severe", "points": 5 }, { "category": "paraphrase", "points": 3 }, { "category": "content-drop", "points": 3 }, { "category": "filler", "points": 1 }, { "category": "mishear-severe", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 15, "duration_s": 69, "mean_score": 95.0, "mean_raw": 5.0, "scores_per_judge": [ 95 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 16, "duration_s": 81, "mean_score": 78.0, "mean_raw": 22.0, "scores_per_judge": [ 78 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "content-drop", "points": 3 }, { "category": "proper-noun", "points": 6 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 17, "duration_s": 100, "mean_score": 80.0, "mean_raw": 20.0, "scores_per_judge": [ 80 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "brand", "points": 10 }, { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 18, "duration_s": 125, "mean_score": 55.0, "mean_raw": 45.0, "scores_per_judge": [ 55 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-severe", "points": 5 }, { "category": "brand", "points": 10 }, { "category": "filler", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 19, "duration_s": 133, "mean_score": 39.0, "mean_raw": 61.0, "scores_per_judge": [ 39 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 }, { "category": "brand", "points": 10 }, { "category": "tech", "points": 8 }, { "category": "mishear-severe", "points": 5 }, { "category": "brand", "points": 10 }, { "category": "mishear-severe", "points": 5 }, { "category": "brand", "points": 10 }, { "category": "content-drop", "points": 3 }, { "category": "filler", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 20, "duration_s": 164, "mean_score": 0.0, "mean_raw": 124.0, "scores_per_judge": [ 0 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "polarity", "points": 35 }, { "category": "semantic-inversion", "points": 20 }, { "category": "numeric", "points": 10 }, { "category": "culture", "points": 8 }, { "category": "culture", "points": 8 }, { "category": "proper-noun", "points": 6 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-minor", "points": 2 }, { "category": "filler", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 21, "duration_s": 5, "mean_score": 100.0, "mean_raw": 0.0, "scores_per_judge": [ 100 ], "deductions_per_judge": { "deepseek_flash": [] }, "judge_disagreement": 0 }, { "idx": 22, "duration_s": 7, "mean_score": 82.0, "mean_raw": 18.0, "scores_per_judge": [ 82 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "brand", "points": 10 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-minor", "points": 2 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 23, "duration_s": 10, "mean_score": 99.0, "mean_raw": 1.0, "scores_per_judge": [ 99 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 24, "duration_s": 14, "mean_score": 77.0, "mean_raw": 23.0, "scores_per_judge": [ 77 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "tech", "points": 8 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 25, "duration_s": 16, "mean_score": 87.0, "mean_raw": 13.0, "scores_per_judge": [ 87 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 }, { "category": "mishear-minor", "points": 2 }, { "category": "mishear-minor", "points": 2 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 26, "duration_s": 23, "mean_score": 99.0, "mean_raw": 1.0, "scores_per_judge": [ 99 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 27, "duration_s": 25, "mean_score": 93.0, "mean_raw": 7.0, "scores_per_judge": [ 93 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 28, "duration_s": 30, "mean_score": 82.0, "mean_raw": 18.0, "scores_per_judge": [ 82 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-minor", "points": 2 }, { "category": "mishear-minor", "points": 2 }, { "category": "filler", "points": 4 } ] }, "judge_disagreement": 0 }, { "idx": 29, "duration_s": 34, "mean_score": 84.0, "mean_raw": 16.0, "scores_per_judge": [ 84 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "brand", "points": 10 }, { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 30, "duration_s": 38, "mean_score": 87.0, "mean_raw": 13.0, "scores_per_judge": [ 87 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 }, { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 3 } ] }, "judge_disagreement": 0 }, { "idx": 31, "duration_s": 44, "mean_score": 87.0, "mean_raw": 13.0, "scores_per_judge": [ 87 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 }, { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 32, "duration_s": 50, "mean_score": 56.0, "mean_raw": 44.0, "scores_per_judge": [ 56 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "tech", "points": 8 }, { "category": "tech", "points": 8 }, { "category": "tech", "points": 8 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 33, "duration_s": 62, "mean_score": 61.0, "mean_raw": 39.0, "scores_per_judge": [ 61 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-minor", "points": 2 }, { "category": "paraphrase", "points": 3 }, { "category": "brand", "points": 10 }, { "category": "mishear-severe", "points": 5 }, { "category": "paraphrase", "points": 3 }, { "category": "proper-noun", "points": 6 }, { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 34, "duration_s": 66, "mean_score": 45.0, "mean_raw": 55.0, "scores_per_judge": [ 45 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "polarity", "points": 35 }, { "category": "culture", "points": 8 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-minor", "points": 2 }, { "category": "filler", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 35, "duration_s": 77, "mean_score": 79.0, "mean_raw": 21.0, "scores_per_judge": [ 79 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 }, { "category": "mishear-severe", "points": 5 }, { "category": "content-drop", "points": 3 }, { "category": "content-drop", "points": 3 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 36, "duration_s": 94, "mean_score": 55.0, "mean_raw": 45.0, "scores_per_judge": [ 55 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "polarity", "points": 35 }, { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 37, "duration_s": 106, "mean_score": 78.0, "mean_raw": 22.0, "scores_per_judge": [ 78 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "tech", "points": 8 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-minor", "points": 2 }, { "category": "mishear-minor", "points": 2 }, { "category": "filler", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 38, "duration_s": 126, "mean_score": 56.0, "mean_raw": 44.0, "scores_per_judge": [ 56 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "proper-noun", "points": 6 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-severe", "points": 5 }, { "category": "content-drop", "points": 3 }, { "category": "tech", "points": 8 }, { "category": "mishear-minor", "points": 2 }, { "category": "mishear-minor", "points": 2 }, { "category": "mishear-minor", "points": 2 }, { "category": "mishear-minor", "points": 2 }, { "category": "filler", "points": 4 } ] }, "judge_disagreement": 0 }, { "idx": 39, "duration_s": 158, "mean_score": 35.0, "mean_raw": 65.0, "scores_per_judge": [ 35 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "tech", "points": 8 }, { "category": "brand", "points": 10 }, { "category": "tech", "points": 8 }, { "category": "proper-noun", "points": 6 }, { "category": "tech", "points": 8 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 40, "duration_s": 279, "mean_score": 0.0, "mean_raw": 112.0, "scores_per_judge": [ 0 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "semantic-inversion", "points": 20 }, { "category": "culture", "points": 8 }, { "category": "culture", "points": 8 }, { "category": "culture", "points": 8 }, { "category": "tech", "points": 8 }, { "category": "dialect", "points": 5 }, { "category": "numeric", "points": 10 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-minor", "points": 2 }, { "category": "mishear-minor", "points": 2 }, { "category": "mishear-minor", "points": 2 }, { "category": "paraphrase", "points": 3 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 41, "duration_s": 34, "mean_score": 78.0, "mean_raw": 22.0, "scores_per_judge": [ 78 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "culture", "points": 8 }, { "category": "proper-noun", "points": 6 }, { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 42, "duration_s": 43, "mean_score": 61.0, "mean_raw": 39.0, "scores_per_judge": [ 61 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "culture", "points": 8 }, { "category": "culture", "points": 8 }, { "category": "culture", "points": 8 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-severe", "points": 5 }, { "category": "content-drop", "points": 3 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 43, "duration_s": 29, "mean_score": 61.0, "mean_raw": 39.0, "scores_per_judge": [ 61 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "culture", "points": 8 }, { "category": "culture", "points": 8 }, { "category": "culture", "points": 8 }, { "category": "culture", "points": 8 }, { "category": "mishear-minor", "points": 2 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 44, "duration_s": 71, "mean_score": 54.0, "mean_raw": 46.0, "scores_per_judge": [ 54 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "proper-noun", "points": 6 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-minor", "points": 2 }, { "category": "mishear-severe", "points": 5 }, { "category": "culture", "points": 8 }, { "category": "culture", "points": 8 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-minor", "points": 2 }, { "category": "filler", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 45, "duration_s": 46, "mean_score": 0.0, "mean_raw": 100.0, "scores_per_judge": [ 0 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "dialect", "points": 5 }, { "category": "dialect", "points": 5 }, { "category": "dialect", "points": 5 }, { "category": "dialect", "points": 5 }, { "category": "culture", "points": 8 }, { "category": "culture", "points": 8 }, { "category": "culture", "points": 8 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-minor", "points": 2 }, { "category": "polarity", "points": 35 }, { "category": "paraphrase", "points": 3 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 46, "duration_s": 62, "mean_score": 79.0, "mean_raw": 21.0, "scores_per_judge": [ 79 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "paraphrase", "points": 3 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "mishear-severe", "points": 5 }, { "category": "content-drop", "points": 3 }, { "category": "mishear-severe", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 47, "duration_s": 67, "mean_score": 84.0, "mean_raw": 16.0, "scores_per_judge": [ 84 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "brand", "points": 10 }, { "category": "mishear-minor", "points": 2 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 48, "duration_s": 72, "mean_score": 41.0, "mean_raw": 59.0, "scores_per_judge": [ 41 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "polarity", "points": 35 }, { "category": "brand", "points": 10 }, { "category": "mishear-severe", "points": 5 }, { "category": "content-drop", "points": 3 }, { "category": "mishear-minor", "points": 2 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 49, "duration_s": 87, "mean_score": 78.0, "mean_raw": 22.0, "scores_per_judge": [ 78 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "proper-noun", "points": 6 }, { "category": "tech", "points": 8 }, { "category": "paraphrase", "points": 3 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 50, "duration_s": 106, "mean_score": 46.0, "mean_raw": 54.0, "scores_per_judge": [ 46 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 }, { "category": "mishear-minor", "points": 2 }, { "category": "culture", "points": 8 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-minor", "points": 2 }, { "category": "culture", "points": 8 }, { "category": "tech", "points": 8 }, { "category": "mishear-minor", "points": 2 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 51, "duration_s": 33, "mean_score": 39.0, "mean_raw": 61.0, "scores_per_judge": [ 39 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "culture", "points": 8 }, { "category": "filler", "points": 1 }, { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 1 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-severe", "points": 5 }, { "category": "numeric", "points": 10 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-severe", "points": 5 }, { "category": "hallucination", "points": 15 } ] }, "judge_disagreement": 0 }, { "idx": 52, "duration_s": 25, "mean_score": 47.0, "mean_raw": 53.0, "scores_per_judge": [ 47 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "proper-noun", "points": 6 }, { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 1 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-minor", "points": 2 }, { "category": "mishear-severe", "points": 5 }, { "category": "proper-noun", "points": 6 }, { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 1 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 1 }, { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 53, "duration_s": 35, "mean_score": 91.0, "mean_raw": 9.0, "scores_per_judge": [ 91 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "proper-noun", "points": 6 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 54, "duration_s": 36, "mean_score": 93.0, "mean_raw": 7.0, "scores_per_judge": [ 93 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-minor", "points": 2 }, { "category": "filler", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 55, "duration_s": 33, "mean_score": 80.0, "mean_raw": 20.0, "scores_per_judge": [ 80 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 56, "duration_s": 15, "mean_score": 89.0, "mean_raw": 11.0, "scores_per_judge": [ 89 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "brand", "points": 10 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 57, "duration_s": 38, "mean_score": 56.0, "mean_raw": 44.0, "scores_per_judge": [ 56 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "polarity", "points": 35 }, { "category": "mishear-minor", "points": 2 }, { "category": "mishear-minor", "points": 2 }, { "category": "filler", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 58, "duration_s": 15, "mean_score": 77.0, "mean_raw": 23.0, "scores_per_judge": [ 77 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-severe", "points": 5 }, { "category": "tech", "points": 8 } ] }, "judge_disagreement": 0 }, { "idx": 59, "duration_s": 28, "mean_score": 85.0, "mean_raw": 15.0, "scores_per_judge": [ 85 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "tech", "points": 8 }, { "category": "mishear-minor", "points": 2 }, { "category": "filler", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 60, "duration_s": 18, "mean_score": 52.0, "mean_raw": 48.0, "scores_per_judge": [ 52 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "polarity", "points": 35 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 61, "duration_s": 19, "mean_score": 84.0, "mean_raw": 16.0, "scores_per_judge": [ 84 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "brand", "points": 10 }, { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 62, "duration_s": 17, "mean_score": 100.0, "mean_raw": 0.0, "scores_per_judge": [ 100 ], "deductions_per_judge": { "deepseek_flash": [] }, "judge_disagreement": 0 }, { "idx": 63, "duration_s": 21, "mean_score": 96.0, "mean_raw": 4.0, "scores_per_judge": [ 96 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-minor", "points": 2 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 64, "duration_s": 23, "mean_score": 88.0, "mean_raw": 12.0, "scores_per_judge": [ 88 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 }, { "category": "paraphrase", "points": 3 }, { "category": "filler", "points": 4 } ] }, "judge_disagreement": 0 }, { "idx": 65, "duration_s": 20, "mean_score": 99.0, "mean_raw": 1.0, "scores_per_judge": [ 99 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 66, "duration_s": 24, "mean_score": 91.0, "mean_raw": 9.0, "scores_per_judge": [ 91 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 }, { "category": "content-drop", "points": 3 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 67, "duration_s": 16, "mean_score": 94.0, "mean_raw": 6.0, "scores_per_judge": [ 94 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 68, "duration_s": 18, "mean_score": 86.0, "mean_raw": 14.0, "scores_per_judge": [ 86 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "brand", "points": 10 }, { "category": "mishear-minor", "points": 2 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 69, "duration_s": 18, "mean_score": 100.0, "mean_raw": 0.0, "scores_per_judge": [ 100 ], "deductions_per_judge": { "deepseek_flash": [] }, "judge_disagreement": 0 }, { "idx": 70, "duration_s": 17, "mean_score": 50.0, "mean_raw": 50.0, "scores_per_judge": [ 50 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "polarity", "points": 35 }, { "category": "tech", "points": 8 }, { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 71, "duration_s": 51, "mean_score": 49.0, "mean_raw": 51.0, "scores_per_judge": [ 49 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 }, { "category": "paraphrase", "points": 3 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-severe", "points": 5 }, { "category": "tech", "points": 8 }, { "category": "brand", "points": 10 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 72, "duration_s": 54, "mean_score": 88.0, "mean_raw": 12.0, "scores_per_judge": [ 88 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "tech", "points": 8 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 73, "duration_s": 56, "mean_score": 61.0, "mean_raw": 39.0, "scores_per_judge": [ 61 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "semantic-inversion", "points": 20 }, { "category": "numeric", "points": 10 }, { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 74, "duration_s": 45, "mean_score": 90.0, "mean_raw": 10.0, "scores_per_judge": [ 90 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 }, { "category": "mishear-minor", "points": 2 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 75, "duration_s": 74, "mean_score": 79.0, "mean_raw": 21.0, "scores_per_judge": [ 79 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-minor", "points": 2 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 76, "duration_s": 58, "mean_score": 66.0, "mean_raw": 34.0, "scores_per_judge": [ 66 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "semantic-inversion", "points": 20 }, { "category": "mishear-severe", "points": 5 }, { "category": "content-drop", "points": 3 }, { "category": "paraphrase", "points": 3 }, { "category": "filler", "points": 3 } ] }, "judge_disagreement": 0 }, { "idx": 77, "duration_s": 130, "mean_score": 49.0, "mean_raw": 51.0, "scores_per_judge": [ 49 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "semantic-inversion", "points": 20 }, { "category": "brand", "points": 10 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-minor", "points": 2 }, { "category": "mishear-minor", "points": 2 }, { "category": "mishear-minor", "points": 2 }, { "category": "mishear-minor", "points": 2 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 78, "duration_s": 61, "mean_score": 90.0, "mean_raw": 10.0, "scores_per_judge": [ 90 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "paraphrase", "points": 3 }, { "category": "mishear-minor", "points": 2 }, { "category": "mishear-minor", "points": 2 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 79, "duration_s": 47, "mean_score": 91.0, "mean_raw": 9.0, "scores_per_judge": [ 91 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 80, "duration_s": 42, "mean_score": 72.0, "mean_raw": 28.0, "scores_per_judge": [ 72 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 81, "duration_s": 38, "mean_score": 80.0, "mean_raw": 20.0, "scores_per_judge": [ 80 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-minor", "points": 2 }, { "category": "mishear-minor", "points": 2 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 82, "duration_s": 38, "mean_score": 95.0, "mean_raw": 5.0, "scores_per_judge": [ 95 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "paraphrase", "points": 3 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 83, "duration_s": 37, "mean_score": 90.0, "mean_raw": 10.0, "scores_per_judge": [ 90 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 84, "duration_s": 36, "mean_score": 96.0, "mean_raw": 4.0, "scores_per_judge": [ 96 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 85, "duration_s": 36, "mean_score": 89.0, "mean_raw": 11.0, "scores_per_judge": [ 89 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 }, { "category": "mishear-minor", "points": 2 }, { "category": "mishear-minor", "points": 2 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 86, "duration_s": 35, "mean_score": 70.0, "mean_raw": 30.0, "scores_per_judge": [ 70 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 }, { "category": "mishear-severe", "points": 5 }, { "category": "tech", "points": 8 }, { "category": "tech", "points": 8 }, { "category": "mishear-minor", "points": 2 }, { "category": "mishear-minor", "points": 2 } ] }, "judge_disagreement": 0 }, { "idx": 87, "duration_s": 34, "mean_score": 85.0, "mean_raw": 15.0, "scores_per_judge": [ 85 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "numeric", "points": 10 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 88, "duration_s": 34, "mean_score": 97.0, "mean_raw": 3.0, "scores_per_judge": [ 97 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-minor", "points": 2 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 89, "duration_s": 34, "mean_score": 88.0, "mean_raw": 12.0, "scores_per_judge": [ 88 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-minor", "points": 2 } ] }, "judge_disagreement": 0 }, { "idx": 90, "duration_s": 33, "mean_score": 77.0, "mean_raw": 23.0, "scores_per_judge": [ 77 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 }, { "category": "tech", "points": 8 }, { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 91, "duration_s": 33, "mean_score": 80.0, "mean_raw": 20.0, "scores_per_judge": [ 80 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 92, "duration_s": 32, "mean_score": 94.0, "mean_raw": 6.0, "scores_per_judge": [ 94 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 93, "duration_s": 32, "mean_score": 96.0, "mean_raw": 4.0, "scores_per_judge": [ 96 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 94, "duration_s": 32, "mean_score": 91.0, "mean_raw": 9.0, "scores_per_judge": [ 91 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 4 } ] }, "judge_disagreement": 0 }, { "idx": 95, "duration_s": 32, "mean_score": 59.0, "mean_raw": 41.0, "scores_per_judge": [ 59 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "polarity", "points": 35 }, { "category": "paraphrase", "points": 3 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 96, "duration_s": 31, "mean_score": 99.0, "mean_raw": 1.0, "scores_per_judge": [ 99 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 97, "duration_s": 30, "mean_score": 97.0, "mean_raw": 3.0, "scores_per_judge": [ 97 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 98, "duration_s": 30, "mean_score": 97.0, "mean_raw": 3.0, "scores_per_judge": [ 97 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 99, "duration_s": 30, "mean_score": 99.0, "mean_raw": 1.0, "scores_per_judge": [ 99 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 100, "duration_s": 30, "mean_score": 92.0, "mean_raw": 8.0, "scores_per_judge": [ 92 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 } ], "total_deductions": 2360, "avg_deductions_per_sample": 23.6, "mean_clamped_score": 76.8, "n_judge_sample_pairs": 100, "gt_chars": 22625, "ded_per_100char": 10.43, "fidelity_score": 47.8, "overall_score": 76.8, "overall_raw_deductions": 23.6, "score_per_judge": { "deepseek_flash": 76.8 }, "per_group_deductions": { "meaning-flip": { "deepseek_flash": 495 }, "entity": { "deepseek_flash": 452 }, "cultural": { "deepseek_flash": 217 }, "fidelity": { "deepseek_flash": 1196 } }, "per_bucket": { "general": { "n_samples": 40, "mean_raw": 26.1, "mean_score": 74.8, "total_raw": 1045.0, "idxs": [ 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40 ], "worst_idx": 20 }, "idiom-test": { "n_samples": 5, "mean_raw": 49.2, "mean_score": 50.8, "total_raw": 246.0, "idxs": [ 41, 42, 43, 44, 45 ], "worst_idx": 45 }, "long-mixed": { "n_samples": 5, "mean_raw": 34.4, "mean_score": 65.6, "total_raw": 172.0, "idxs": [ 46, 47, 48, 49, 50 ], "worst_idx": 48 } }, "per_group_per_100char": { "meaning-flip": 2.188, "entity": 1.998, "cultural": 0.959, "fidelity": 5.286 }, "per_group_fidelity": { "meaning-flip": 47.5, "entity": 64.6, "cultural": 42.0, "fidelity": 38.1 } }, "gemini_3_5_flash_stt": { "scores_by_judge": { "deepseek_flash": [ 79, 100, 100, 92, 87, 86, 97, 100, 94, 97, 90, 88, 55, 81, 82, 90, 90, 81, 60, 67, 99, 86, 100, 85, 99, 100, 96, 95, 98, 91, 97, 95, 92, 79, 91, 84, 93, 93, 71, 36, 92, 86, 78, 89, 88, 88, 96, 78, 89, 82, 95, 95, 90, 87, 95, 89, 59, 84, 92, 97, 99, 100, 97, 95, 99, 95, 98, 99, 94, 89, 85, 89, 68, 95, 91, 95, 62, 91, 85, 98, 87, 99, 94, 94, 90, 89, 85, 97, 100, 87, 94, 99, 86, 95, 76, 100, 100, 97, 96, 97 ] }, "per_category_deductions": { "polarity": { "deepseek_flash": 35 }, "semantic-inversion": { "deepseek_flash": 100 }, "hallucination": { "deepseek_flash": 0 }, "numeric": { "deepseek_flash": 20 }, "brand": { "deepseek_flash": 110 }, "tech": { "deepseek_flash": 56 }, "culture": { "deepseek_flash": 104 }, "proper-noun": { "deepseek_flash": 48 }, "dialect": { "deepseek_flash": 10 }, "mishear-severe": { "deepseek_flash": 200 }, "content-drop": { "deepseek_flash": 21 }, "paraphrase": { "deepseek_flash": 18 }, "mishear-minor": { "deepseek_flash": 70 }, "filler": { "deepseek_flash": 277 }, "format": { "deepseek_flash": 0 }, "unscoreable": { "deepseek_flash": 0 } }, "per_tier_deductions": { "tier1": { "deepseek_flash": 155 }, "tier2": { "deepseek_flash": 328 }, "tier3": { "deepseek_flash": 309 }, "tier4": { "deepseek_flash": 277 }, "unscoreable": { "deepseek_flash": 0 } }, "raw_deductions_by_judge": { "deepseek_flash": [ 21, 0, 0, 8, 13, 14, 3, 0, 6, 3, 10, 12, 45, 19, 18, 10, 10, 19, 40, 33, 1, 14, 0, 15, 1, 0, 4, 5, 2, 9, 3, 5, 8, 21, 9, 16, 7, 7, 29, 64, 8, 14, 22, 11, 12, 12, 4, 22, 11, 18, 5, 5, 10, 13, 5, 11, 41, 16, 8, 3, 1, 0, 3, 5, 1, 5, 2, 1, 6, 11, 15, 11, 32, 5, 9, 5, 38, 9, 15, 2, 13, 1, 6, 6, 10, 11, 15, 3, 0, 13, 6, 1, 14, 5, 24, 0, 0, 3, 4, 3 ] }, "sample_details": [ { "idx": 1, "duration_s": 5, "mean_score": 79.0, "mean_raw": 21.0, "scores_per_judge": [ 79 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "semantic-inversion", "points": 20 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 2, "duration_s": 7, "mean_score": 100.0, "mean_raw": 0.0, "scores_per_judge": [ 100 ], "deductions_per_judge": { "deepseek_flash": [] }, "judge_disagreement": 0 }, { "idx": 3, "duration_s": 8, "mean_score": 100.0, "mean_raw": 0.0, "scores_per_judge": [ 100 ], "deductions_per_judge": { "deepseek_flash": [] }, "judge_disagreement": 0 }, { "idx": 4, "duration_s": 11, "mean_score": 92.0, "mean_raw": 8.0, "scores_per_judge": [ 92 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "culture", "points": 8 } ] }, "judge_disagreement": 0 }, { "idx": 5, "duration_s": 16, "mean_score": 87.0, "mean_raw": 13.0, "scores_per_judge": [ 87 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "culture", "points": 8 }, { "category": "mishear-minor", "points": 2 }, { "category": "filler", "points": 3 } ] }, "judge_disagreement": 0 }, { "idx": 6, "duration_s": 17, "mean_score": 86.0, "mean_raw": 14.0, "scores_per_judge": [ 86 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-minor", "points": 2 } ] }, "judge_disagreement": 0 }, { "idx": 7, "duration_s": 20, "mean_score": 97.0, "mean_raw": 3.0, "scores_per_judge": [ 97 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-minor", "points": 2 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 8, "duration_s": 30, "mean_score": 100.0, "mean_raw": 0.0, "scores_per_judge": [ 100 ], "deductions_per_judge": { "deepseek_flash": [] }, "judge_disagreement": 0 }, { "idx": 9, "duration_s": 32, "mean_score": 94.0, "mean_raw": 6.0, "scores_per_judge": [ 94 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "content-drop", "points": 3 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 10, "duration_s": 35, "mean_score": 97.0, "mean_raw": 3.0, "scores_per_judge": [ 97 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 11, "duration_s": 40, "mean_score": 90.0, "mean_raw": 10.0, "scores_per_judge": [ 90 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 12, "duration_s": 46, "mean_score": 88.0, "mean_raw": 12.0, "scores_per_judge": [ 88 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "culture", "points": 8 }, { "category": "filler", "points": 4 } ] }, "judge_disagreement": 0 }, { "idx": 13, "duration_s": 60, "mean_score": 55.0, "mean_raw": 45.0, "scores_per_judge": [ 55 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "brand", "points": 10 }, { "category": "brand", "points": 10 }, { "category": "brand", "points": 10 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 14, "duration_s": 64, "mean_score": 81.0, "mean_raw": 19.0, "scores_per_judge": [ 81 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "culture", "points": 8 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 15, "duration_s": 69, "mean_score": 82.0, "mean_raw": 18.0, "scores_per_judge": [ 82 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "culture", "points": 8 }, { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 16, "duration_s": 81, "mean_score": 90.0, "mean_raw": 10.0, "scores_per_judge": [ 90 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "proper-noun", "points": 6 }, { "category": "mishear-minor", "points": 2 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 17, "duration_s": 100, "mean_score": 90.0, "mean_raw": 10.0, "scores_per_judge": [ 90 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 18, "duration_s": 125, "mean_score": 81.0, "mean_raw": 19.0, "scores_per_judge": [ 81 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 19, "duration_s": 133, "mean_score": 60.0, "mean_raw": 40.0, "scores_per_judge": [ 60 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "brand", "points": 10 }, { "category": "brand", "points": 10 }, { "category": "brand", "points": 10 }, { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 20, "duration_s": 164, "mean_score": 67.0, "mean_raw": 33.0, "scores_per_judge": [ 67 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 }, { "category": "culture", "points": 8 }, { "category": "mishear-severe", "points": 5 }, { "category": "proper-noun", "points": 6 }, { "category": "mishear-minor", "points": 2 }, { "category": "mishear-minor", "points": 2 }, { "category": "filler", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 21, "duration_s": 5, "mean_score": 99.0, "mean_raw": 1.0, "scores_per_judge": [ 99 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 22, "duration_s": 7, "mean_score": 86.0, "mean_raw": 14.0, "scores_per_judge": [ 86 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "proper-noun", "points": 6 }, { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 23, "duration_s": 10, "mean_score": 100.0, "mean_raw": 0.0, "scores_per_judge": [ 100 ], "deductions_per_judge": { "deepseek_flash": [] }, "judge_disagreement": 0 }, { "idx": 24, "duration_s": 14, "mean_score": 85.0, "mean_raw": 15.0, "scores_per_judge": [ 85 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 }, { "category": "tech", "points": 8 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 25, "duration_s": 16, "mean_score": 99.0, "mean_raw": 1.0, "scores_per_judge": [ 99 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 26, "duration_s": 23, "mean_score": 100.0, "mean_raw": 0.0, "scores_per_judge": [ 100 ], "deductions_per_judge": { "deepseek_flash": [] }, "judge_disagreement": 0 }, { "idx": 27, "duration_s": 25, "mean_score": 96.0, "mean_raw": 4.0, "scores_per_judge": [ 96 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 2 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 28, "duration_s": 30, "mean_score": 95.0, "mean_raw": 5.0, "scores_per_judge": [ 95 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 29, "duration_s": 34, "mean_score": 98.0, "mean_raw": 2.0, "scores_per_judge": [ 98 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 30, "duration_s": 38, "mean_score": 91.0, "mean_raw": 9.0, "scores_per_judge": [ 91 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-minor", "points": 2 }, { "category": "mishear-minor", "points": 2 }, { "category": "mishear-severe", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 31, "duration_s": 44, "mean_score": 97.0, "mean_raw": 3.0, "scores_per_judge": [ 97 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 32, "duration_s": 50, "mean_score": 95.0, "mean_raw": 5.0, "scores_per_judge": [ 95 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 33, "duration_s": 62, "mean_score": 92.0, "mean_raw": 8.0, "scores_per_judge": [ 92 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "proper-noun", "points": 6 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 34, "duration_s": 66, "mean_score": 79.0, "mean_raw": 21.0, "scores_per_judge": [ 79 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "proper-noun", "points": 6 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 35, "duration_s": 77, "mean_score": 91.0, "mean_raw": 9.0, "scores_per_judge": [ 91 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 36, "duration_s": 94, "mean_score": 84.0, "mean_raw": 16.0, "scores_per_judge": [ 84 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "culture", "points": 8 }, { "category": "content-drop", "points": 3 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 37, "duration_s": 106, "mean_score": 93.0, "mean_raw": 7.0, "scores_per_judge": [ 93 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-minor", "points": 2 }, { "category": "filler", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 38, "duration_s": 126, "mean_score": 93.0, "mean_raw": 7.0, "scores_per_judge": [ 93 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 2 } ] }, "judge_disagreement": 0 }, { "idx": 39, "duration_s": 158, "mean_score": 71.0, "mean_raw": 29.0, "scores_per_judge": [ 71 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "brand", "points": 10 }, { "category": "tech", "points": 8 }, { "category": "proper-noun", "points": 6 }, { "category": "filler", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 40, "duration_s": 279, "mean_score": 36.0, "mean_raw": 64.0, "scores_per_judge": [ 36 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "semantic-inversion", "points": 20 }, { "category": "numeric", "points": 10 }, { "category": "culture", "points": 8 }, { "category": "dialect", "points": 5 }, { "category": "dialect", "points": 5 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-severe", "points": 5 }, { "category": "paraphrase", "points": 3 }, { "category": "filler", "points": 3 } ] }, "judge_disagreement": 0 }, { "idx": 41, "duration_s": 34, "mean_score": 92.0, "mean_raw": 8.0, "scores_per_judge": [ 92 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "content-drop", "points": 3 }, { "category": "filler", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 42, "duration_s": 43, "mean_score": 86.0, "mean_raw": 14.0, "scores_per_judge": [ 86 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "culture", "points": 8 }, { "category": "content-drop", "points": 3 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 43, "duration_s": 29, "mean_score": 78.0, "mean_raw": 22.0, "scores_per_judge": [ 78 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "culture", "points": 8 }, { "category": "proper-noun", "points": 6 }, { "category": "mishear-minor", "points": 2 }, { "category": "mishear-minor", "points": 2 }, { "category": "filler", "points": 4 } ] }, "judge_disagreement": 0 }, { "idx": 44, "duration_s": 71, "mean_score": 89.0, "mean_raw": 11.0, "scores_per_judge": [ 89 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "culture", "points": 8 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 45, "duration_s": 46, "mean_score": 88.0, "mean_raw": 12.0, "scores_per_judge": [ 88 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 }, { "category": "mishear-minor", "points": 2 }, { "category": "mishear-minor", "points": 2 }, { "category": "filler", "points": 3 } ] }, "judge_disagreement": 0 }, { "idx": 46, "duration_s": 62, "mean_score": 88.0, "mean_raw": 12.0, "scores_per_judge": [ 88 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "mishear-minor", "points": 2 }, { "category": "mishear-severe", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 47, "duration_s": 67, "mean_score": 96.0, "mean_raw": 4.0, "scores_per_judge": [ 96 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 48, "duration_s": 72, "mean_score": 78.0, "mean_raw": 22.0, "scores_per_judge": [ 78 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "brand", "points": 10 }, { "category": "tech", "points": 8 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 49, "duration_s": 87, "mean_score": 89.0, "mean_raw": 11.0, "scores_per_judge": [ 89 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "content-drop", "points": 3 }, { "category": "paraphrase", "points": 3 }, { "category": "filler", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 50, "duration_s": 106, "mean_score": 82.0, "mean_raw": 18.0, "scores_per_judge": [ 82 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "culture", "points": 8 }, { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 51, "duration_s": 33, "mean_score": 95.0, "mean_raw": 5.0, "scores_per_judge": [ 95 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 52, "duration_s": 25, "mean_score": 95.0, "mean_raw": 5.0, "scores_per_judge": [ 95 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 53, "duration_s": 35, "mean_score": 90.0, "mean_raw": 10.0, "scores_per_judge": [ 90 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "culture", "points": 8 }, { "category": "filler", "points": 2 } ] }, "judge_disagreement": 0 }, { "idx": 54, "duration_s": 36, "mean_score": 87.0, "mean_raw": 13.0, "scores_per_judge": [ 87 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "tech", "points": 8 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 55, "duration_s": 33, "mean_score": 95.0, "mean_raw": 5.0, "scores_per_judge": [ 95 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 56, "duration_s": 15, "mean_score": 89.0, "mean_raw": 11.0, "scores_per_judge": [ 89 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "brand", "points": 10 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 57, "duration_s": 38, "mean_score": 59.0, "mean_raw": 41.0, "scores_per_judge": [ 59 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "polarity", "points": 35 }, { "category": "mishear-minor", "points": 2 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 58, "duration_s": 15, "mean_score": 84.0, "mean_raw": 16.0, "scores_per_judge": [ 84 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "brand", "points": 10 }, { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 59, "duration_s": 28, "mean_score": 92.0, "mean_raw": 8.0, "scores_per_judge": [ 92 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "mishear-minor", "points": 2 }, { "category": "mishear-minor", "points": 2 } ] }, "judge_disagreement": 0 }, { "idx": 60, "duration_s": 18, "mean_score": 97.0, "mean_raw": 3.0, "scores_per_judge": [ 97 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 61, "duration_s": 19, "mean_score": 99.0, "mean_raw": 1.0, "scores_per_judge": [ 99 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 62, "duration_s": 17, "mean_score": 100.0, "mean_raw": 0.0, "scores_per_judge": [ 100 ], "deductions_per_judge": { "deepseek_flash": [] }, "judge_disagreement": 0 }, { "idx": 63, "duration_s": 21, "mean_score": 97.0, "mean_raw": 3.0, "scores_per_judge": [ 97 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 64, "duration_s": 23, "mean_score": 95.0, "mean_raw": 5.0, "scores_per_judge": [ 95 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 65, "duration_s": 20, "mean_score": 99.0, "mean_raw": 1.0, "scores_per_judge": [ 99 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 66, "duration_s": 24, "mean_score": 95.0, "mean_raw": 5.0, "scores_per_judge": [ 95 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "content-drop", "points": 3 }, { "category": "mishear-minor", "points": 2 } ] }, "judge_disagreement": 0 }, { "idx": 67, "duration_s": 16, "mean_score": 98.0, "mean_raw": 2.0, "scores_per_judge": [ 98 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 68, "duration_s": 18, "mean_score": 99.0, "mean_raw": 1.0, "scores_per_judge": [ 99 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 69, "duration_s": 18, "mean_score": 94.0, "mean_raw": 6.0, "scores_per_judge": [ 94 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 70, "duration_s": 17, "mean_score": 89.0, "mean_raw": 11.0, "scores_per_judge": [ 89 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 }, { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 71, "duration_s": 51, "mean_score": 85.0, "mean_raw": 15.0, "scores_per_judge": [ 85 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 }, { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 72, "duration_s": 54, "mean_score": 89.0, "mean_raw": 11.0, "scores_per_judge": [ 89 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "tech", "points": 8 }, { "category": "filler", "points": 2 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 73, "duration_s": 56, "mean_score": 68.0, "mean_raw": 32.0, "scores_per_judge": [ 68 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "semantic-inversion", "points": 20 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-minor", "points": 2 }, { "category": "mishear-minor", "points": 2 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 74, "duration_s": 45, "mean_score": 95.0, "mean_raw": 5.0, "scores_per_judge": [ 95 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 5 }, { "category": "format", "points": 0 } ] }, "judge_disagreement": 0 }, { "idx": 75, "duration_s": 74, "mean_score": 91.0, "mean_raw": 9.0, "scores_per_judge": [ 91 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "mishear-minor", "points": 2 }, { "category": "mishear-minor", "points": 2 } ] }, "judge_disagreement": 0 }, { "idx": 76, "duration_s": 58, "mean_score": 95.0, "mean_raw": 5.0, "scores_per_judge": [ 95 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 77, "duration_s": 130, "mean_score": 62.0, "mean_raw": 38.0, "scores_per_judge": [ 62 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "semantic-inversion", "points": 20 }, { "category": "brand", "points": 10 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-minor", "points": 2 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 78, "duration_s": 61, "mean_score": 91.0, "mean_raw": 9.0, "scores_per_judge": [ 91 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 4 } ] }, "judge_disagreement": 0 }, { "idx": 79, "duration_s": 47, "mean_score": 85.0, "mean_raw": 15.0, "scores_per_judge": [ 85 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 }, { "category": "content-drop", "points": 3 }, { "category": "paraphrase", "points": 3 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 80, "duration_s": 42, "mean_score": 98.0, "mean_raw": 2.0, "scores_per_judge": [ 98 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 81, "duration_s": 38, "mean_score": 87.0, "mean_raw": 13.0, "scores_per_judge": [ 87 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "proper-noun", "points": 6 }, { "category": "paraphrase", "points": 3 }, { "category": "mishear-minor", "points": 2 }, { "category": "mishear-minor", "points": 2 } ] }, "judge_disagreement": 0 }, { "idx": 82, "duration_s": 38, "mean_score": 99.0, "mean_raw": 1.0, "scores_per_judge": [ 99 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 83, "duration_s": 37, "mean_score": 94.0, "mean_raw": 6.0, "scores_per_judge": [ 94 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 84, "duration_s": 36, "mean_score": 94.0, "mean_raw": 6.0, "scores_per_judge": [ 94 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "mishear-minor", "points": 2 } ] }, "judge_disagreement": 0 }, { "idx": 85, "duration_s": 36, "mean_score": 90.0, "mean_raw": 10.0, "scores_per_judge": [ 90 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-minor", "points": 2 }, { "category": "mishear-minor", "points": 2 }, { "category": "mishear-minor", "points": 2 }, { "category": "mishear-minor", "points": 2 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 86, "duration_s": 35, "mean_score": 89.0, "mean_raw": 11.0, "scores_per_judge": [ 89 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "tech", "points": 8 }, { "category": "filler", "points": 3 } ] }, "judge_disagreement": 0 }, { "idx": 87, "duration_s": 34, "mean_score": 85.0, "mean_raw": 15.0, "scores_per_judge": [ 85 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "numeric", "points": 10 }, { "category": "mishear-minor", "points": 2 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 88, "duration_s": 34, "mean_score": 97.0, "mean_raw": 3.0, "scores_per_judge": [ 97 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-minor", "points": 2 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 89, "duration_s": 34, "mean_score": 100.0, "mean_raw": 0.0, "scores_per_judge": [ 100 ], "deductions_per_judge": { "deepseek_flash": [] }, "judge_disagreement": 0 }, { "idx": 90, "duration_s": 33, "mean_score": 87.0, "mean_raw": 13.0, "scores_per_judge": [ 87 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 }, { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 3 } ] }, "judge_disagreement": 0 }, { "idx": 91, "duration_s": 33, "mean_score": 94.0, "mean_raw": 6.0, "scores_per_judge": [ 94 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "paraphrase", "points": 3 } ] }, "judge_disagreement": 0 }, { "idx": 92, "duration_s": 32, "mean_score": 99.0, "mean_raw": 1.0, "scores_per_judge": [ 99 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 93, "duration_s": 32, "mean_score": 86.0, "mean_raw": 14.0, "scores_per_judge": [ 86 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "tech", "points": 8 }, { "category": "mishear-minor", "points": 2 }, { "category": "filler", "points": 4 } ] }, "judge_disagreement": 0 }, { "idx": 94, "duration_s": 32, "mean_score": 95.0, "mean_raw": 5.0, "scores_per_judge": [ 95 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-minor", "points": 2 }, { "category": "filler", "points": 3 } ] }, "judge_disagreement": 0 }, { "idx": 95, "duration_s": 32, "mean_score": 76.0, "mean_raw": 24.0, "scores_per_judge": [ 76 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "semantic-inversion", "points": 20 }, { "category": "paraphrase", "points": 3 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 96, "duration_s": 31, "mean_score": 100.0, "mean_raw": 0.0, "scores_per_judge": [ 100 ], "deductions_per_judge": { "deepseek_flash": [] }, "judge_disagreement": 0 }, { "idx": 97, "duration_s": 30, "mean_score": 100.0, "mean_raw": 0.0, "scores_per_judge": [ 100 ], "deductions_per_judge": { "deepseek_flash": [] }, "judge_disagreement": 0 }, { "idx": 98, "duration_s": 30, "mean_score": 97.0, "mean_raw": 3.0, "scores_per_judge": [ 97 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 99, "duration_s": 30, "mean_score": 96.0, "mean_raw": 4.0, "scores_per_judge": [ 96 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 100, "duration_s": 30, "mean_score": 97.0, "mean_raw": 3.0, "scores_per_judge": [ 97 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-minor", "points": 2 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 } ], "total_deductions": 1069, "avg_deductions_per_sample": 10.69, "mean_clamped_score": 89.3, "n_judge_sample_pairs": 100, "gt_chars": 22625, "ded_per_100char": 4.72, "fidelity_score": 76.4, "overall_score": 89.3, "overall_raw_deductions": 10.69, "score_per_judge": { "deepseek_flash": 89.3 }, "per_group_deductions": { "meaning-flip": { "deepseek_flash": 155 }, "entity": { "deepseek_flash": 214 }, "cultural": { "deepseek_flash": 114 }, "fidelity": { "deepseek_flash": 586 } }, "per_bucket": { "general": { "n_samples": 40, "mean_raw": 12.6, "mean_score": 87.4, "total_raw": 504.0, "idxs": [ 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40 ], "worst_idx": 40 }, "idiom-test": { "n_samples": 5, "mean_raw": 13.4, "mean_score": 86.6, "total_raw": 67.0, "idxs": [ 41, 42, 43, 44, 45 ], "worst_idx": 43 }, "long-mixed": { "n_samples": 5, "mean_raw": 13.4, "mean_score": 86.6, "total_raw": 67.0, "idxs": [ 46, 47, 48, 49, 50 ], "worst_idx": 48 } }, "per_group_per_100char": { "meaning-flip": 0.685, "entity": 0.946, "cultural": 0.504, "fidelity": 2.59 }, "per_group_fidelity": { "meaning-flip": 83.6, "entity": 83.2, "cultural": 69.5, "fidelity": 69.7 } }, "gemini_3_5_transcribe": { "scores_by_judge": { "deepseek_flash": [ 100, 100, 99, 98, 99, 87, 99, 99, 97, 99, 85, 91, 78, 90, 97, 81, 85, 82, 48, 56, 100, 88, 100, 89, 98, 96, 91, 85, 87, 84, 98, 91, 82, 70, 87, 87, 87, 75, 70, 17, 93, 73, 76, 90, 70, 95, 84, 85, 92, 75, 90, 89, 96, 97, 95, 79, 85, 91, 59, 62, 96, 100, 97, 96, 100, 86, 89, 94, 89, 85, 95, 98, 81, 95, 90, 71, 55, 98, 95, 97, 94, 96, 96, 98, 96, 89, 86, 100, 94, 91, 93, 98, 88, 96, 98, 100, 97, 98, 97, 98 ] }, "per_category_deductions": { "polarity": { "deepseek_flash": 70 }, "semantic-inversion": { "deepseek_flash": 100 }, "hallucination": { "deepseek_flash": 15 }, "numeric": { "deepseek_flash": 20 }, "brand": { "deepseek_flash": 150 }, "tech": { "deepseek_flash": 72 }, "culture": { "deepseek_flash": 72 }, "proper-noun": { "deepseek_flash": 54 }, "dialect": { "deepseek_flash": 25 }, "mishear-severe": { "deepseek_flash": 195 }, "content-drop": { "deepseek_flash": 39 }, "paraphrase": { "deepseek_flash": 18 }, "mishear-minor": { "deepseek_flash": 70 }, "filler": { "deepseek_flash": 262 }, "format": { "deepseek_flash": 0 }, "unscoreable": { "deepseek_flash": 0 } }, "per_tier_deductions": { "tier1": { "deepseek_flash": 205 }, "tier2": { "deepseek_flash": 373 }, "tier3": { "deepseek_flash": 322 }, "tier4": { "deepseek_flash": 262 }, "unscoreable": { "deepseek_flash": 0 } }, "raw_deductions_by_judge": { "deepseek_flash": [ 0, 0, 1, 2, 1, 13, 1, 1, 3, 1, 15, 9, 22, 10, 3, 19, 15, 18, 52, 44, 0, 12, 0, 11, 2, 4, 9, 15, 13, 16, 2, 9, 18, 30, 13, 13, 13, 25, 30, 83, 7, 27, 24, 10, 30, 5, 16, 15, 8, 25, 10, 11, 4, 3, 5, 21, 15, 9, 41, 38, 4, 0, 3, 4, 0, 14, 11, 6, 11, 15, 5, 2, 19, 5, 10, 29, 45, 2, 5, 3, 6, 4, 4, 2, 4, 11, 14, 0, 6, 9, 7, 2, 12, 4, 2, 0, 3, 2, 3, 2 ] }, "sample_details": [ { "idx": 1, "duration_s": 5, "mean_score": 100.0, "mean_raw": 0.0, "scores_per_judge": [ 100 ], "deductions_per_judge": { "deepseek_flash": [] }, "judge_disagreement": 0 }, { "idx": 2, "duration_s": 7, "mean_score": 100.0, "mean_raw": 0.0, "scores_per_judge": [ 100 ], "deductions_per_judge": { "deepseek_flash": [] }, "judge_disagreement": 0 }, { "idx": 3, "duration_s": 8, "mean_score": 99.0, "mean_raw": 1.0, "scores_per_judge": [ 99 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 4, "duration_s": 11, "mean_score": 98.0, "mean_raw": 2.0, "scores_per_judge": [ 98 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 2 } ] }, "judge_disagreement": 0 }, { "idx": 5, "duration_s": 16, "mean_score": 99.0, "mean_raw": 1.0, "scores_per_judge": [ 99 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 6, "duration_s": 17, "mean_score": 87.0, "mean_raw": 13.0, "scores_per_judge": [ 87 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-minor", "points": 2 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 7, "duration_s": 20, "mean_score": 99.0, "mean_raw": 1.0, "scores_per_judge": [ 99 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 8, "duration_s": 30, "mean_score": 99.0, "mean_raw": 1.0, "scores_per_judge": [ 99 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 9, "duration_s": 32, "mean_score": 97.0, "mean_raw": 3.0, "scores_per_judge": [ 97 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 10, "duration_s": 35, "mean_score": 99.0, "mean_raw": 1.0, "scores_per_judge": [ 99 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 11, "duration_s": 40, "mean_score": 85.0, "mean_raw": 15.0, "scores_per_judge": [ 85 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 5 }, { "category": "content-drop", "points": 3 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-minor", "points": 2 } ] }, "judge_disagreement": 0 }, { "idx": 12, "duration_s": 46, "mean_score": 91.0, "mean_raw": 9.0, "scores_per_judge": [ 91 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 4 } ] }, "judge_disagreement": 0 }, { "idx": 13, "duration_s": 60, "mean_score": 78.0, "mean_raw": 22.0, "scores_per_judge": [ 78 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "brand", "points": 10 }, { "category": "brand", "points": 10 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 14, "duration_s": 64, "mean_score": 90.0, "mean_raw": 10.0, "scores_per_judge": [ 90 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 15, "duration_s": 69, "mean_score": 97.0, "mean_raw": 3.0, "scores_per_judge": [ 97 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 16, "duration_s": 81, "mean_score": 81.0, "mean_raw": 19.0, "scores_per_judge": [ 81 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-minor", "points": 2 }, { "category": "proper-noun", "points": 6 }, { "category": "proper-noun", "points": 6 }, { "category": "filler", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 17, "duration_s": 100, "mean_score": 85.0, "mean_raw": 15.0, "scores_per_judge": [ 85 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "brand", "points": 10 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 18, "duration_s": 125, "mean_score": 82.0, "mean_raw": 18.0, "scores_per_judge": [ 82 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 }, { "category": "content-drop", "points": 3 }, { "category": "mishear-minor", "points": 2 }, { "category": "mishear-minor", "points": 2 }, { "category": "mishear-minor", "points": 2 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 19, "duration_s": 133, "mean_score": 48.0, "mean_raw": 52.0, "scores_per_judge": [ 48 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "brand", "points": 10 }, { "category": "brand", "points": 10 }, { "category": "brand", "points": 10 }, { "category": "brand", "points": 10 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-minor", "points": 2 }, { "category": "filler", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 20, "duration_s": 164, "mean_score": 56.0, "mean_raw": 44.0, "scores_per_judge": [ 56 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-severe", "points": 5 }, { "category": "proper-noun", "points": 6 }, { "category": "mishear-minor", "points": 2 }, { "category": "mishear-minor", "points": 2 }, { "category": "mishear-minor", "points": 2 }, { "category": "mishear-minor", "points": 2 }, { "category": "filler", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 21, "duration_s": 5, "mean_score": 100.0, "mean_raw": 0.0, "scores_per_judge": [ 100 ], "deductions_per_judge": { "deepseek_flash": [] }, "judge_disagreement": 0 }, { "idx": 22, "duration_s": 7, "mean_score": 88.0, "mean_raw": 12.0, "scores_per_judge": [ 88 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "proper-noun", "points": 6 }, { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 23, "duration_s": 10, "mean_score": 100.0, "mean_raw": 0.0, "scores_per_judge": [ 100 ], "deductions_per_judge": { "deepseek_flash": [] }, "judge_disagreement": 0 }, { "idx": 24, "duration_s": 14, "mean_score": 89.0, "mean_raw": 11.0, "scores_per_judge": [ 89 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "tech", "points": 8 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 25, "duration_s": 16, "mean_score": 98.0, "mean_raw": 2.0, "scores_per_judge": [ 98 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 26, "duration_s": 23, "mean_score": 96.0, "mean_raw": 4.0, "scores_per_judge": [ 96 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "content-drop", "points": 3 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 27, "duration_s": 25, "mean_score": 91.0, "mean_raw": 9.0, "scores_per_judge": [ 91 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 }, { "category": "paraphrase", "points": 3 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 28, "duration_s": 30, "mean_score": 85.0, "mean_raw": 15.0, "scores_per_judge": [ 85 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 }, { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 29, "duration_s": 34, "mean_score": 87.0, "mean_raw": 13.0, "scores_per_judge": [ 87 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "brand", "points": 10 }, { "category": "mishear-minor", "points": 2 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 30, "duration_s": 38, "mean_score": 84.0, "mean_raw": 16.0, "scores_per_judge": [ 84 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 }, { "category": "culture", "points": 8 }, { "category": "filler", "points": 3 } ] }, "judge_disagreement": 0 }, { "idx": 31, "duration_s": 44, "mean_score": 98.0, "mean_raw": 2.0, "scores_per_judge": [ 98 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-minor", "points": 2 } ] }, "judge_disagreement": 0 }, { "idx": 32, "duration_s": 50, "mean_score": 91.0, "mean_raw": 9.0, "scores_per_judge": [ 91 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "mishear-severe", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 33, "duration_s": 62, "mean_score": 82.0, "mean_raw": 18.0, "scores_per_judge": [ 82 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "brand", "points": 10 }, { "category": "proper-noun", "points": 6 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 34, "duration_s": 66, "mean_score": 70.0, "mean_raw": 30.0, "scores_per_judge": [ 70 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "semantic-inversion", "points": 20 }, { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 35, "duration_s": 77, "mean_score": 87.0, "mean_raw": 13.0, "scores_per_judge": [ 87 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "tech", "points": 8 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 36, "duration_s": 94, "mean_score": 87.0, "mean_raw": 13.0, "scores_per_judge": [ 87 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "tech", "points": 8 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 37, "duration_s": 106, "mean_score": 87.0, "mean_raw": 13.0, "scores_per_judge": [ 87 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "tech", "points": 8 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 38, "duration_s": 126, "mean_score": 75.0, "mean_raw": 25.0, "scores_per_judge": [ 75 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 }, { "category": "mishear-minor", "points": 2 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-severe", "points": 5 }, { "category": "tech", "points": 8 } ] }, "judge_disagreement": 0 }, { "idx": 39, "duration_s": 158, "mean_score": 70.0, "mean_raw": 30.0, "scores_per_judge": [ 70 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "proper-noun", "points": 6 }, { "category": "brand", "points": 10 }, { "category": "tech", "points": 8 }, { "category": "mishear-minor", "points": 2 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 2 } ] }, "judge_disagreement": 0 }, { "idx": 40, "duration_s": 279, "mean_score": 17.0, "mean_raw": 83.0, "scores_per_judge": [ 17 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "semantic-inversion", "points": 20 }, { "category": "mishear-severe", "points": 5 }, { "category": "culture", "points": 8 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-severe", "points": 5 }, { "category": "content-drop", "points": 3 }, { "category": "content-drop", "points": 3 }, { "category": "content-drop", "points": 3 }, { "category": "numeric", "points": 10 }, { "category": "paraphrase", "points": 3 }, { "category": "culture", "points": 8 }, { "category": "dialect", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 41, "duration_s": 34, "mean_score": 93.0, "mean_raw": 7.0, "scores_per_judge": [ 93 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-minor", "points": 2 }, { "category": "filler", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 42, "duration_s": 43, "mean_score": 73.0, "mean_raw": 27.0, "scores_per_judge": [ 73 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "semantic-inversion", "points": 20 }, { "category": "dialect", "points": 5 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 43, "duration_s": 29, "mean_score": 76.0, "mean_raw": 24.0, "scores_per_judge": [ 76 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 }, { "category": "culture", "points": 8 }, { "category": "paraphrase", "points": 3 }, { "category": "paraphrase", "points": 3 }, { "category": "filler", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 44, "duration_s": 71, "mean_score": 90.0, "mean_raw": 10.0, "scores_per_judge": [ 90 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "culture", "points": 8 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 45, "duration_s": 46, "mean_score": 70.0, "mean_raw": 30.0, "scores_per_judge": [ 70 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "dialect", "points": 5 }, { "category": "dialect", "points": 5 }, { "category": "dialect", "points": 5 }, { "category": "culture", "points": 8 }, { "category": "content-drop", "points": 3 }, { "category": "filler", "points": 4 } ] }, "judge_disagreement": 0 }, { "idx": 46, "duration_s": 62, "mean_score": 95.0, "mean_raw": 5.0, "scores_per_judge": [ 95 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 47, "duration_s": 67, "mean_score": 84.0, "mean_raw": 16.0, "scores_per_judge": [ 84 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "brand", "points": 10 }, { "category": "mishear-minor", "points": 2 }, { "category": "mishear-minor", "points": 2 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 48, "duration_s": 72, "mean_score": 85.0, "mean_raw": 15.0, "scores_per_judge": [ 85 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "brand", "points": 10 }, { "category": "filler", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 49, "duration_s": 87, "mean_score": 92.0, "mean_raw": 8.0, "scores_per_judge": [ 92 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "content-drop", "points": 3 }, { "category": "filler", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 50, "duration_s": 106, "mean_score": 75.0, "mean_raw": 25.0, "scores_per_judge": [ 75 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 }, { "category": "culture", "points": 8 }, { "category": "culture", "points": 8 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 51, "duration_s": 33, "mean_score": 90.0, "mean_raw": 10.0, "scores_per_judge": [ 90 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "culture", "points": 8 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 52, "duration_s": 25, "mean_score": 89.0, "mean_raw": 11.0, "scores_per_judge": [ 89 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "proper-noun", "points": 6 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 53, "duration_s": 35, "mean_score": 96.0, "mean_raw": 4.0, "scores_per_judge": [ 96 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "paraphrase", "points": 3 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 54, "duration_s": 36, "mean_score": 97.0, "mean_raw": 3.0, "scores_per_judge": [ 97 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 55, "duration_s": 33, "mean_score": 95.0, "mean_raw": 5.0, "scores_per_judge": [ 95 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 56, "duration_s": 15, "mean_score": 79.0, "mean_raw": 21.0, "scores_per_judge": [ 79 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "brand", "points": 10 }, { "category": "brand", "points": 10 } ] }, "judge_disagreement": 0 }, { "idx": 57, "duration_s": 38, "mean_score": 85.0, "mean_raw": 15.0, "scores_per_judge": [ 85 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-minor", "points": 2 }, { "category": "mishear-minor", "points": 2 }, { "category": "mishear-minor", "points": 2 }, { "category": "mishear-minor", "points": 2 }, { "category": "mishear-minor", "points": 2 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 58, "duration_s": 15, "mean_score": 91.0, "mean_raw": 9.0, "scores_per_judge": [ 91 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "tech", "points": 8 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 59, "duration_s": 28, "mean_score": 59.0, "mean_raw": 41.0, "scores_per_judge": [ 59 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "polarity", "points": 35 }, { "category": "mishear-minor", "points": 2 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 60, "duration_s": 18, "mean_score": 62.0, "mean_raw": 38.0, "scores_per_judge": [ 62 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "polarity", "points": 35 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 61, "duration_s": 19, "mean_score": 96.0, "mean_raw": 4.0, "scores_per_judge": [ 96 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 62, "duration_s": 17, "mean_score": 100.0, "mean_raw": 0.0, "scores_per_judge": [ 100 ], "deductions_per_judge": { "deepseek_flash": [] }, "judge_disagreement": 0 }, { "idx": 63, "duration_s": 21, "mean_score": 97.0, "mean_raw": 3.0, "scores_per_judge": [ 97 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 64, "duration_s": 23, "mean_score": 96.0, "mean_raw": 4.0, "scores_per_judge": [ 96 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "mishear-minor", "points": 2 } ] }, "judge_disagreement": 0 }, { "idx": 65, "duration_s": 20, "mean_score": 100.0, "mean_raw": 0.0, "scores_per_judge": [ 100 ], "deductions_per_judge": { "deepseek_flash": [] }, "judge_disagreement": 0 }, { "idx": 66, "duration_s": 24, "mean_score": 86.0, "mean_raw": 14.0, "scores_per_judge": [ 86 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "tech", "points": 8 }, { "category": "paraphrase", "points": 3 }, { "category": "mishear-minor", "points": 2 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 67, "duration_s": 16, "mean_score": 89.0, "mean_raw": 11.0, "scores_per_judge": [ 89 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 }, { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 68, "duration_s": 18, "mean_score": 94.0, "mean_raw": 6.0, "scores_per_judge": [ 94 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "content-drop", "points": 3 }, { "category": "content-drop", "points": 3 } ] }, "judge_disagreement": 0 }, { "idx": 69, "duration_s": 18, "mean_score": 89.0, "mean_raw": 11.0, "scores_per_judge": [ 89 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "proper-noun", "points": 6 }, { "category": "mishear-minor", "points": 2 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 70, "duration_s": 17, "mean_score": 85.0, "mean_raw": 15.0, "scores_per_judge": [ 85 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 }, { "category": "mishear-severe", "points": 5 }, { "category": "content-drop", "points": 3 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 71, "duration_s": 51, "mean_score": 95.0, "mean_raw": 5.0, "scores_per_judge": [ 95 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "content-drop", "points": 3 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 72, "duration_s": 54, "mean_score": 98.0, "mean_raw": 2.0, "scores_per_judge": [ 98 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 2 } ] }, "judge_disagreement": 0 }, { "idx": 73, "duration_s": 56, "mean_score": 81.0, "mean_raw": 19.0, "scores_per_judge": [ 81 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "hallucination", "points": 15 }, { "category": "mishear-minor", "points": 2 }, { "category": "filler", "points": 2 } ] }, "judge_disagreement": 0 }, { "idx": 74, "duration_s": 45, "mean_score": 95.0, "mean_raw": 5.0, "scores_per_judge": [ 95 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 75, "duration_s": 74, "mean_score": 90.0, "mean_raw": 10.0, "scores_per_judge": [ 90 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 76, "duration_s": 58, "mean_score": 71.0, "mean_raw": 29.0, "scores_per_judge": [ 71 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "semantic-inversion", "points": 20 }, { "category": "proper-noun", "points": 6 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 77, "duration_s": 130, "mean_score": 55.0, "mean_raw": 45.0, "scores_per_judge": [ 55 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "semantic-inversion", "points": 20 }, { "category": "brand", "points": 10 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-minor", "points": 2 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 78, "duration_s": 61, "mean_score": 98.0, "mean_raw": 2.0, "scores_per_judge": [ 98 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 79, "duration_s": 47, "mean_score": 95.0, "mean_raw": 5.0, "scores_per_judge": [ 95 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 3 }, { "category": "mishear-minor", "points": 2 } ] }, "judge_disagreement": 0 }, { "idx": 80, "duration_s": 42, "mean_score": 97.0, "mean_raw": 3.0, "scores_per_judge": [ 97 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 81, "duration_s": 38, "mean_score": 94.0, "mean_raw": 6.0, "scores_per_judge": [ 94 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 82, "duration_s": 38, "mean_score": 96.0, "mean_raw": 4.0, "scores_per_judge": [ 96 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "content-drop", "points": 3 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 83, "duration_s": 37, "mean_score": 96.0, "mean_raw": 4.0, "scores_per_judge": [ 96 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 84, "duration_s": 36, "mean_score": 98.0, "mean_raw": 2.0, "scores_per_judge": [ 98 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 85, "duration_s": 36, "mean_score": 96.0, "mean_raw": 4.0, "scores_per_judge": [ 96 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-minor", "points": 2 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 86, "duration_s": 35, "mean_score": 89.0, "mean_raw": 11.0, "scores_per_judge": [ 89 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "tech", "points": 8 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 87, "duration_s": 34, "mean_score": 86.0, "mean_raw": 14.0, "scores_per_judge": [ 86 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "numeric", "points": 10 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 88, "duration_s": 34, "mean_score": 100.0, "mean_raw": 0.0, "scores_per_judge": [ 100 ], "deductions_per_judge": { "deepseek_flash": [] }, "judge_disagreement": 0 }, { "idx": 89, "duration_s": 34, "mean_score": 94.0, "mean_raw": 6.0, "scores_per_judge": [ 94 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 90, "duration_s": 33, "mean_score": 91.0, "mean_raw": 9.0, "scores_per_judge": [ 91 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 91, "duration_s": 33, "mean_score": 93.0, "mean_raw": 7.0, "scores_per_judge": [ 93 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "mishear-minor", "points": 2 }, { "category": "mishear-minor", "points": 2 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 92, "duration_s": 32, "mean_score": 98.0, "mean_raw": 2.0, "scores_per_judge": [ 98 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 93, "duration_s": 32, "mean_score": 88.0, "mean_raw": 12.0, "scores_per_judge": [ 88 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 }, { "category": "mishear-minor", "points": 2 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 94, "duration_s": 32, "mean_score": 96.0, "mean_raw": 4.0, "scores_per_judge": [ 96 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-minor", "points": 2 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 95, "duration_s": 32, "mean_score": 98.0, "mean_raw": 2.0, "scores_per_judge": [ 98 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 96, "duration_s": 31, "mean_score": 100.0, "mean_raw": 0.0, "scores_per_judge": [ 100 ], "deductions_per_judge": { "deepseek_flash": [] }, "judge_disagreement": 0 }, { "idx": 97, "duration_s": 30, "mean_score": 97.0, "mean_raw": 3.0, "scores_per_judge": [ 97 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 98, "duration_s": 30, "mean_score": 98.0, "mean_raw": 2.0, "scores_per_judge": [ 98 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 99, "duration_s": 30, "mean_score": 97.0, "mean_raw": 3.0, "scores_per_judge": [ 97 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 3 } ] }, "judge_disagreement": 0 }, { "idx": 100, "duration_s": 30, "mean_score": 98.0, "mean_raw": 2.0, "scores_per_judge": [ 98 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 } ], "total_deductions": 1162, "avg_deductions_per_sample": 11.62, "mean_clamped_score": 88.4, "n_judge_sample_pairs": 100, "gt_chars": 22625, "ded_per_100char": 5.14, "fidelity_score": 74.3, "overall_score": 88.4, "overall_raw_deductions": 11.62, "score_per_judge": { "deepseek_flash": 88.4 }, "per_group_deductions": { "meaning-flip": { "deepseek_flash": 205 }, "entity": { "deepseek_flash": 276 }, "cultural": { "deepseek_flash": 97 }, "fidelity": { "deepseek_flash": 584 } }, "per_bucket": { "general": { "n_samples": 40, "mean_raw": 13.7, "mean_score": 86.3, "total_raw": 548.0, "idxs": [ 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40 ], "worst_idx": 40 }, "idiom-test": { "n_samples": 5, "mean_raw": 19.6, "mean_score": 80.4, "total_raw": 98.0, "idxs": [ 41, 42, 43, 44, 45 ], "worst_idx": 45 }, "long-mixed": { "n_samples": 5, "mean_raw": 13.8, "mean_score": 86.2, "total_raw": 69.0, "idxs": [ 46, 47, 48, 49, 50 ], "worst_idx": 50 } }, "per_group_per_100char": { "meaning-flip": 0.906, "entity": 1.22, "cultural": 0.429, "fidelity": 2.581 }, "per_group_fidelity": { "meaning-flip": 78.3, "entity": 78.4, "cultural": 74.1, "fidelity": 69.8 } }, "gemini_3_6_flash_stt": { "scores_by_judge": { "deepseek_flash": [ 79, 100, 99, 98, 61, 94, 97, 100, 96, 90, 96, 82, 70, 73, 91, 85, 89, 73, 57, 75, 100, 86, 100, 95, 77, 97, 93, 82, 95, 95, 79, 95, 90, 85, 77, 82, 78, 79, 76, 36, 84, 72, 87, 79, 65, 91, 81, 70, 87, 81, 79, 95, 94, 93, 96, 96, 84, 97, 88, 61, 96, 100, 82, 94, 98, 93, 95, 87, 95, 80, 77, 85, 71, 96, 89, 77, 49, 88, 88, 90, 92, 90, 93, 95, 90, 88, 81, 83, 99, 95, 91, 93, 80, 78, 94, 98, 100, 98, 90, 98 ] }, "per_category_deductions": { "polarity": { "deepseek_flash": 70 }, "semantic-inversion": { "deepseek_flash": 120 }, "hallucination": { "deepseek_flash": 45 }, "numeric": { "deepseek_flash": 40 }, "brand": { "deepseek_flash": 90 }, "tech": { "deepseek_flash": 104 }, "culture": { "deepseek_flash": 88 }, "proper-noun": { "deepseek_flash": 18 }, "dialect": { "deepseek_flash": 15 }, "mishear-severe": { "deepseek_flash": 300 }, "content-drop": { "deepseek_flash": 30 }, "paraphrase": { "deepseek_flash": 33 }, "mishear-minor": { "deepseek_flash": 94 }, "filler": { "deepseek_flash": 315 }, "format": { "deepseek_flash": 0 }, "unscoreable": { "deepseek_flash": 0 } }, "per_tier_deductions": { "tier1": { "deepseek_flash": 275 }, "tier2": { "deepseek_flash": 315 }, "tier3": { "deepseek_flash": 457 }, "tier4": { "deepseek_flash": 315 }, "unscoreable": { "deepseek_flash": 0 } }, "raw_deductions_by_judge": { "deepseek_flash": [ 21, 0, 1, 2, 39, 6, 3, 0, 4, 10, 4, 18, 30, 27, 9, 15, 11, 27, 43, 25, 0, 14, 0, 5, 23, 3, 7, 18, 5, 5, 21, 5, 10, 15, 23, 18, 22, 21, 24, 64, 16, 28, 13, 21, 35, 9, 19, 30, 13, 19, 21, 5, 6, 7, 4, 4, 16, 3, 12, 39, 4, 0, 18, 6, 2, 7, 5, 13, 5, 20, 23, 15, 29, 4, 11, 23, 51, 12, 12, 10, 8, 10, 7, 5, 10, 12, 19, 17, 1, 5, 9, 7, 20, 22, 6, 2, 0, 2, 10, 2 ] }, "sample_details": [ { "idx": 1, "duration_s": 5, "mean_score": 79.0, "mean_raw": 21.0, "scores_per_judge": [ 79 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "semantic-inversion", "points": 20 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 2, "duration_s": 7, "mean_score": 100.0, "mean_raw": 0.0, "scores_per_judge": [ 100 ], "deductions_per_judge": { "deepseek_flash": [] }, "judge_disagreement": 0 }, { "idx": 3, "duration_s": 8, "mean_score": 99.0, "mean_raw": 1.0, "scores_per_judge": [ 99 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 4, "duration_s": 11, "mean_score": 98.0, "mean_raw": 2.0, "scores_per_judge": [ 98 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 5, "duration_s": 16, "mean_score": 61.0, "mean_raw": 39.0, "scores_per_judge": [ 61 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "polarity", "points": 35 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 6, "duration_s": 17, "mean_score": 94.0, "mean_raw": 6.0, "scores_per_judge": [ 94 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 7, "duration_s": 20, "mean_score": 97.0, "mean_raw": 3.0, "scores_per_judge": [ 97 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-minor", "points": 2 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 8, "duration_s": 30, "mean_score": 100.0, "mean_raw": 0.0, "scores_per_judge": [ 100 ], "deductions_per_judge": { "deepseek_flash": [] }, "judge_disagreement": 0 }, { "idx": 9, "duration_s": 32, "mean_score": 96.0, "mean_raw": 4.0, "scores_per_judge": [ 96 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 10, "duration_s": 35, "mean_score": 90.0, "mean_raw": 10.0, "scores_per_judge": [ 90 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-minor", "points": 2 }, { "category": "content-drop", "points": 3 }, { "category": "filler", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 11, "duration_s": 40, "mean_score": 96.0, "mean_raw": 4.0, "scores_per_judge": [ 96 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 12, "duration_s": 46, "mean_score": 82.0, "mean_raw": 18.0, "scores_per_judge": [ 82 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "culture", "points": 8 }, { "category": "paraphrase", "points": 3 }, { "category": "mishear-minor", "points": 2 }, { "category": "filler", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 13, "duration_s": 60, "mean_score": 70.0, "mean_raw": 30.0, "scores_per_judge": [ 70 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "brand", "points": 10 }, { "category": "tech", "points": 8 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-minor", "points": 2 }, { "category": "filler", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 14, "duration_s": 64, "mean_score": 73.0, "mean_raw": 27.0, "scores_per_judge": [ 73 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "proper-noun", "points": 6 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-severe", "points": 5 }, { "category": "content-drop", "points": 3 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 15, "duration_s": 69, "mean_score": 91.0, "mean_raw": 9.0, "scores_per_judge": [ 91 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-minor", "points": 2 }, { "category": "mishear-minor", "points": 2 }, { "category": "filler", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 16, "duration_s": 81, "mean_score": 85.0, "mean_raw": 15.0, "scores_per_judge": [ 85 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 }, { "category": "mishear-minor", "points": 2 }, { "category": "mishear-minor", "points": 2 }, { "category": "mishear-minor", "points": 2 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 17, "duration_s": 100, "mean_score": 89.0, "mean_raw": 11.0, "scores_per_judge": [ 89 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "tech", "points": 8 } ] }, "judge_disagreement": 0 }, { "idx": 18, "duration_s": 125, "mean_score": 73.0, "mean_raw": 27.0, "scores_per_judge": [ 73 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-minor", "points": 2 }, { "category": "numeric", "points": 10 }, { "category": "filler", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 19, "duration_s": 133, "mean_score": 57.0, "mean_raw": 43.0, "scores_per_judge": [ 57 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 5 }, { "category": "tech", "points": 8 }, { "category": "semantic-inversion", "points": 20 }, { "category": "brand", "points": 10 } ] }, "judge_disagreement": 0 }, { "idx": 20, "duration_s": 164, "mean_score": 75.0, "mean_raw": 25.0, "scores_per_judge": [ 75 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-minor", "points": 2 }, { "category": "mishear-minor", "points": 2 }, { "category": "paraphrase", "points": 3 }, { "category": "content-drop", "points": 3 }, { "category": "filler", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 21, "duration_s": 5, "mean_score": 100.0, "mean_raw": 0.0, "scores_per_judge": [ 100 ], "deductions_per_judge": { "deepseek_flash": [] }, "judge_disagreement": 0 }, { "idx": 22, "duration_s": 7, "mean_score": 86.0, "mean_raw": 14.0, "scores_per_judge": [ 86 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "brand", "points": 10 }, { "category": "paraphrase", "points": 3 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 23, "duration_s": 10, "mean_score": 100.0, "mean_raw": 0.0, "scores_per_judge": [ 100 ], "deductions_per_judge": { "deepseek_flash": [] }, "judge_disagreement": 0 }, { "idx": 24, "duration_s": 14, "mean_score": 95.0, "mean_raw": 5.0, "scores_per_judge": [ 95 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 25, "duration_s": 16, "mean_score": 77.0, "mean_raw": 23.0, "scores_per_judge": [ 77 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "semantic-inversion", "points": 20 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 26, "duration_s": 23, "mean_score": 97.0, "mean_raw": 3.0, "scores_per_judge": [ 97 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-minor", "points": 2 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 27, "duration_s": 25, "mean_score": 93.0, "mean_raw": 7.0, "scores_per_judge": [ 93 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 28, "duration_s": 30, "mean_score": 82.0, "mean_raw": 18.0, "scores_per_judge": [ 82 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 29, "duration_s": 34, "mean_score": 95.0, "mean_raw": 5.0, "scores_per_judge": [ 95 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 30, "duration_s": 38, "mean_score": 95.0, "mean_raw": 5.0, "scores_per_judge": [ 95 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "paraphrase", "points": 3 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 31, "duration_s": 44, "mean_score": 79.0, "mean_raw": 21.0, "scores_per_judge": [ 79 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "hallucination", "points": 15 }, { "category": "mishear-minor", "points": 2 }, { "category": "mishear-minor", "points": 2 }, { "category": "filler", "points": 2 } ] }, "judge_disagreement": 0 }, { "idx": 32, "duration_s": 50, "mean_score": 95.0, "mean_raw": 5.0, "scores_per_judge": [ 95 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 33, "duration_s": 62, "mean_score": 90.0, "mean_raw": 10.0, "scores_per_judge": [ 90 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 34, "duration_s": 66, "mean_score": 85.0, "mean_raw": 15.0, "scores_per_judge": [ 85 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "proper-noun", "points": 6 }, { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 4 } ] }, "judge_disagreement": 0 }, { "idx": 35, "duration_s": 77, "mean_score": 77.0, "mean_raw": 23.0, "scores_per_judge": [ 77 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "tech", "points": 8 }, { "category": "mishear-severe", "points": 5 }, { "category": "content-drop", "points": 3 }, { "category": "mishear-minor", "points": 2 }, { "category": "filler", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 36, "duration_s": 94, "mean_score": 82.0, "mean_raw": 18.0, "scores_per_judge": [ 82 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 }, { "category": "tech", "points": 8 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 37, "duration_s": 106, "mean_score": 78.0, "mean_raw": 22.0, "scores_per_judge": [ 78 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-minor", "points": 2 }, { "category": "filler", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 38, "duration_s": 126, "mean_score": 79.0, "mean_raw": 21.0, "scores_per_judge": [ 79 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 }, { "category": "paraphrase", "points": 3 }, { "category": "mishear-minor", "points": 2 }, { "category": "mishear-minor", "points": 2 }, { "category": "mishear-minor", "points": 2 }, { "category": "mishear-minor", "points": 2 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 39, "duration_s": 158, "mean_score": 76.0, "mean_raw": 24.0, "scores_per_judge": [ 76 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "brand", "points": 10 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-minor", "points": 2 }, { "category": "mishear-minor", "points": 2 }, { "category": "filler", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 40, "duration_s": 279, "mean_score": 36.0, "mean_raw": 64.0, "scores_per_judge": [ 36 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "semantic-inversion", "points": 20 }, { "category": "numeric", "points": 10 }, { "category": "culture", "points": 8 }, { "category": "tech", "points": 8 }, { "category": "dialect", "points": 5 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-severe", "points": 5 }, { "category": "paraphrase", "points": 3 } ] }, "judge_disagreement": 0 }, { "idx": 41, "duration_s": 34, "mean_score": 84.0, "mean_raw": 16.0, "scores_per_judge": [ 84 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "culture", "points": 8 }, { "category": "paraphrase", "points": 3 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 42, "duration_s": 43, "mean_score": 72.0, "mean_raw": 28.0, "scores_per_judge": [ 72 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "culture", "points": 8 }, { "category": "mishear-minor", "points": 2 }, { "category": "culture", "points": 8 }, { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 43, "duration_s": 29, "mean_score": 87.0, "mean_raw": 13.0, "scores_per_judge": [ 87 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "culture", "points": 8 }, { "category": "filler", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 44, "duration_s": 71, "mean_score": 79.0, "mean_raw": 21.0, "scores_per_judge": [ 79 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "culture", "points": 8 }, { "category": "culture", "points": 8 }, { "category": "filler", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 45, "duration_s": 46, "mean_score": 65.0, "mean_raw": 35.0, "scores_per_judge": [ 65 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "dialect", "points": 5 }, { "category": "dialect", "points": 5 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 46, "duration_s": 62, "mean_score": 91.0, "mean_raw": 9.0, "scores_per_judge": [ 91 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-minor", "points": 2 }, { "category": "mishear-minor", "points": 2 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 47, "duration_s": 67, "mean_score": 81.0, "mean_raw": 19.0, "scores_per_judge": [ 81 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "brand", "points": 10 }, { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 48, "duration_s": 72, "mean_score": 70.0, "mean_raw": 30.0, "scores_per_judge": [ 70 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "brand", "points": 10 }, { "category": "brand", "points": 10 }, { "category": "content-drop", "points": 3 }, { "category": "mishear-minor", "points": 2 }, { "category": "filler", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 49, "duration_s": 87, "mean_score": 87.0, "mean_raw": 13.0, "scores_per_judge": [ 87 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "proper-noun", "points": 6 }, { "category": "mishear-minor", "points": 2 }, { "category": "filler", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 50, "duration_s": 106, "mean_score": 81.0, "mean_raw": 19.0, "scores_per_judge": [ 81 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "culture", "points": 8 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-minor", "points": 2 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 51, "duration_s": 33, "mean_score": 79.0, "mean_raw": 21.0, "scores_per_judge": [ 79 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 1 }, { "category": "mishear-severe", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 52, "duration_s": 25, "mean_score": 95.0, "mean_raw": 5.0, "scores_per_judge": [ 95 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 53, "duration_s": 35, "mean_score": 94.0, "mean_raw": 6.0, "scores_per_judge": [ 94 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 54, "duration_s": 36, "mean_score": 93.0, "mean_raw": 7.0, "scores_per_judge": [ 93 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-minor", "points": 2 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 55, "duration_s": 33, "mean_score": 96.0, "mean_raw": 4.0, "scores_per_judge": [ 96 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 56, "duration_s": 15, "mean_score": 96.0, "mean_raw": 4.0, "scores_per_judge": [ 96 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 57, "duration_s": 38, "mean_score": 84.0, "mean_raw": 16.0, "scores_per_judge": [ 84 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 }, { "category": "tech", "points": 8 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 58, "duration_s": 15, "mean_score": 97.0, "mean_raw": 3.0, "scores_per_judge": [ 97 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 59, "duration_s": 28, "mean_score": 88.0, "mean_raw": 12.0, "scores_per_judge": [ 88 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 }, { "category": "mishear-minor", "points": 2 }, { "category": "filler", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 60, "duration_s": 18, "mean_score": 61.0, "mean_raw": 39.0, "scores_per_judge": [ 61 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "polarity", "points": 35 }, { "category": "filler", "points": 4 } ] }, "judge_disagreement": 0 }, { "idx": 61, "duration_s": 19, "mean_score": 96.0, "mean_raw": 4.0, "scores_per_judge": [ 96 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "content-drop", "points": 3 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 62, "duration_s": 17, "mean_score": 100.0, "mean_raw": 0.0, "scores_per_judge": [ 100 ], "deductions_per_judge": { "deepseek_flash": [] }, "judge_disagreement": 0 }, { "idx": 63, "duration_s": 21, "mean_score": 82.0, "mean_raw": 18.0, "scores_per_judge": [ 82 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "hallucination", "points": 15 }, { "category": "filler", "points": 3 } ] }, "judge_disagreement": 0 }, { "idx": 64, "duration_s": 23, "mean_score": 94.0, "mean_raw": 6.0, "scores_per_judge": [ 94 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "paraphrase", "points": 3 } ] }, "judge_disagreement": 0 }, { "idx": 65, "duration_s": 20, "mean_score": 98.0, "mean_raw": 2.0, "scores_per_judge": [ 98 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 2 } ] }, "judge_disagreement": 0 }, { "idx": 66, "duration_s": 24, "mean_score": 93.0, "mean_raw": 7.0, "scores_per_judge": [ 93 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 67, "duration_s": 16, "mean_score": 95.0, "mean_raw": 5.0, "scores_per_judge": [ 95 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 68, "duration_s": 18, "mean_score": 87.0, "mean_raw": 13.0, "scores_per_judge": [ 87 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "brand", "points": 10 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 69, "duration_s": 18, "mean_score": 95.0, "mean_raw": 5.0, "scores_per_judge": [ 95 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 70, "duration_s": 17, "mean_score": 80.0, "mean_raw": 20.0, "scores_per_judge": [ 80 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "tech", "points": 8 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-minor", "points": 2 }, { "category": "filler", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 71, "duration_s": 51, "mean_score": 77.0, "mean_raw": 23.0, "scores_per_judge": [ 77 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 }, { "category": "tech", "points": 8 }, { "category": "mishear-minor", "points": 2 }, { "category": "content-drop", "points": 3 }, { "category": "filler", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 72, "duration_s": 54, "mean_score": 85.0, "mean_raw": 15.0, "scores_per_judge": [ 85 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "tech", "points": 8 }, { "category": "mishear-minor", "points": 2 }, { "category": "mishear-minor", "points": 2 }, { "category": "mishear-minor", "points": 2 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 73, "duration_s": 56, "mean_score": 71.0, "mean_raw": 29.0, "scores_per_judge": [ 71 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 }, { "category": "semantic-inversion", "points": 20 }, { "category": "paraphrase", "points": 3 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 74, "duration_s": 45, "mean_score": 96.0, "mean_raw": 4.0, "scores_per_judge": [ 96 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 4 } ] }, "judge_disagreement": 0 }, { "idx": 75, "duration_s": 74, "mean_score": 89.0, "mean_raw": 11.0, "scores_per_judge": [ 89 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "content-drop", "points": 3 }, { "category": "paraphrase", "points": 3 }, { "category": "filler", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 76, "duration_s": 58, "mean_score": 77.0, "mean_raw": 23.0, "scores_per_judge": [ 77 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "culture", "points": 8 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 77, "duration_s": 130, "mean_score": 49.0, "mean_raw": 51.0, "scores_per_judge": [ 49 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "semantic-inversion", "points": 20 }, { "category": "brand", "points": 10 }, { "category": "tech", "points": 8 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 78, "duration_s": 61, "mean_score": 88.0, "mean_raw": 12.0, "scores_per_judge": [ 88 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 }, { "category": "mishear-minor", "points": 2 }, { "category": "mishear-minor", "points": 2 }, { "category": "filler", "points": 3 } ] }, "judge_disagreement": 0 }, { "idx": 79, "duration_s": 47, "mean_score": 88.0, "mean_raw": 12.0, "scores_per_judge": [ 88 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "content-drop", "points": 3 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "mishear-severe", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 80, "duration_s": 42, "mean_score": 90.0, "mean_raw": 10.0, "scores_per_judge": [ 90 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 81, "duration_s": 38, "mean_score": 92.0, "mean_raw": 8.0, "scores_per_judge": [ 92 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 82, "duration_s": 38, "mean_score": 90.0, "mean_raw": 10.0, "scores_per_judge": [ 90 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "tech", "points": 8 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 83, "duration_s": 37, "mean_score": 93.0, "mean_raw": 7.0, "scores_per_judge": [ 93 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-minor", "points": 2 }, { "category": "filler", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 84, "duration_s": 36, "mean_score": 95.0, "mean_raw": 5.0, "scores_per_judge": [ 95 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 85, "duration_s": 36, "mean_score": 90.0, "mean_raw": 10.0, "scores_per_judge": [ 90 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 }, { "category": "mishear-minor", "points": 2 }, { "category": "filler", "points": 3 } ] }, "judge_disagreement": 0 }, { "idx": 86, "duration_s": 35, "mean_score": 88.0, "mean_raw": 12.0, "scores_per_judge": [ 88 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "tech", "points": 8 }, { "category": "mishear-minor", "points": 2 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 87, "duration_s": 34, "mean_score": 81.0, "mean_raw": 19.0, "scores_per_judge": [ 81 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "numeric", "points": 10 }, { "category": "mishear-minor", "points": 2 }, { "category": "mishear-minor", "points": 2 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 88, "duration_s": 34, "mean_score": 83.0, "mean_raw": 17.0, "scores_per_judge": [ 83 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "culture", "points": 8 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-minor", "points": 2 }, { "category": "filler", "points": 2 } ] }, "judge_disagreement": 0 }, { "idx": 89, "duration_s": 34, "mean_score": 99.0, "mean_raw": 1.0, "scores_per_judge": [ 99 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 90, "duration_s": 33, "mean_score": 95.0, "mean_raw": 5.0, "scores_per_judge": [ 95 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-minor", "points": 2 }, { "category": "mishear-minor", "points": 2 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 91, "duration_s": 33, "mean_score": 91.0, "mean_raw": 9.0, "scores_per_judge": [ 91 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 92, "duration_s": 32, "mean_score": 93.0, "mean_raw": 7.0, "scores_per_judge": [ 93 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 93, "duration_s": 32, "mean_score": 80.0, "mean_raw": 20.0, "scores_per_judge": [ 80 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "hallucination", "points": 15 }, { "category": "content-drop", "points": 3 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 94, "duration_s": 32, "mean_score": 78.0, "mean_raw": 22.0, "scores_per_judge": [ 78 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "numeric", "points": 10 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-minor", "points": 2 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 95, "duration_s": 32, "mean_score": 94.0, "mean_raw": 6.0, "scores_per_judge": [ 94 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "paraphrase", "points": 3 } ] }, "judge_disagreement": 0 }, { "idx": 96, "duration_s": 31, "mean_score": 98.0, "mean_raw": 2.0, "scores_per_judge": [ 98 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 97, "duration_s": 30, "mean_score": 100.0, "mean_raw": 0.0, "scores_per_judge": [ 100 ], "deductions_per_judge": { "deepseek_flash": [] }, "judge_disagreement": 0 }, { "idx": 98, "duration_s": 30, "mean_score": 98.0, "mean_raw": 2.0, "scores_per_judge": [ 98 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 99, "duration_s": 30, "mean_score": 90.0, "mean_raw": 10.0, "scores_per_judge": [ 90 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 }, { "category": "mishear-severe", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 100, "duration_s": 30, "mean_score": 98.0, "mean_raw": 2.0, "scores_per_judge": [ 98 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 } ], "total_deductions": 1362, "avg_deductions_per_sample": 13.62, "mean_clamped_score": 86.4, "n_judge_sample_pairs": 100, "gt_chars": 22625, "ded_per_100char": 6.02, "fidelity_score": 69.9, "overall_score": 86.4, "overall_raw_deductions": 13.62, "score_per_judge": { "deepseek_flash": 86.4 }, "per_group_deductions": { "meaning-flip": { "deepseek_flash": 275 }, "entity": { "deepseek_flash": 212 }, "cultural": { "deepseek_flash": 103 }, "fidelity": { "deepseek_flash": 772 } }, "per_bucket": { "general": { "n_samples": 40, "mean_raw": 14.9, "mean_score": 85.0, "total_raw": 598.0, "idxs": [ 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40 ], "worst_idx": 40 }, "idiom-test": { "n_samples": 5, "mean_raw": 22.6, "mean_score": 77.4, "total_raw": 113.0, "idxs": [ 41, 42, 43, 44, 45 ], "worst_idx": 45 }, "long-mixed": { "n_samples": 5, "mean_raw": 18.0, "mean_score": 82.0, "total_raw": 90.0, "idxs": [ 46, 47, 48, 49, 50 ], "worst_idx": 48 } }, "per_group_per_100char": { "meaning-flip": 1.215, "entity": 0.937, "cultural": 0.455, "fidelity": 3.412 }, "per_group_fidelity": { "meaning-flip": 70.9, "entity": 83.4, "cultural": 72.5, "fidelity": 60.0 } }, "gemini_3_7_flash_stt": { "scores_by_judge": { "deepseek_flash": [ 99, 99, null, 100, 98, 98, 98, 99, 98, 97, 88, 95, 83, 85, 90, 89, 77, 75, 70, 82, 99, 87, 100, 99, 98, 62, 98, 87, 98, 91, 87, 95, 95, 80, 93, 98, 90, 88, 85, 32, 89, 77, 87, 83, null, 93, 88, 92, 100, 83, 94, 96, 97, null, 90, 57, 60, 97, 94, 62, 100, 100, 95, 96, 100, 85, 97, 100, 93, 98, 94, 92, 79, 95, 87, 89, 63, 93, 94, 97, 97, 98, 91, 95, 91, 89, 97, 100, 97, 82, 98, 99, 83, 87, 99, 98, 98, 98, 99, 97 ] }, "per_category_deductions": { "polarity": { "deepseek_flash": 105 }, "semantic-inversion": { "deepseek_flash": 80 }, "hallucination": { "deepseek_flash": 30 }, "numeric": { "deepseek_flash": 40 }, "brand": { "deepseek_flash": 50 }, "tech": { "deepseek_flash": 40 }, "culture": { "deepseek_flash": 56 }, "proper-noun": { "deepseek_flash": 24 }, "dialect": { "deepseek_flash": 0 }, "mishear-severe": { "deepseek_flash": 180 }, "content-drop": { "deepseek_flash": 12 }, "paraphrase": { "deepseek_flash": 15 }, "mishear-minor": { "deepseek_flash": 56 }, "filler": { "deepseek_flash": 251 }, "format": { "deepseek_flash": 0 }, "unscoreable": { "deepseek_flash": 0 } }, "per_tier_deductions": { "tier1": { "deepseek_flash": 255 }, "tier2": { "deepseek_flash": 170 }, "tier3": { "deepseek_flash": 263 }, "tier4": { "deepseek_flash": 251 }, "unscoreable": { "deepseek_flash": 0 } }, "raw_deductions_by_judge": { "deepseek_flash": [ 1, 1, null, 0, 2, 2, 2, 1, 2, 3, 12, 5, 17, 15, 10, 11, 23, 25, 30, 18, 1, 13, 0, 1, 2, 38, 2, 13, 2, 9, 13, 5, 5, 20, 7, 2, 10, 12, 15, 68, 11, 23, 13, 17, null, 7, 12, 8, 0, 17, 6, 4, 3, null, 10, 43, 40, 3, 6, 38, 0, 0, 5, 4, 0, 15, 3, 0, 7, 2, 6, 8, 21, 5, 13, 11, 37, 7, 6, 3, 3, 2, 9, 5, 9, 11, 3, 0, 3, 18, 2, 1, 17, 13, 1, 2, 2, 2, 1, 3 ] }, "sample_details": [ { "idx": 1, "duration_s": 5, "mean_score": 99.0, "mean_raw": 1.0, "scores_per_judge": [ 99 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 2, "duration_s": 7, "mean_score": 99.0, "mean_raw": 1.0, "scores_per_judge": [ 99 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 4, "duration_s": 11, "mean_score": 100.0, "mean_raw": 0.0, "scores_per_judge": [ 100 ], "deductions_per_judge": { "deepseek_flash": [] }, "judge_disagreement": 0 }, { "idx": 5, "duration_s": 16, "mean_score": 98.0, "mean_raw": 2.0, "scores_per_judge": [ 98 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 2 } ] }, "judge_disagreement": 0 }, { "idx": 6, "duration_s": 17, "mean_score": 98.0, "mean_raw": 2.0, "scores_per_judge": [ 98 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 7, "duration_s": 20, "mean_score": 98.0, "mean_raw": 2.0, "scores_per_judge": [ 98 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 8, "duration_s": 30, "mean_score": 99.0, "mean_raw": 1.0, "scores_per_judge": [ 99 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 9, "duration_s": 32, "mean_score": 98.0, "mean_raw": 2.0, "scores_per_judge": [ 98 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 2 } ] }, "judge_disagreement": 0 }, { "idx": 10, "duration_s": 35, "mean_score": 97.0, "mean_raw": 3.0, "scores_per_judge": [ 97 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 11, "duration_s": 40, "mean_score": 88.0, "mean_raw": 12.0, "scores_per_judge": [ 88 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 }, { "category": "mishear-minor", "points": 2 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 12, "duration_s": 46, "mean_score": 95.0, "mean_raw": 5.0, "scores_per_judge": [ 95 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 13, "duration_s": 60, "mean_score": 83.0, "mean_raw": 17.0, "scores_per_judge": [ 83 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "brand", "points": 10 }, { "category": "mishear-minor", "points": 2 }, { "category": "filler", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 14, "duration_s": 64, "mean_score": 85.0, "mean_raw": 15.0, "scores_per_judge": [ 85 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 }, { "category": "mishear-minor", "points": 2 }, { "category": "mishear-minor", "points": 2 }, { "category": "mishear-minor", "points": 2 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 15, "duration_s": 69, "mean_score": 90.0, "mean_raw": 10.0, "scores_per_judge": [ 90 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 16, "duration_s": 81, "mean_score": 89.0, "mean_raw": 11.0, "scores_per_judge": [ 89 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "proper-noun", "points": 6 }, { "category": "filler", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 17, "duration_s": 100, "mean_score": 77.0, "mean_raw": 23.0, "scores_per_judge": [ 77 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "brand", "points": 10 }, { "category": "tech", "points": 8 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 18, "duration_s": 125, "mean_score": 75.0, "mean_raw": 25.0, "scores_per_judge": [ 75 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "tech", "points": 8 }, { "category": "mishear-severe", "points": 5 }, { "category": "content-drop", "points": 3 }, { "category": "mishear-minor", "points": 2 }, { "category": "mishear-minor", "points": 2 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 19, "duration_s": 133, "mean_score": 70.0, "mean_raw": 30.0, "scores_per_judge": [ 70 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "brand", "points": 10 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 20, "duration_s": 164, "mean_score": 82.0, "mean_raw": 18.0, "scores_per_judge": [ 82 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 }, { "category": "mishear-minor", "points": 2 }, { "category": "proper-noun", "points": 6 }, { "category": "filler", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 21, "duration_s": 5, "mean_score": 99.0, "mean_raw": 1.0, "scores_per_judge": [ 99 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 22, "duration_s": 7, "mean_score": 87.0, "mean_raw": 13.0, "scores_per_judge": [ 87 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "proper-noun", "points": 6 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-minor", "points": 2 } ] }, "judge_disagreement": 0 }, { "idx": 23, "duration_s": 10, "mean_score": 100.0, "mean_raw": 0.0, "scores_per_judge": [ 100 ], "deductions_per_judge": { "deepseek_flash": [] }, "judge_disagreement": 0 }, { "idx": 24, "duration_s": 14, "mean_score": 99.0, "mean_raw": 1.0, "scores_per_judge": [ 99 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 25, "duration_s": 16, "mean_score": 98.0, "mean_raw": 2.0, "scores_per_judge": [ 98 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 26, "duration_s": 23, "mean_score": 62.0, "mean_raw": 38.0, "scores_per_judge": [ 62 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "polarity", "points": 35 }, { "category": "filler", "points": 1 }, { "category": "mishear-minor", "points": 2 } ] }, "judge_disagreement": 0 }, { "idx": 27, "duration_s": 25, "mean_score": 98.0, "mean_raw": 2.0, "scores_per_judge": [ 98 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 28, "duration_s": 30, "mean_score": 87.0, "mean_raw": 13.0, "scores_per_judge": [ 87 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 }, { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 29, "duration_s": 34, "mean_score": 98.0, "mean_raw": 2.0, "scores_per_judge": [ 98 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 30, "duration_s": 38, "mean_score": 91.0, "mean_raw": 9.0, "scores_per_judge": [ 91 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "culture", "points": 8 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 31, "duration_s": 44, "mean_score": 87.0, "mean_raw": 13.0, "scores_per_judge": [ 87 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "culture", "points": 8 }, { "category": "filler", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 32, "duration_s": 50, "mean_score": 95.0, "mean_raw": 5.0, "scores_per_judge": [ 95 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 33, "duration_s": 62, "mean_score": 95.0, "mean_raw": 5.0, "scores_per_judge": [ 95 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 34, "duration_s": 66, "mean_score": 80.0, "mean_raw": 20.0, "scores_per_judge": [ 80 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "hallucination", "points": 15 }, { "category": "filler", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 35, "duration_s": 77, "mean_score": 93.0, "mean_raw": 7.0, "scores_per_judge": [ 93 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 5 }, { "category": "mishear-minor", "points": 2 } ] }, "judge_disagreement": 0 }, { "idx": 36, "duration_s": 94, "mean_score": 98.0, "mean_raw": 2.0, "scores_per_judge": [ 98 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 37, "duration_s": 106, "mean_score": 90.0, "mean_raw": 10.0, "scores_per_judge": [ 90 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 5 }, { "category": "content-drop", "points": 3 }, { "category": "mishear-minor", "points": 2 } ] }, "judge_disagreement": 0 }, { "idx": 38, "duration_s": 126, "mean_score": 88.0, "mean_raw": 12.0, "scores_per_judge": [ 88 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 }, { "category": "mishear-minor", "points": 2 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 39, "duration_s": 158, "mean_score": 85.0, "mean_raw": 15.0, "scores_per_judge": [ 85 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-minor", "points": 2 }, { "category": "filler", "points": 3 } ] }, "judge_disagreement": 0 }, { "idx": 40, "duration_s": 279, "mean_score": 32.0, "mean_raw": 68.0, "scores_per_judge": [ 32 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "semantic-inversion", "points": 20 }, { "category": "culture", "points": 8 }, { "category": "numeric", "points": 10 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-minor", "points": 2 }, { "category": "paraphrase", "points": 3 } ] }, "judge_disagreement": 0 }, { "idx": 41, "duration_s": 34, "mean_score": 89.0, "mean_raw": 11.0, "scores_per_judge": [ 89 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "culture", "points": 8 }, { "category": "mishear-minor", "points": 2 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 42, "duration_s": 43, "mean_score": 77.0, "mean_raw": 23.0, "scores_per_judge": [ 77 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "semantic-inversion", "points": 20 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 43, "duration_s": 29, "mean_score": 87.0, "mean_raw": 13.0, "scores_per_judge": [ 87 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "culture", "points": 8 }, { "category": "filler", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 44, "duration_s": 71, "mean_score": 83.0, "mean_raw": 17.0, "scores_per_judge": [ 83 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "proper-noun", "points": 6 }, { "category": "culture", "points": 8 }, { "category": "mishear-minor", "points": 2 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 46, "duration_s": 62, "mean_score": 93.0, "mean_raw": 7.0, "scores_per_judge": [ 93 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-minor", "points": 2 }, { "category": "filler", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 47, "duration_s": 67, "mean_score": 88.0, "mean_raw": 12.0, "scores_per_judge": [ 88 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "tech", "points": 8 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 48, "duration_s": 72, "mean_score": 92.0, "mean_raw": 8.0, "scores_per_judge": [ 92 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 49, "duration_s": 87, "mean_score": 100.0, "mean_raw": 0.0, "scores_per_judge": [ 100 ], "deductions_per_judge": { "deepseek_flash": [] }, "judge_disagreement": 0 }, { "idx": 50, "duration_s": 106, "mean_score": 83.0, "mean_raw": 17.0, "scores_per_judge": [ 83 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "culture", "points": 8 }, { "category": "mishear-severe", "points": 5 }, { "category": "paraphrase", "points": 3 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 51, "duration_s": 33, "mean_score": 94.0, "mean_raw": 6.0, "scores_per_judge": [ 94 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 52, "duration_s": 25, "mean_score": 96.0, "mean_raw": 4.0, "scores_per_judge": [ 96 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 53, "duration_s": 35, "mean_score": 97.0, "mean_raw": 3.0, "scores_per_judge": [ 97 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 55, "duration_s": 33, "mean_score": 90.0, "mean_raw": 10.0, "scores_per_judge": [ 90 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 }, { "category": "mishear-minor", "points": 2 }, { "category": "filler", "points": 3 } ] }, "judge_disagreement": 0 }, { "idx": 56, "duration_s": 15, "mean_score": 57.0, "mean_raw": 43.0, "scores_per_judge": [ 57 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "numeric", "points": 10 }, { "category": "numeric", "points": 10 }, { "category": "numeric", "points": 10 }, { "category": "brand", "points": 10 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 57, "duration_s": 38, "mean_score": 60.0, "mean_raw": 40.0, "scores_per_judge": [ 60 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "polarity", "points": 35 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 58, "duration_s": 15, "mean_score": 97.0, "mean_raw": 3.0, "scores_per_judge": [ 97 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 59, "duration_s": 28, "mean_score": 94.0, "mean_raw": 6.0, "scores_per_judge": [ 94 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 60, "duration_s": 18, "mean_score": 62.0, "mean_raw": 38.0, "scores_per_judge": [ 62 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "polarity", "points": 35 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 61, "duration_s": 19, "mean_score": 100.0, "mean_raw": 0.0, "scores_per_judge": [ 100 ], "deductions_per_judge": { "deepseek_flash": [] }, "judge_disagreement": 0 }, { "idx": 62, "duration_s": 17, "mean_score": 100.0, "mean_raw": 0.0, "scores_per_judge": [ 100 ], "deductions_per_judge": { "deepseek_flash": [] }, "judge_disagreement": 0 }, { "idx": 63, "duration_s": 21, "mean_score": 95.0, "mean_raw": 5.0, "scores_per_judge": [ 95 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 64, "duration_s": 23, "mean_score": 96.0, "mean_raw": 4.0, "scores_per_judge": [ 96 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 65, "duration_s": 20, "mean_score": 100.0, "mean_raw": 0.0, "scores_per_judge": [ 100 ], "deductions_per_judge": { "deepseek_flash": [] }, "judge_disagreement": 0 }, { "idx": 66, "duration_s": 24, "mean_score": 85.0, "mean_raw": 15.0, "scores_per_judge": [ 85 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "paraphrase", "points": 3 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 67, "duration_s": 16, "mean_score": 97.0, "mean_raw": 3.0, "scores_per_judge": [ 97 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-minor", "points": 2 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 68, "duration_s": 18, "mean_score": 100.0, "mean_raw": 0.0, "scores_per_judge": [ 100 ], "deductions_per_judge": { "deepseek_flash": [] }, "judge_disagreement": 0 }, { "idx": 69, "duration_s": 18, "mean_score": 93.0, "mean_raw": 7.0, "scores_per_judge": [ 93 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 70, "duration_s": 17, "mean_score": 98.0, "mean_raw": 2.0, "scores_per_judge": [ 98 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 71, "duration_s": 51, "mean_score": 94.0, "mean_raw": 6.0, "scores_per_judge": [ 94 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "content-drop", "points": 3 } ] }, "judge_disagreement": 0 }, { "idx": 72, "duration_s": 54, "mean_score": 92.0, "mean_raw": 8.0, "scores_per_judge": [ 92 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 2 } ] }, "judge_disagreement": 0 }, { "idx": 73, "duration_s": 56, "mean_score": 79.0, "mean_raw": 21.0, "scores_per_judge": [ 79 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "semantic-inversion", "points": 20 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 74, "duration_s": 45, "mean_score": 95.0, "mean_raw": 5.0, "scores_per_judge": [ 95 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 75, "duration_s": 74, "mean_score": 87.0, "mean_raw": 13.0, "scores_per_judge": [ 87 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 }, { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 76, "duration_s": 58, "mean_score": 89.0, "mean_raw": 11.0, "scores_per_judge": [ 89 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 }, { "category": "paraphrase", "points": 3 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 77, "duration_s": 130, "mean_score": 63.0, "mean_raw": 37.0, "scores_per_judge": [ 63 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "semantic-inversion", "points": 20 }, { "category": "brand", "points": 10 }, { "category": "mishear-minor", "points": 2 }, { "category": "filler", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 78, "duration_s": 61, "mean_score": 93.0, "mean_raw": 7.0, "scores_per_judge": [ 93 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 5 }, { "category": "mishear-minor", "points": 2 } ] }, "judge_disagreement": 0 }, { "idx": 79, "duration_s": 47, "mean_score": 94.0, "mean_raw": 6.0, "scores_per_judge": [ 94 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "content-drop", "points": 3 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 80, "duration_s": 42, "mean_score": 97.0, "mean_raw": 3.0, "scores_per_judge": [ 97 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 81, "duration_s": 38, "mean_score": 97.0, "mean_raw": 3.0, "scores_per_judge": [ 97 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-minor", "points": 2 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 82, "duration_s": 38, "mean_score": 98.0, "mean_raw": 2.0, "scores_per_judge": [ 98 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 83, "duration_s": 37, "mean_score": 91.0, "mean_raw": 9.0, "scores_per_judge": [ 91 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 }, { "category": "paraphrase", "points": 3 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 84, "duration_s": 36, "mean_score": 95.0, "mean_raw": 5.0, "scores_per_judge": [ 95 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 85, "duration_s": 36, "mean_score": 91.0, "mean_raw": 9.0, "scores_per_judge": [ 91 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "mishear-minor", "points": 2 }, { "category": "mishear-severe", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 86, "duration_s": 35, "mean_score": 89.0, "mean_raw": 11.0, "scores_per_judge": [ 89 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "tech", "points": 8 }, { "category": "mishear-minor", "points": 2 } ] }, "judge_disagreement": 0 }, { "idx": 87, "duration_s": 34, "mean_score": 97.0, "mean_raw": 3.0, "scores_per_judge": [ 97 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 88, "duration_s": 34, "mean_score": 100.0, "mean_raw": 0.0, "scores_per_judge": [ 100 ], "deductions_per_judge": { "deepseek_flash": [] }, "judge_disagreement": 0 }, { "idx": 89, "duration_s": 34, "mean_score": 97.0, "mean_raw": 3.0, "scores_per_judge": [ 97 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "mishear-minor", "points": 2 } ] }, "judge_disagreement": 0 }, { "idx": 90, "duration_s": 33, "mean_score": 82.0, "mean_raw": 18.0, "scores_per_judge": [ 82 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "hallucination", "points": 15 }, { "category": "mishear-minor", "points": 2 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 91, "duration_s": 33, "mean_score": 98.0, "mean_raw": 2.0, "scores_per_judge": [ 98 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 92, "duration_s": 32, "mean_score": 99.0, "mean_raw": 1.0, "scores_per_judge": [ 99 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 93, "duration_s": 32, "mean_score": 83.0, "mean_raw": 17.0, "scores_per_judge": [ 83 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "tech", "points": 8 }, { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 2 } ] }, "judge_disagreement": 0 }, { "idx": 94, "duration_s": 32, "mean_score": 87.0, "mean_raw": 13.0, "scores_per_judge": [ 87 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 }, { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 3 } ] }, "judge_disagreement": 0 }, { "idx": 95, "duration_s": 32, "mean_score": 99.0, "mean_raw": 1.0, "scores_per_judge": [ 99 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 96, "duration_s": 31, "mean_score": 98.0, "mean_raw": 2.0, "scores_per_judge": [ 98 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 97, "duration_s": 30, "mean_score": 98.0, "mean_raw": 2.0, "scores_per_judge": [ 98 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 98, "duration_s": 30, "mean_score": 98.0, "mean_raw": 2.0, "scores_per_judge": [ 98 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 99, "duration_s": 30, "mean_score": 99.0, "mean_raw": 1.0, "scores_per_judge": [ 99 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 100, "duration_s": 30, "mean_score": 97.0, "mean_raw": 3.0, "scores_per_judge": [ 97 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-minor", "points": 2 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 } ], "total_deductions": 939, "avg_deductions_per_sample": 9.68, "mean_clamped_score": 90.3, "n_judge_sample_pairs": 97, "gt_chars": 22209, "ded_per_100char": 4.23, "fidelity_score": 78.9, "overall_score": 90.3, "overall_raw_deductions": 9.68, "score_per_judge": { "deepseek_flash": 90.3 }, "per_group_deductions": { "meaning-flip": { "deepseek_flash": 255 }, "entity": { "deepseek_flash": 114 }, "cultural": { "deepseek_flash": 56 }, "fidelity": { "deepseek_flash": 514 } }, "per_bucket": { "general": { "n_samples": 39, "mean_raw": 10.7, "mean_score": 89.3, "total_raw": 418.0, "idxs": [ 1, 2, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40 ], "worst_idx": 40 }, "idiom-test": { "n_samples": 4, "mean_raw": 16.0, "mean_score": 84.0, "total_raw": 64.0, "idxs": [ 41, 42, 43, 44 ], "worst_idx": 42 }, "long-mixed": { "n_samples": 5, "mean_raw": 8.8, "mean_score": 91.2, "total_raw": 44.0, "idxs": [ 46, 47, 48, 49, 50 ], "worst_idx": 50 } }, "per_group_per_100char": { "meaning-flip": 1.148, "entity": 0.513, "cultural": 0.252, "fidelity": 2.314 }, "per_group_fidelity": { "meaning-flip": 72.5, "entity": 90.9, "cultural": 84.8, "fidelity": 72.9 } }, "gemini_3_8_flash_stt": { "scores_by_judge": { "deepseek_flash": [ 100, 99, null, 98, 97, 92, 99, 99, 87, 96, 95, 93, 97, 91, 86, 83, 95, 99, 72, 89, 99, 86, 99, 91, 98, 98, 98, 88, 99, 92, 96, 80, 95, 84, 87, 89, 90, 93, 89, 49, 96, 97, 90, 91, 88, 95, 96, 98, 95, 96, 96, 95, 98, null, 92, 97, 64, 98, 90, 63, 99, 100, 98, 96, 97, 95, 99, 100, 95, 90, 96, 97, 80, 97, null, 97, 85, 76, 92, 95, 94, 94, 100, 97, 91, 92, 94, 99, 96, 93, 99, 98, 90, 90, 98, 100, 98, 98, 95, 97 ] }, "per_category_deductions": { "polarity": { "deepseek_flash": 70 }, "semantic-inversion": { "deepseek_flash": 40 }, "hallucination": { "deepseek_flash": 45 }, "numeric": { "deepseek_flash": 10 }, "brand": { "deepseek_flash": 20 }, "tech": { "deepseek_flash": 40 }, "culture": { "deepseek_flash": 40 }, "proper-noun": { "deepseek_flash": 24 }, "dialect": { "deepseek_flash": 0 }, "mishear-severe": { "deepseek_flash": 150 }, "content-drop": { "deepseek_flash": 3 }, "paraphrase": { "deepseek_flash": 27 }, "mishear-minor": { "deepseek_flash": 48 }, "filler": { "deepseek_flash": 183 }, "format": { "deepseek_flash": 1 }, "unscoreable": { "deepseek_flash": 0 } }, "per_tier_deductions": { "tier1": { "deepseek_flash": 165 }, "tier2": { "deepseek_flash": 124 }, "tier3": { "deepseek_flash": 228 }, "tier4": { "deepseek_flash": 184 }, "unscoreable": { "deepseek_flash": 0 } }, "raw_deductions_by_judge": { "deepseek_flash": [ 0, 1, null, 2, 3, 8, 1, 1, 13, 4, 5, 7, 3, 9, 14, 17, 5, 1, 28, 11, 1, 14, 1, 9, 2, 2, 2, 12, 1, 8, 4, 20, 5, 16, 13, 11, 10, 7, 11, 51, 4, 3, 10, 9, 12, 5, 4, 2, 5, 4, 4, 5, 2, null, 8, 3, 36, 2, 10, 37, 1, 0, 2, 4, 3, 5, 1, 0, 5, 10, 4, 3, 20, 3, null, 3, 15, 24, 8, 5, 6, 6, 0, 3, 9, 8, 6, 1, 4, 7, 1, 2, 10, 10, 2, 0, 2, 2, 5, 3 ] }, "sample_details": [ { "idx": 1, "duration_s": 5, "mean_score": 100.0, "mean_raw": 0.0, "scores_per_judge": [ 100 ], "deductions_per_judge": { "deepseek_flash": [] }, "judge_disagreement": 0 }, { "idx": 2, "duration_s": 7, "mean_score": 99.0, "mean_raw": 1.0, "scores_per_judge": [ 99 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 4, "duration_s": 11, "mean_score": 98.0, "mean_raw": 2.0, "scores_per_judge": [ 98 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-minor", "points": 2 } ] }, "judge_disagreement": 0 }, { "idx": 5, "duration_s": 16, "mean_score": 97.0, "mean_raw": 3.0, "scores_per_judge": [ 97 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 3 } ] }, "judge_disagreement": 0 }, { "idx": 6, "duration_s": 17, "mean_score": 92.0, "mean_raw": 8.0, "scores_per_judge": [ 92 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 }, { "category": "paraphrase", "points": 3 } ] }, "judge_disagreement": 0 }, { "idx": 7, "duration_s": 20, "mean_score": 99.0, "mean_raw": 1.0, "scores_per_judge": [ 99 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 8, "duration_s": 30, "mean_score": 99.0, "mean_raw": 1.0, "scores_per_judge": [ 99 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 9, "duration_s": 32, "mean_score": 87.0, "mean_raw": 13.0, "scores_per_judge": [ 87 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "proper-noun", "points": 6 }, { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 10, "duration_s": 35, "mean_score": 96.0, "mean_raw": 4.0, "scores_per_judge": [ 96 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 11, "duration_s": 40, "mean_score": 95.0, "mean_raw": 5.0, "scores_per_judge": [ 95 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 12, "duration_s": 46, "mean_score": 93.0, "mean_raw": 7.0, "scores_per_judge": [ 93 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "proper-noun", "points": 6 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 13, "duration_s": 60, "mean_score": 97.0, "mean_raw": 3.0, "scores_per_judge": [ 97 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 14, "duration_s": 64, "mean_score": 91.0, "mean_raw": 9.0, "scores_per_judge": [ 91 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 }, { "category": "mishear-minor", "points": 2 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 15, "duration_s": 69, "mean_score": 86.0, "mean_raw": 14.0, "scores_per_judge": [ 86 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 }, { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 4 } ] }, "judge_disagreement": 0 }, { "idx": 16, "duration_s": 81, "mean_score": 83.0, "mean_raw": 17.0, "scores_per_judge": [ 83 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "proper-noun", "points": 6 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-minor", "points": 2 }, { "category": "mishear-minor", "points": 2 }, { "category": "filler", "points": 2 } ] }, "judge_disagreement": 0 }, { "idx": 17, "duration_s": 100, "mean_score": 95.0, "mean_raw": 5.0, "scores_per_judge": [ 95 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 18, "duration_s": 125, "mean_score": 99.0, "mean_raw": 1.0, "scores_per_judge": [ 99 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 19, "duration_s": 133, "mean_score": 72.0, "mean_raw": 28.0, "scores_per_judge": [ 72 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 }, { "category": "culture", "points": 8 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 20, "duration_s": 164, "mean_score": 89.0, "mean_raw": 11.0, "scores_per_judge": [ 89 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "proper-noun", "points": 6 }, { "category": "filler", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 21, "duration_s": 5, "mean_score": 99.0, "mean_raw": 1.0, "scores_per_judge": [ 99 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "format", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 22, "duration_s": 7, "mean_score": 86.0, "mean_raw": 14.0, "scores_per_judge": [ 86 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "brand", "points": 10 }, { "category": "paraphrase", "points": 3 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 23, "duration_s": 10, "mean_score": 99.0, "mean_raw": 1.0, "scores_per_judge": [ 99 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 24, "duration_s": 14, "mean_score": 91.0, "mean_raw": 9.0, "scores_per_judge": [ 91 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "tech", "points": 8 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 25, "duration_s": 16, "mean_score": 98.0, "mean_raw": 2.0, "scores_per_judge": [ 98 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 2 } ] }, "judge_disagreement": 0 }, { "idx": 26, "duration_s": 23, "mean_score": 98.0, "mean_raw": 2.0, "scores_per_judge": [ 98 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-minor", "points": 2 } ] }, "judge_disagreement": 0 }, { "idx": 27, "duration_s": 25, "mean_score": 98.0, "mean_raw": 2.0, "scores_per_judge": [ 98 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 28, "duration_s": 30, "mean_score": 88.0, "mean_raw": 12.0, "scores_per_judge": [ 88 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 }, { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 2 } ] }, "judge_disagreement": 0 }, { "idx": 29, "duration_s": 34, "mean_score": 99.0, "mean_raw": 1.0, "scores_per_judge": [ 99 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 30, "duration_s": 38, "mean_score": 92.0, "mean_raw": 8.0, "scores_per_judge": [ 92 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "culture", "points": 8 } ] }, "judge_disagreement": 0 }, { "idx": 31, "duration_s": 44, "mean_score": 96.0, "mean_raw": 4.0, "scores_per_judge": [ 96 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-minor", "points": 2 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 32, "duration_s": 50, "mean_score": 80.0, "mean_raw": 20.0, "scores_per_judge": [ 80 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "semantic-inversion", "points": 20 } ] }, "judge_disagreement": 0 }, { "idx": 33, "duration_s": 62, "mean_score": 95.0, "mean_raw": 5.0, "scores_per_judge": [ 95 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 34, "duration_s": 66, "mean_score": 84.0, "mean_raw": 16.0, "scores_per_judge": [ 84 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "hallucination", "points": 15 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 35, "duration_s": 77, "mean_score": 87.0, "mean_raw": 13.0, "scores_per_judge": [ 87 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "tech", "points": 8 }, { "category": "filler", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 36, "duration_s": 94, "mean_score": 89.0, "mean_raw": 11.0, "scores_per_judge": [ 89 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 }, { "category": "paraphrase", "points": 3 }, { "category": "mishear-minor", "points": 2 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 37, "duration_s": 106, "mean_score": 90.0, "mean_raw": 10.0, "scores_per_judge": [ 90 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 38, "duration_s": 126, "mean_score": 93.0, "mean_raw": 7.0, "scores_per_judge": [ 93 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-minor", "points": 2 }, { "category": "mishear-severe", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 39, "duration_s": 158, "mean_score": 89.0, "mean_raw": 11.0, "scores_per_judge": [ 89 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-minor", "points": 2 }, { "category": "mishear-minor", "points": 2 }, { "category": "mishear-minor", "points": 2 }, { "category": "filler", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 40, "duration_s": 279, "mean_score": 49.0, "mean_raw": 51.0, "scores_per_judge": [ 49 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "semantic-inversion", "points": 20 }, { "category": "numeric", "points": 10 }, { "category": "culture", "points": 8 }, { "category": "mishear-severe", "points": 5 }, { "category": "paraphrase", "points": 3 }, { "category": "paraphrase", "points": 3 }, { "category": "mishear-minor", "points": 2 } ] }, "judge_disagreement": 0 }, { "idx": 41, "duration_s": 34, "mean_score": 96.0, "mean_raw": 4.0, "scores_per_judge": [ 96 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 42, "duration_s": 43, "mean_score": 97.0, "mean_raw": 3.0, "scores_per_judge": [ 97 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 3 } ] }, "judge_disagreement": 0 }, { "idx": 43, "duration_s": 29, "mean_score": 90.0, "mean_raw": 10.0, "scores_per_judge": [ 90 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "culture", "points": 8 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 44, "duration_s": 71, "mean_score": 91.0, "mean_raw": 9.0, "scores_per_judge": [ 91 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "culture", "points": 8 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 45, "duration_s": 46, "mean_score": 88.0, "mean_raw": 12.0, "scores_per_judge": [ 88 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-minor", "points": 2 } ] }, "judge_disagreement": 0 }, { "idx": 46, "duration_s": 62, "mean_score": 95.0, "mean_raw": 5.0, "scores_per_judge": [ 95 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 47, "duration_s": 67, "mean_score": 96.0, "mean_raw": 4.0, "scores_per_judge": [ 96 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 48, "duration_s": 72, "mean_score": 98.0, "mean_raw": 2.0, "scores_per_judge": [ 98 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-minor", "points": 2 } ] }, "judge_disagreement": 0 }, { "idx": 49, "duration_s": 87, "mean_score": 95.0, "mean_raw": 5.0, "scores_per_judge": [ 95 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 50, "duration_s": 106, "mean_score": 96.0, "mean_raw": 4.0, "scores_per_judge": [ 96 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-minor", "points": 2 }, { "category": "mishear-minor", "points": 2 } ] }, "judge_disagreement": 0 }, { "idx": 51, "duration_s": 33, "mean_score": 96.0, "mean_raw": 4.0, "scores_per_judge": [ 96 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-minor", "points": 2 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 52, "duration_s": 25, "mean_score": 95.0, "mean_raw": 5.0, "scores_per_judge": [ 95 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 53, "duration_s": 35, "mean_score": 98.0, "mean_raw": 2.0, "scores_per_judge": [ 98 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 55, "duration_s": 33, "mean_score": 92.0, "mean_raw": 8.0, "scores_per_judge": [ 92 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 56, "duration_s": 15, "mean_score": 97.0, "mean_raw": 3.0, "scores_per_judge": [ 97 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 3 } ] }, "judge_disagreement": 0 }, { "idx": 57, "duration_s": 38, "mean_score": 64.0, "mean_raw": 36.0, "scores_per_judge": [ 64 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "polarity", "points": 35 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 58, "duration_s": 15, "mean_score": 98.0, "mean_raw": 2.0, "scores_per_judge": [ 98 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 59, "duration_s": 28, "mean_score": 90.0, "mean_raw": 10.0, "scores_per_judge": [ 90 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 }, { "category": "mishear-minor", "points": 2 }, { "category": "filler", "points": 3 } ] }, "judge_disagreement": 0 }, { "idx": 60, "duration_s": 18, "mean_score": 63.0, "mean_raw": 37.0, "scores_per_judge": [ 63 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "polarity", "points": 35 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 61, "duration_s": 19, "mean_score": 99.0, "mean_raw": 1.0, "scores_per_judge": [ 99 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 62, "duration_s": 17, "mean_score": 100.0, "mean_raw": 0.0, "scores_per_judge": [ 100 ], "deductions_per_judge": { "deepseek_flash": [] }, "judge_disagreement": 0 }, { "idx": 63, "duration_s": 21, "mean_score": 98.0, "mean_raw": 2.0, "scores_per_judge": [ 98 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 64, "duration_s": 23, "mean_score": 96.0, "mean_raw": 4.0, "scores_per_judge": [ 96 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 65, "duration_s": 20, "mean_score": 97.0, "mean_raw": 3.0, "scores_per_judge": [ 97 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "paraphrase", "points": 3 } ] }, "judge_disagreement": 0 }, { "idx": 66, "duration_s": 24, "mean_score": 95.0, "mean_raw": 5.0, "scores_per_judge": [ 95 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 67, "duration_s": 16, "mean_score": 99.0, "mean_raw": 1.0, "scores_per_judge": [ 99 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 68, "duration_s": 18, "mean_score": 100.0, "mean_raw": 0.0, "scores_per_judge": [ 100 ], "deductions_per_judge": { "deepseek_flash": [] }, "judge_disagreement": 0 }, { "idx": 69, "duration_s": 18, "mean_score": 95.0, "mean_raw": 5.0, "scores_per_judge": [ 95 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 70, "duration_s": 17, "mean_score": 90.0, "mean_raw": 10.0, "scores_per_judge": [ 90 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "tech", "points": 8 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 71, "duration_s": 51, "mean_score": 96.0, "mean_raw": 4.0, "scores_per_judge": [ 96 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "paraphrase", "points": 3 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 72, "duration_s": 54, "mean_score": 97.0, "mean_raw": 3.0, "scores_per_judge": [ 97 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 73, "duration_s": 56, "mean_score": 80.0, "mean_raw": 20.0, "scores_per_judge": [ 80 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "hallucination", "points": 15 }, { "category": "mishear-severe", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 74, "duration_s": 45, "mean_score": 97.0, "mean_raw": 3.0, "scores_per_judge": [ 97 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 76, "duration_s": 58, "mean_score": 97.0, "mean_raw": 3.0, "scores_per_judge": [ 97 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "paraphrase", "points": 3 } ] }, "judge_disagreement": 0 }, { "idx": 77, "duration_s": 130, "mean_score": 85.0, "mean_raw": 15.0, "scores_per_judge": [ 85 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "brand", "points": 10 }, { "category": "mishear-severe", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 78, "duration_s": 61, "mean_score": 76.0, "mean_raw": 24.0, "scores_per_judge": [ 76 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "hallucination", "points": 15 }, { "category": "mishear-minor", "points": 2 }, { "category": "mishear-minor", "points": 2 }, { "category": "filler", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 79, "duration_s": 47, "mean_score": 92.0, "mean_raw": 8.0, "scores_per_judge": [ 92 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "content-drop", "points": 3 }, { "category": "paraphrase", "points": 3 } ] }, "judge_disagreement": 0 }, { "idx": 80, "duration_s": 42, "mean_score": 95.0, "mean_raw": 5.0, "scores_per_judge": [ 95 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 81, "duration_s": 38, "mean_score": 94.0, "mean_raw": 6.0, "scores_per_judge": [ 94 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 82, "duration_s": 38, "mean_score": 94.0, "mean_raw": 6.0, "scores_per_judge": [ 94 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "mishear-severe", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 83, "duration_s": 37, "mean_score": 100.0, "mean_raw": 0.0, "scores_per_judge": [ 100 ], "deductions_per_judge": { "deepseek_flash": [] }, "judge_disagreement": 0 }, { "idx": 84, "duration_s": 36, "mean_score": 97.0, "mean_raw": 3.0, "scores_per_judge": [ 97 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-minor", "points": 2 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 85, "duration_s": 36, "mean_score": 91.0, "mean_raw": 9.0, "scores_per_judge": [ 91 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 }, { "category": "mishear-minor", "points": 2 }, { "category": "filler", "points": 2 } ] }, "judge_disagreement": 0 }, { "idx": 86, "duration_s": 35, "mean_score": 92.0, "mean_raw": 8.0, "scores_per_judge": [ 92 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "tech", "points": 8 } ] }, "judge_disagreement": 0 }, { "idx": 87, "duration_s": 34, "mean_score": 94.0, "mean_raw": 6.0, "scores_per_judge": [ 94 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 88, "duration_s": 34, "mean_score": 99.0, "mean_raw": 1.0, "scores_per_judge": [ 99 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 89, "duration_s": 34, "mean_score": 96.0, "mean_raw": 4.0, "scores_per_judge": [ 96 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "mishear-minor", "points": 2 } ] }, "judge_disagreement": 0 }, { "idx": 90, "duration_s": 33, "mean_score": 93.0, "mean_raw": 7.0, "scores_per_judge": [ 93 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 91, "duration_s": 33, "mean_score": 99.0, "mean_raw": 1.0, "scores_per_judge": [ 99 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 92, "duration_s": 32, "mean_score": 98.0, "mean_raw": 2.0, "scores_per_judge": [ 98 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 93, "duration_s": 32, "mean_score": 90.0, "mean_raw": 10.0, "scores_per_judge": [ 90 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "tech", "points": 8 }, { "category": "mishear-minor", "points": 2 } ] }, "judge_disagreement": 0 }, { "idx": 94, "duration_s": 32, "mean_score": 90.0, "mean_raw": 10.0, "scores_per_judge": [ 90 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 95, "duration_s": 32, "mean_score": 98.0, "mean_raw": 2.0, "scores_per_judge": [ 98 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 96, "duration_s": 31, "mean_score": 100.0, "mean_raw": 0.0, "scores_per_judge": [ 100 ], "deductions_per_judge": { "deepseek_flash": [] }, "judge_disagreement": 0 }, { "idx": 97, "duration_s": 30, "mean_score": 98.0, "mean_raw": 2.0, "scores_per_judge": [ 98 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 98, "duration_s": 30, "mean_score": 98.0, "mean_raw": 2.0, "scores_per_judge": [ 98 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 2 } ] }, "judge_disagreement": 0 }, { "idx": 99, "duration_s": 30, "mean_score": 95.0, "mean_raw": 5.0, "scores_per_judge": [ 95 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 100, "duration_s": 30, "mean_score": 97.0, "mean_raw": 3.0, "scores_per_judge": [ 97 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 } ], "total_deductions": 701, "avg_deductions_per_sample": 7.23, "mean_clamped_score": 92.8, "n_judge_sample_pairs": 97, "gt_chars": 22134, "ded_per_100char": 3.17, "fidelity_score": 84.2, "overall_score": 92.8, "overall_raw_deductions": 7.23, "score_per_judge": { "deepseek_flash": 92.8 }, "per_group_deductions": { "meaning-flip": { "deepseek_flash": 165 }, "entity": { "deepseek_flash": 84 }, "cultural": { "deepseek_flash": 40 }, "fidelity": { "deepseek_flash": 412 } }, "per_bucket": { "general": { "n_samples": 39, "mean_raw": 8.5, "mean_score": 91.5, "total_raw": 333.0, "idxs": [ 1, 2, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40 ], "worst_idx": 40 }, "idiom-test": { "n_samples": 5, "mean_raw": 7.6, "mean_score": 92.4, "total_raw": 38.0, "idxs": [ 41, 42, 43, 44, 45 ], "worst_idx": 45 }, "long-mixed": { "n_samples": 5, "mean_raw": 4.0, "mean_score": 96.0, "total_raw": 20.0, "idxs": [ 46, 47, 48, 49, 50 ], "worst_idx": 46 } }, "per_group_per_100char": { "meaning-flip": 0.745, "entity": 0.38, "cultural": 0.181, "fidelity": 1.861 }, "per_group_fidelity": { "meaning-flip": 82.1, "entity": 93.3, "cultural": 89.1, "fidelity": 78.2 } }, "gemini_flash_stt": { "scores_by_judge": { "deepseek_flash": [ 79, 100, 99, 99, 94, 89, 94, 99, 96, 94, 95, 87, 78, 64, 85, 85, 87, 90, 65, 89, 100, 85, 100, 88, 87, 96, 97, 89, 95, 94, 97, 90, 87, 84, 87, 80, 95, 84, 71, 37, 97, 95, 87, 80, 65, 83, 93, 77, 90, 82, 85, 95, 94, 95, 95, 98, 59, 99, 87, 59, 95, 99, 96, 97, 98, 92, 99, 99, 97, 97, 88, 97, 86, 95, 86, 73, 58, 95, 84, 97, 94, 95, 85, 97, 92, 81, 95, 99, 97, 86, 96, 97, 91, 96, 96, 99, 98, 96, 97, 98 ] }, "per_category_deductions": { "polarity": { "deepseek_flash": 70 }, "semantic-inversion": { "deepseek_flash": 100 }, "hallucination": { "deepseek_flash": 45 }, "numeric": { "deepseek_flash": 10 }, "brand": { "deepseek_flash": 70 }, "tech": { "deepseek_flash": 56 }, "culture": { "deepseek_flash": 56 }, "proper-noun": { "deepseek_flash": 42 }, "dialect": { "deepseek_flash": 5 }, "mishear-severe": { "deepseek_flash": 140 }, "content-drop": { "deepseek_flash": 21 }, "paraphrase": { "deepseek_flash": 30 }, "mishear-minor": { "deepseek_flash": 70 }, "filler": { "deepseek_flash": 346 }, "format": { "deepseek_flash": 0 }, "unscoreable": { "deepseek_flash": 0 } }, "per_tier_deductions": { "tier1": { "deepseek_flash": 225 }, "tier2": { "deepseek_flash": 229 }, "tier3": { "deepseek_flash": 261 }, "tier4": { "deepseek_flash": 346 }, "unscoreable": { "deepseek_flash": 0 } }, "raw_deductions_by_judge": { "deepseek_flash": [ 21, 0, 1, 1, 6, 11, 6, 1, 4, 6, 5, 13, 22, 36, 15, 15, 13, 10, 35, 11, 0, 15, 0, 12, 13, 4, 3, 11, 5, 6, 3, 10, 13, 16, 13, 20, 5, 16, 29, 63, 3, 5, 13, 20, 35, 17, 7, 23, 10, 18, 15, 5, 6, 5, 5, 2, 41, 1, 13, 41, 5, 1, 4, 3, 2, 8, 1, 1, 3, 3, 12, 3, 14, 5, 14, 27, 42, 5, 16, 3, 6, 5, 15, 3, 8, 19, 5, 1, 3, 14, 4, 3, 9, 4, 4, 1, 2, 4, 3, 2 ] }, "sample_details": [ { "idx": 1, "duration_s": 5, "mean_score": 79.0, "mean_raw": 21.0, "scores_per_judge": [ 79 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "semantic-inversion", "points": 20 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 2, "duration_s": 7, "mean_score": 100.0, "mean_raw": 0.0, "scores_per_judge": [ 100 ], "deductions_per_judge": { "deepseek_flash": [] }, "judge_disagreement": 0 }, { "idx": 3, "duration_s": 8, "mean_score": 99.0, "mean_raw": 1.0, "scores_per_judge": [ 99 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 4, "duration_s": 11, "mean_score": 99.0, "mean_raw": 1.0, "scores_per_judge": [ 99 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 5, "duration_s": 16, "mean_score": 94.0, "mean_raw": 6.0, "scores_per_judge": [ 94 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "paraphrase", "points": 3 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 6, "duration_s": 17, "mean_score": 89.0, "mean_raw": 11.0, "scores_per_judge": [ 89 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 }, { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 7, "duration_s": 20, "mean_score": 94.0, "mean_raw": 6.0, "scores_per_judge": [ 94 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "paraphrase", "points": 3 } ] }, "judge_disagreement": 0 }, { "idx": 8, "duration_s": 30, "mean_score": 99.0, "mean_raw": 1.0, "scores_per_judge": [ 99 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 9, "duration_s": 32, "mean_score": 96.0, "mean_raw": 4.0, "scores_per_judge": [ 96 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 10, "duration_s": 35, "mean_score": 94.0, "mean_raw": 6.0, "scores_per_judge": [ 94 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "mishear-minor", "points": 2 } ] }, "judge_disagreement": 0 }, { "idx": 11, "duration_s": 40, "mean_score": 95.0, "mean_raw": 5.0, "scores_per_judge": [ 95 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 12, "duration_s": 46, "mean_score": 87.0, "mean_raw": 13.0, "scores_per_judge": [ 87 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "culture", "points": 8 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 13, "duration_s": 60, "mean_score": 78.0, "mean_raw": 22.0, "scores_per_judge": [ 78 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "brand", "points": 10 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-minor", "points": 2 }, { "category": "mishear-minor", "points": 2 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 14, "duration_s": 64, "mean_score": 64.0, "mean_raw": 36.0, "scores_per_judge": [ 64 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "proper-noun", "points": 6 }, { "category": "mishear-severe", "points": 5 }, { "category": "semantic-inversion", "points": 20 }, { "category": "filler", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 15, "duration_s": 69, "mean_score": 85.0, "mean_raw": 15.0, "scores_per_judge": [ 85 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 }, { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 16, "duration_s": 81, "mean_score": 85.0, "mean_raw": 15.0, "scores_per_judge": [ 85 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "proper-noun", "points": 6 }, { "category": "mishear-minor", "points": 2 }, { "category": "mishear-minor", "points": 2 }, { "category": "filler", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 17, "duration_s": 100, "mean_score": 87.0, "mean_raw": 13.0, "scores_per_judge": [ 87 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "tech", "points": 8 }, { "category": "filler", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 18, "duration_s": 125, "mean_score": 90.0, "mean_raw": 10.0, "scores_per_judge": [ 90 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 19, "duration_s": 133, "mean_score": 65.0, "mean_raw": 35.0, "scores_per_judge": [ 65 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "brand", "points": 10 }, { "category": "brand", "points": 10 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 20, "duration_s": 164, "mean_score": 89.0, "mean_raw": 11.0, "scores_per_judge": [ 89 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "proper-noun", "points": 6 }, { "category": "filler", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 21, "duration_s": 5, "mean_score": 100.0, "mean_raw": 0.0, "scores_per_judge": [ 100 ], "deductions_per_judge": { "deepseek_flash": [] }, "judge_disagreement": 0 }, { "idx": 22, "duration_s": 7, "mean_score": 85.0, "mean_raw": 15.0, "scores_per_judge": [ 85 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "brand", "points": 10 }, { "category": "paraphrase", "points": 3 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 23, "duration_s": 10, "mean_score": 100.0, "mean_raw": 0.0, "scores_per_judge": [ 100 ], "deductions_per_judge": { "deepseek_flash": [] }, "judge_disagreement": 0 }, { "idx": 24, "duration_s": 14, "mean_score": 88.0, "mean_raw": 12.0, "scores_per_judge": [ 88 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "tech", "points": 8 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 25, "duration_s": 16, "mean_score": 87.0, "mean_raw": 13.0, "scores_per_judge": [ 87 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "tech", "points": 8 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 26, "duration_s": 23, "mean_score": 96.0, "mean_raw": 4.0, "scores_per_judge": [ 96 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 27, "duration_s": 25, "mean_score": 97.0, "mean_raw": 3.0, "scores_per_judge": [ 97 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 28, "duration_s": 30, "mean_score": 89.0, "mean_raw": 11.0, "scores_per_judge": [ 89 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "content-drop", "points": 3 }, { "category": "filler", "points": 1 }, { "category": "mishear-severe", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 29, "duration_s": 34, "mean_score": 95.0, "mean_raw": 5.0, "scores_per_judge": [ 95 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 30, "duration_s": 38, "mean_score": 94.0, "mean_raw": 6.0, "scores_per_judge": [ 94 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-minor", "points": 2 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 31, "duration_s": 44, "mean_score": 97.0, "mean_raw": 3.0, "scores_per_judge": [ 97 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 32, "duration_s": 50, "mean_score": 90.0, "mean_raw": 10.0, "scores_per_judge": [ 90 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "mishear-severe", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 33, "duration_s": 62, "mean_score": 87.0, "mean_raw": 13.0, "scores_per_judge": [ 87 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "proper-noun", "points": 6 }, { "category": "mishear-minor", "points": 2 }, { "category": "filler", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 34, "duration_s": 66, "mean_score": 84.0, "mean_raw": 16.0, "scores_per_judge": [ 84 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "content-drop", "points": 3 }, { "category": "culture", "points": 8 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 35, "duration_s": 77, "mean_score": 87.0, "mean_raw": 13.0, "scores_per_judge": [ 87 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "tech", "points": 8 }, { "category": "filler", "points": 2 }, { "category": "filler", "points": 2 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 36, "duration_s": 94, "mean_score": 80.0, "mean_raw": 20.0, "scores_per_judge": [ 80 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "culture", "points": 8 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-minor", "points": 2 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 37, "duration_s": 106, "mean_score": 95.0, "mean_raw": 5.0, "scores_per_judge": [ 95 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 38, "duration_s": 126, "mean_score": 84.0, "mean_raw": 16.0, "scores_per_judge": [ 84 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 }, { "category": "mishear-minor", "points": 2 }, { "category": "mishear-minor", "points": 2 }, { "category": "mishear-minor", "points": 2 }, { "category": "filler", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 39, "duration_s": 158, "mean_score": 71.0, "mean_raw": 29.0, "scores_per_judge": [ 71 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "brand", "points": 10 }, { "category": "proper-noun", "points": 6 }, { "category": "tech", "points": 8 }, { "category": "filler", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 40, "duration_s": 279, "mean_score": 37.0, "mean_raw": 63.0, "scores_per_judge": [ 37 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "semantic-inversion", "points": 20 }, { "category": "hallucination", "points": 15 }, { "category": "numeric", "points": 10 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-severe", "points": 5 }, { "category": "paraphrase", "points": 3 }, { "category": "filler", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 41, "duration_s": 34, "mean_score": 97.0, "mean_raw": 3.0, "scores_per_judge": [ 97 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 42, "duration_s": 43, "mean_score": 95.0, "mean_raw": 5.0, "scores_per_judge": [ 95 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 43, "duration_s": 29, "mean_score": 87.0, "mean_raw": 13.0, "scores_per_judge": [ 87 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "culture", "points": 8 }, { "category": "filler", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 44, "duration_s": 71, "mean_score": 80.0, "mean_raw": 20.0, "scores_per_judge": [ 80 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "proper-noun", "points": 6 }, { "category": "culture", "points": 8 }, { "category": "paraphrase", "points": 3 }, { "category": "filler", "points": 3 } ] }, "judge_disagreement": 0 }, { "idx": 45, "duration_s": 46, "mean_score": 65.0, "mean_raw": 35.0, "scores_per_judge": [ 65 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "dialect", "points": 5 }, { "category": "hallucination", "points": 15 }, { "category": "proper-noun", "points": 6 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-minor", "points": 2 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 46, "duration_s": 62, "mean_score": 83.0, "mean_raw": 17.0, "scores_per_judge": [ 83 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "content-drop", "points": 3 }, { "category": "mishear-minor", "points": 2 }, { "category": "mishear-minor", "points": 2 }, { "category": "mishear-severe", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 47, "duration_s": 67, "mean_score": 93.0, "mean_raw": 7.0, "scores_per_judge": [ 93 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 5 }, { "category": "mishear-minor", "points": 2 } ] }, "judge_disagreement": 0 }, { "idx": 48, "duration_s": 72, "mean_score": 77.0, "mean_raw": 23.0, "scores_per_judge": [ 77 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "brand", "points": 10 }, { "category": "tech", "points": 8 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 49, "duration_s": 87, "mean_score": 90.0, "mean_raw": 10.0, "scores_per_judge": [ 90 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 50, "duration_s": 106, "mean_score": 82.0, "mean_raw": 18.0, "scores_per_judge": [ 82 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "culture", "points": 8 }, { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 51, "duration_s": 33, "mean_score": 85.0, "mean_raw": 15.0, "scores_per_judge": [ 85 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "culture", "points": 8 }, { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 52, "duration_s": 25, "mean_score": 95.0, "mean_raw": 5.0, "scores_per_judge": [ 95 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 53, "duration_s": 35, "mean_score": 94.0, "mean_raw": 6.0, "scores_per_judge": [ 94 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "mishear-minor", "points": 2 } ] }, "judge_disagreement": 0 }, { "idx": 54, "duration_s": 36, "mean_score": 95.0, "mean_raw": 5.0, "scores_per_judge": [ 95 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 55, "duration_s": 33, "mean_score": 95.0, "mean_raw": 5.0, "scores_per_judge": [ 95 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 2 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 56, "duration_s": 15, "mean_score": 98.0, "mean_raw": 2.0, "scores_per_judge": [ 98 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 57, "duration_s": 38, "mean_score": 59.0, "mean_raw": 41.0, "scores_per_judge": [ 59 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "polarity", "points": 35 }, { "category": "mishear-minor", "points": 2 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 58, "duration_s": 15, "mean_score": 99.0, "mean_raw": 1.0, "scores_per_judge": [ 99 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 59, "duration_s": 28, "mean_score": 87.0, "mean_raw": 13.0, "scores_per_judge": [ 87 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "paraphrase", "points": 3 }, { "category": "paraphrase", "points": 3 }, { "category": "mishear-minor", "points": 2 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 60, "duration_s": 18, "mean_score": 59.0, "mean_raw": 41.0, "scores_per_judge": [ 59 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "polarity", "points": 35 }, { "category": "content-drop", "points": 3 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 61, "duration_s": 19, "mean_score": 95.0, "mean_raw": 5.0, "scores_per_judge": [ 95 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "mishear-minor", "points": 2 } ] }, "judge_disagreement": 0 }, { "idx": 62, "duration_s": 17, "mean_score": 99.0, "mean_raw": 1.0, "scores_per_judge": [ 99 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 63, "duration_s": 21, "mean_score": 96.0, "mean_raw": 4.0, "scores_per_judge": [ 96 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 64, "duration_s": 23, "mean_score": 97.0, "mean_raw": 3.0, "scores_per_judge": [ 97 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 65, "duration_s": 20, "mean_score": 98.0, "mean_raw": 2.0, "scores_per_judge": [ 98 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 66, "duration_s": 24, "mean_score": 92.0, "mean_raw": 8.0, "scores_per_judge": [ 92 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 }, { "category": "mishear-minor", "points": 2 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 67, "duration_s": 16, "mean_score": 99.0, "mean_raw": 1.0, "scores_per_judge": [ 99 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 68, "duration_s": 18, "mean_score": 99.0, "mean_raw": 1.0, "scores_per_judge": [ 99 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 69, "duration_s": 18, "mean_score": 97.0, "mean_raw": 3.0, "scores_per_judge": [ 97 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-minor", "points": 2 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 70, "duration_s": 17, "mean_score": 97.0, "mean_raw": 3.0, "scores_per_judge": [ 97 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 71, "duration_s": 51, "mean_score": 88.0, "mean_raw": 12.0, "scores_per_judge": [ 88 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 }, { "category": "mishear-minor", "points": 2 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 72, "duration_s": 54, "mean_score": 97.0, "mean_raw": 3.0, "scores_per_judge": [ 97 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 73, "duration_s": 56, "mean_score": 86.0, "mean_raw": 14.0, "scores_per_judge": [ 86 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 }, { "category": "paraphrase", "points": 3 }, { "category": "mishear-minor", "points": 2 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 74, "duration_s": 45, "mean_score": 95.0, "mean_raw": 5.0, "scores_per_judge": [ 95 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 75, "duration_s": 74, "mean_score": 86.0, "mean_raw": 14.0, "scores_per_judge": [ 86 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-minor", "points": 2 }, { "category": "mishear-minor", "points": 2 }, { "category": "content-drop", "points": 3 }, { "category": "mishear-minor", "points": 2 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 76, "duration_s": 58, "mean_score": 73.0, "mean_raw": 27.0, "scores_per_judge": [ 73 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "semantic-inversion", "points": 20 }, { "category": "paraphrase", "points": 3 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 77, "duration_s": 130, "mean_score": 58.0, "mean_raw": 42.0, "scores_per_judge": [ 58 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "semantic-inversion", "points": 20 }, { "category": "brand", "points": 10 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-minor", "points": 2 }, { "category": "filler", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 78, "duration_s": 61, "mean_score": 95.0, "mean_raw": 5.0, "scores_per_judge": [ 95 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 79, "duration_s": 47, "mean_score": 84.0, "mean_raw": 16.0, "scores_per_judge": [ 84 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 5 }, { "category": "content-drop", "points": 3 }, { "category": "mishear-severe", "points": 5 }, { "category": "paraphrase", "points": 3 } ] }, "judge_disagreement": 0 }, { "idx": 80, "duration_s": 42, "mean_score": 97.0, "mean_raw": 3.0, "scores_per_judge": [ 97 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 81, "duration_s": 38, "mean_score": 94.0, "mean_raw": 6.0, "scores_per_judge": [ 94 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-minor", "points": 2 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 82, "duration_s": 38, "mean_score": 95.0, "mean_raw": 5.0, "scores_per_judge": [ 95 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 83, "duration_s": 37, "mean_score": 85.0, "mean_raw": 15.0, "scores_per_judge": [ 85 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 }, { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 84, "duration_s": 36, "mean_score": 97.0, "mean_raw": 3.0, "scores_per_judge": [ 97 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-minor", "points": 2 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 85, "duration_s": 36, "mean_score": 92.0, "mean_raw": 8.0, "scores_per_judge": [ 92 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "mishear-minor", "points": 2 }, { "category": "mishear-minor", "points": 2 } ] }, "judge_disagreement": 0 }, { "idx": 86, "duration_s": 35, "mean_score": 81.0, "mean_raw": 19.0, "scores_per_judge": [ 81 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "hallucination", "points": 15 }, { "category": "filler", "points": 4 } ] }, "judge_disagreement": 0 }, { "idx": 87, "duration_s": 34, "mean_score": 95.0, "mean_raw": 5.0, "scores_per_judge": [ 95 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 88, "duration_s": 34, "mean_score": 99.0, "mean_raw": 1.0, "scores_per_judge": [ 99 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 89, "duration_s": 34, "mean_score": 97.0, "mean_raw": 3.0, "scores_per_judge": [ 97 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 90, "duration_s": 33, "mean_score": 86.0, "mean_raw": 14.0, "scores_per_judge": [ 86 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "tech", "points": 8 }, { "category": "mishear-minor", "points": 2 }, { "category": "mishear-minor", "points": 2 }, { "category": "filler", "points": 2 } ] }, "judge_disagreement": 0 }, { "idx": 91, "duration_s": 33, "mean_score": 96.0, "mean_raw": 4.0, "scores_per_judge": [ 96 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 92, "duration_s": 32, "mean_score": 97.0, "mean_raw": 3.0, "scores_per_judge": [ 97 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 93, "duration_s": 32, "mean_score": 91.0, "mean_raw": 9.0, "scores_per_judge": [ 91 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "content-drop", "points": 3 }, { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 94, "duration_s": 32, "mean_score": 96.0, "mean_raw": 4.0, "scores_per_judge": [ 96 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-minor", "points": 2 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 95, "duration_s": 32, "mean_score": 96.0, "mean_raw": 4.0, "scores_per_judge": [ 96 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 96, "duration_s": 31, "mean_score": 99.0, "mean_raw": 1.0, "scores_per_judge": [ 99 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 97, "duration_s": 30, "mean_score": 98.0, "mean_raw": 2.0, "scores_per_judge": [ 98 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 98, "duration_s": 30, "mean_score": 96.0, "mean_raw": 4.0, "scores_per_judge": [ 96 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 99, "duration_s": 30, "mean_score": 97.0, "mean_raw": 3.0, "scores_per_judge": [ 97 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "mishear-minor", "points": 2 } ] }, "judge_disagreement": 0 }, { "idx": 100, "duration_s": 30, "mean_score": 98.0, "mean_raw": 2.0, "scores_per_judge": [ 98 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 } ], "total_deductions": 1061, "avg_deductions_per_sample": 10.61, "mean_clamped_score": 89.4, "n_judge_sample_pairs": 100, "gt_chars": 22625, "ded_per_100char": 4.69, "fidelity_score": 76.6, "overall_score": 89.4, "overall_raw_deductions": 10.61, "score_per_judge": { "deepseek_flash": 89.4 }, "per_group_deductions": { "meaning-flip": { "deepseek_flash": 225 }, "entity": { "deepseek_flash": 168 }, "cultural": { "deepseek_flash": 61 }, "fidelity": { "deepseek_flash": 607 } }, "per_bucket": { "general": { "n_samples": 40, "mean_raw": 12.2, "mean_score": 87.8, "total_raw": 489.0, "idxs": [ 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40 ], "worst_idx": 40 }, "idiom-test": { "n_samples": 5, "mean_raw": 15.2, "mean_score": 84.8, "total_raw": 76.0, "idxs": [ 41, 42, 43, 44, 45 ], "worst_idx": 45 }, "long-mixed": { "n_samples": 5, "mean_raw": 15.0, "mean_score": 85.0, "total_raw": 75.0, "idxs": [ 46, 47, 48, 49, 50 ], "worst_idx": 48 } }, "per_group_per_100char": { "meaning-flip": 0.994, "entity": 0.743, "cultural": 0.27, "fidelity": 2.683 }, "per_group_fidelity": { "meaning-flip": 76.2, "entity": 86.8, "cultural": 83.7, "fidelity": 68.6 } }, "gemini_pro_stt": { "scores_by_judge": { "deepseek_flash": [ 100, 100, 99, 100, 92, 97, 99, 100, 95, 94, 92, 95, 82, 70, 90, 84, 77, 88, 87, 82, 100, 90, 100, 89, 100, 100, 99, 93, 88, 95, 98, 77, 95, 95, 74, 53, 95, 83, 75, 54, 92, 99, 97, 100, 93, 87, 78, 89, 82, 89, 97, 98, 96, 99, 95, 98, 98, 100, 97, 62, 95, 100, 97, 98, 99, 95, 100, 99, 95, 91, 88, 91, 93, 97, 94, 94, 60, 96, 94, 89, 92, 99, 90, 95, 97, 98, 95, 99, 100, 97, 97, 99, 93, 99, 97, 99, 99, 100, 99, 97 ] }, "per_category_deductions": { "polarity": { "deepseek_flash": 70 }, "semantic-inversion": { "deepseek_flash": 60 }, "hallucination": { "deepseek_flash": 0 }, "numeric": { "deepseek_flash": 10 }, "brand": { "deepseek_flash": 70 }, "tech": { "deepseek_flash": 80 }, "culture": { "deepseek_flash": 8 }, "proper-noun": { "deepseek_flash": 18 }, "dialect": { "deepseek_flash": 0 }, "mishear-severe": { "deepseek_flash": 155 }, "content-drop": { "deepseek_flash": 18 }, "paraphrase": { "deepseek_flash": 18 }, "mishear-minor": { "deepseek_flash": 30 }, "filler": { "deepseek_flash": 234 }, "format": { "deepseek_flash": 1 }, "unscoreable": { "deepseek_flash": 0 } }, "per_tier_deductions": { "tier1": { "deepseek_flash": 140 }, "tier2": { "deepseek_flash": 176 }, "tier3": { "deepseek_flash": 221 }, "tier4": { "deepseek_flash": 235 }, "unscoreable": { "deepseek_flash": 0 } }, "raw_deductions_by_judge": { "deepseek_flash": [ 0, 0, 1, 0, 8, 3, 1, 0, 5, 6, 8, 5, 18, 30, 10, 16, 23, 12, 13, 18, 0, 10, 0, 11, 0, 0, 1, 7, 12, 5, 2, 23, 5, 5, 26, 47, 5, 17, 25, 46, 8, 1, 3, 0, 7, 13, 22, 11, 18, 11, 3, 2, 4, 1, 5, 2, 2, 0, 3, 38, 5, 0, 3, 2, 1, 5, 0, 1, 5, 9, 12, 9, 7, 3, 6, 6, 40, 4, 6, 11, 8, 1, 10, 5, 3, 2, 5, 1, 0, 3, 3, 1, 7, 1, 3, 1, 1, 0, 1, 3 ] }, "sample_details": [ { "idx": 1, "duration_s": 5, "mean_score": 100.0, "mean_raw": 0.0, "scores_per_judge": [ 100 ], "deductions_per_judge": { "deepseek_flash": [] }, "judge_disagreement": 0 }, { "idx": 2, "duration_s": 7, "mean_score": 100.0, "mean_raw": 0.0, "scores_per_judge": [ 100 ], "deductions_per_judge": { "deepseek_flash": [] }, "judge_disagreement": 0 }, { "idx": 3, "duration_s": 8, "mean_score": 99.0, "mean_raw": 1.0, "scores_per_judge": [ 99 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 4, "duration_s": 11, "mean_score": 100.0, "mean_raw": 0.0, "scores_per_judge": [ 100 ], "deductions_per_judge": { "deepseek_flash": [] }, "judge_disagreement": 0 }, { "idx": 5, "duration_s": 16, "mean_score": 92.0, "mean_raw": 8.0, "scores_per_judge": [ 92 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "culture", "points": 8 } ] }, "judge_disagreement": 0 }, { "idx": 6, "duration_s": 17, "mean_score": 97.0, "mean_raw": 3.0, "scores_per_judge": [ 97 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "paraphrase", "points": 3 } ] }, "judge_disagreement": 0 }, { "idx": 7, "duration_s": 20, "mean_score": 99.0, "mean_raw": 1.0, "scores_per_judge": [ 99 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 8, "duration_s": 30, "mean_score": 100.0, "mean_raw": 0.0, "scores_per_judge": [ 100 ], "deductions_per_judge": { "deepseek_flash": [] }, "judge_disagreement": 0 }, { "idx": 9, "duration_s": 32, "mean_score": 95.0, "mean_raw": 5.0, "scores_per_judge": [ 95 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 10, "duration_s": 35, "mean_score": 94.0, "mean_raw": 6.0, "scores_per_judge": [ 94 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "paraphrase", "points": 3 } ] }, "judge_disagreement": 0 }, { "idx": 11, "duration_s": 40, "mean_score": 92.0, "mean_raw": 8.0, "scores_per_judge": [ 92 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "content-drop", "points": 3 }, { "category": "filler", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 12, "duration_s": 46, "mean_score": 95.0, "mean_raw": 5.0, "scores_per_judge": [ 95 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 13, "duration_s": 60, "mean_score": 82.0, "mean_raw": 18.0, "scores_per_judge": [ 82 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "brand", "points": 10 }, { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 14, "duration_s": 64, "mean_score": 70.0, "mean_raw": 30.0, "scores_per_judge": [ 70 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 }, { "category": "proper-noun", "points": 6 }, { "category": "proper-noun", "points": 6 }, { "category": "content-drop", "points": 3 }, { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 15, "duration_s": 69, "mean_score": 90.0, "mean_raw": 10.0, "scores_per_judge": [ 90 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 16, "duration_s": 81, "mean_score": 84.0, "mean_raw": 16.0, "scores_per_judge": [ 84 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "proper-noun", "points": 6 }, { "category": "paraphrase", "points": 3 }, { "category": "mishear-minor", "points": 2 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 17, "duration_s": 100, "mean_score": 77.0, "mean_raw": 23.0, "scores_per_judge": [ 77 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "brand", "points": 10 }, { "category": "tech", "points": 8 }, { "category": "filler", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 18, "duration_s": 125, "mean_score": 88.0, "mean_raw": 12.0, "scores_per_judge": [ 88 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 }, { "category": "mishear-minor", "points": 2 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 19, "duration_s": 133, "mean_score": 87.0, "mean_raw": 13.0, "scores_per_judge": [ 87 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "tech", "points": 8 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 20, "duration_s": 164, "mean_score": 82.0, "mean_raw": 18.0, "scores_per_judge": [ 82 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-minor", "points": 2 }, { "category": "mishear-minor", "points": 2 }, { "category": "filler", "points": 4 } ] }, "judge_disagreement": 0 }, { "idx": 21, "duration_s": 5, "mean_score": 100.0, "mean_raw": 0.0, "scores_per_judge": [ 100 ], "deductions_per_judge": { "deepseek_flash": [] }, "judge_disagreement": 0 }, { "idx": 22, "duration_s": 7, "mean_score": 90.0, "mean_raw": 10.0, "scores_per_judge": [ 90 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "brand", "points": 10 } ] }, "judge_disagreement": 0 }, { "idx": 23, "duration_s": 10, "mean_score": 100.0, "mean_raw": 0.0, "scores_per_judge": [ 100 ], "deductions_per_judge": { "deepseek_flash": [] }, "judge_disagreement": 0 }, { "idx": 24, "duration_s": 14, "mean_score": 89.0, "mean_raw": 11.0, "scores_per_judge": [ 89 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "tech", "points": 8 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 25, "duration_s": 16, "mean_score": 100.0, "mean_raw": 0.0, "scores_per_judge": [ 100 ], "deductions_per_judge": { "deepseek_flash": [] }, "judge_disagreement": 0 }, { "idx": 26, "duration_s": 23, "mean_score": 100.0, "mean_raw": 0.0, "scores_per_judge": [ 100 ], "deductions_per_judge": { "deepseek_flash": [] }, "judge_disagreement": 0 }, { "idx": 27, "duration_s": 25, "mean_score": 99.0, "mean_raw": 1.0, "scores_per_judge": [ 99 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 28, "duration_s": 30, "mean_score": 93.0, "mean_raw": 7.0, "scores_per_judge": [ 93 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 29, "duration_s": 34, "mean_score": 88.0, "mean_raw": 12.0, "scores_per_judge": [ 88 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "brand", "points": 10 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 30, "duration_s": 38, "mean_score": 95.0, "mean_raw": 5.0, "scores_per_judge": [ 95 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 31, "duration_s": 44, "mean_score": 98.0, "mean_raw": 2.0, "scores_per_judge": [ 98 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 32, "duration_s": 50, "mean_score": 77.0, "mean_raw": 23.0, "scores_per_judge": [ 77 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "semantic-inversion", "points": 20 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 33, "duration_s": 62, "mean_score": 95.0, "mean_raw": 5.0, "scores_per_judge": [ 95 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 3 }, { "category": "filler", "points": 2 } ] }, "judge_disagreement": 0 }, { "idx": 34, "duration_s": 66, "mean_score": 95.0, "mean_raw": 5.0, "scores_per_judge": [ 95 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-minor", "points": 2 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 35, "duration_s": 77, "mean_score": 74.0, "mean_raw": 26.0, "scores_per_judge": [ 74 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "tech", "points": 8 }, { "category": "tech", "points": 8 }, { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 36, "duration_s": 94, "mean_score": 53.0, "mean_raw": 47.0, "scores_per_judge": [ 53 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "polarity", "points": 35 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-minor", "points": 2 }, { "category": "filler", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 37, "duration_s": 106, "mean_score": 95.0, "mean_raw": 5.0, "scores_per_judge": [ 95 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 38, "duration_s": 126, "mean_score": 83.0, "mean_raw": 17.0, "scores_per_judge": [ 83 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "content-drop", "points": 3 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-minor", "points": 2 }, { "category": "mishear-minor", "points": 2 }, { "category": "filler", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 39, "duration_s": 158, "mean_score": 75.0, "mean_raw": 25.0, "scores_per_judge": [ 75 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "brand", "points": 10 }, { "category": "tech", "points": 8 }, { "category": "mishear-minor", "points": 2 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 40, "duration_s": 279, "mean_score": 54.0, "mean_raw": 46.0, "scores_per_judge": [ 54 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "semantic-inversion", "points": 20 }, { "category": "numeric", "points": 10 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-severe", "points": 5 }, { "category": "paraphrase", "points": 3 }, { "category": "content-drop", "points": 3 } ] }, "judge_disagreement": 0 }, { "idx": 41, "duration_s": 34, "mean_score": 92.0, "mean_raw": 8.0, "scores_per_judge": [ 92 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 42, "duration_s": 43, "mean_score": 99.0, "mean_raw": 1.0, "scores_per_judge": [ 99 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 43, "duration_s": 29, "mean_score": 97.0, "mean_raw": 3.0, "scores_per_judge": [ 97 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 44, "duration_s": 71, "mean_score": 100.0, "mean_raw": 0.0, "scores_per_judge": [ 100 ], "deductions_per_judge": { "deepseek_flash": [] }, "judge_disagreement": 0 }, { "idx": 45, "duration_s": 46, "mean_score": 93.0, "mean_raw": 7.0, "scores_per_judge": [ 93 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-minor", "points": 2 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 46, "duration_s": 62, "mean_score": 87.0, "mean_raw": 13.0, "scores_per_judge": [ 87 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "content-drop", "points": 3 }, { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 47, "duration_s": 67, "mean_score": 78.0, "mean_raw": 22.0, "scores_per_judge": [ 78 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "brand", "points": 10 }, { "category": "tech", "points": 8 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 48, "duration_s": 72, "mean_score": 89.0, "mean_raw": 11.0, "scores_per_judge": [ 89 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "tech", "points": 8 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 49, "duration_s": 87, "mean_score": 82.0, "mean_raw": 18.0, "scores_per_judge": [ 82 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 5 }, { "category": "mishear-severe", "points": 5 }, { "category": "tech", "points": 8 } ] }, "judge_disagreement": 0 }, { "idx": 50, "duration_s": 106, "mean_score": 89.0, "mean_raw": 11.0, "scores_per_judge": [ 89 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 }, { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 51, "duration_s": 33, "mean_score": 97.0, "mean_raw": 3.0, "scores_per_judge": [ 97 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 52, "duration_s": 25, "mean_score": 98.0, "mean_raw": 2.0, "scores_per_judge": [ 98 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 53, "duration_s": 35, "mean_score": 96.0, "mean_raw": 4.0, "scores_per_judge": [ 96 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 54, "duration_s": 36, "mean_score": 99.0, "mean_raw": 1.0, "scores_per_judge": [ 99 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 55, "duration_s": 33, "mean_score": 95.0, "mean_raw": 5.0, "scores_per_judge": [ 95 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 56, "duration_s": 15, "mean_score": 98.0, "mean_raw": 2.0, "scores_per_judge": [ 98 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 57, "duration_s": 38, "mean_score": 98.0, "mean_raw": 2.0, "scores_per_judge": [ 98 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 58, "duration_s": 15, "mean_score": 100.0, "mean_raw": 0.0, "scores_per_judge": [ 100 ], "deductions_per_judge": { "deepseek_flash": [] }, "judge_disagreement": 0 }, { "idx": 59, "duration_s": 28, "mean_score": 97.0, "mean_raw": 3.0, "scores_per_judge": [ 97 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 60, "duration_s": 18, "mean_score": 62.0, "mean_raw": 38.0, "scores_per_judge": [ 62 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "polarity", "points": 35 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 61, "duration_s": 19, "mean_score": 95.0, "mean_raw": 5.0, "scores_per_judge": [ 95 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 62, "duration_s": 17, "mean_score": 100.0, "mean_raw": 0.0, "scores_per_judge": [ 100 ], "deductions_per_judge": { "deepseek_flash": [] }, "judge_disagreement": 0 }, { "idx": 63, "duration_s": 21, "mean_score": 97.0, "mean_raw": 3.0, "scores_per_judge": [ 97 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-minor", "points": 2 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 64, "duration_s": 23, "mean_score": 98.0, "mean_raw": 2.0, "scores_per_judge": [ 98 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 65, "duration_s": 20, "mean_score": 99.0, "mean_raw": 1.0, "scores_per_judge": [ 99 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 66, "duration_s": 24, "mean_score": 95.0, "mean_raw": 5.0, "scores_per_judge": [ 95 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-minor", "points": 2 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 67, "duration_s": 16, "mean_score": 100.0, "mean_raw": 0.0, "scores_per_judge": [ 100 ], "deductions_per_judge": { "deepseek_flash": [] }, "judge_disagreement": 0 }, { "idx": 68, "duration_s": 18, "mean_score": 99.0, "mean_raw": 1.0, "scores_per_judge": [ 99 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "format", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 69, "duration_s": 18, "mean_score": 95.0, "mean_raw": 5.0, "scores_per_judge": [ 95 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 70, "duration_s": 17, "mean_score": 91.0, "mean_raw": 9.0, "scores_per_judge": [ 91 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "tech", "points": 8 } ] }, "judge_disagreement": 0 }, { "idx": 71, "duration_s": 51, "mean_score": 88.0, "mean_raw": 12.0, "scores_per_judge": [ 88 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-minor", "points": 2 } ] }, "judge_disagreement": 0 }, { "idx": 72, "duration_s": 54, "mean_score": 91.0, "mean_raw": 9.0, "scores_per_judge": [ 91 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 73, "duration_s": 56, "mean_score": 93.0, "mean_raw": 7.0, "scores_per_judge": [ 93 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 74, "duration_s": 45, "mean_score": 97.0, "mean_raw": 3.0, "scores_per_judge": [ 97 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 75, "duration_s": 74, "mean_score": 94.0, "mean_raw": 6.0, "scores_per_judge": [ 94 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-minor", "points": 2 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 76, "duration_s": 58, "mean_score": 94.0, "mean_raw": 6.0, "scores_per_judge": [ 94 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "paraphrase", "points": 3 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 77, "duration_s": 130, "mean_score": 60.0, "mean_raw": 40.0, "scores_per_judge": [ 60 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "semantic-inversion", "points": 20 }, { "category": "brand", "points": 10 }, { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 78, "duration_s": 61, "mean_score": 96.0, "mean_raw": 4.0, "scores_per_judge": [ 96 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 79, "duration_s": 47, "mean_score": 94.0, "mean_raw": 6.0, "scores_per_judge": [ 94 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "content-drop", "points": 3 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 80, "duration_s": 42, "mean_score": 89.0, "mean_raw": 11.0, "scores_per_judge": [ 89 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 }, { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 81, "duration_s": 38, "mean_score": 92.0, "mean_raw": 8.0, "scores_per_judge": [ 92 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 }, { "category": "mishear-minor", "points": 2 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 82, "duration_s": 38, "mean_score": 99.0, "mean_raw": 1.0, "scores_per_judge": [ 99 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 83, "duration_s": 37, "mean_score": 90.0, "mean_raw": 10.0, "scores_per_judge": [ 90 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 84, "duration_s": 36, "mean_score": 95.0, "mean_raw": 5.0, "scores_per_judge": [ 95 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 85, "duration_s": 36, "mean_score": 97.0, "mean_raw": 3.0, "scores_per_judge": [ 97 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 86, "duration_s": 35, "mean_score": 98.0, "mean_raw": 2.0, "scores_per_judge": [ 98 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 87, "duration_s": 34, "mean_score": 95.0, "mean_raw": 5.0, "scores_per_judge": [ 95 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 88, "duration_s": 34, "mean_score": 99.0, "mean_raw": 1.0, "scores_per_judge": [ 99 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 89, "duration_s": 34, "mean_score": 100.0, "mean_raw": 0.0, "scores_per_judge": [ 100 ], "deductions_per_judge": { "deepseek_flash": [] }, "judge_disagreement": 0 }, { "idx": 90, "duration_s": 33, "mean_score": 97.0, "mean_raw": 3.0, "scores_per_judge": [ 97 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 91, "duration_s": 33, "mean_score": 97.0, "mean_raw": 3.0, "scores_per_judge": [ 97 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 2 } ] }, "judge_disagreement": 0 }, { "idx": 92, "duration_s": 32, "mean_score": 99.0, "mean_raw": 1.0, "scores_per_judge": [ 99 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 93, "duration_s": 32, "mean_score": 93.0, "mean_raw": 7.0, "scores_per_judge": [ 93 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 94, "duration_s": 32, "mean_score": 99.0, "mean_raw": 1.0, "scores_per_judge": [ 99 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 95, "duration_s": 32, "mean_score": 97.0, "mean_raw": 3.0, "scores_per_judge": [ 97 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "paraphrase", "points": 3 } ] }, "judge_disagreement": 0 }, { "idx": 96, "duration_s": 31, "mean_score": 99.0, "mean_raw": 1.0, "scores_per_judge": [ 99 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 97, "duration_s": 30, "mean_score": 99.0, "mean_raw": 1.0, "scores_per_judge": [ 99 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 98, "duration_s": 30, "mean_score": 100.0, "mean_raw": 0.0, "scores_per_judge": [ 100 ], "deductions_per_judge": { "deepseek_flash": [] }, "judge_disagreement": 0 }, { "idx": 99, "duration_s": 30, "mean_score": 99.0, "mean_raw": 1.0, "scores_per_judge": [ 99 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 100, "duration_s": 30, "mean_score": 97.0, "mean_raw": 3.0, "scores_per_judge": [ 97 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 } ], "total_deductions": 772, "avg_deductions_per_sample": 7.72, "mean_clamped_score": 92.3, "n_judge_sample_pairs": 100, "gt_chars": 22625, "ded_per_100char": 3.41, "fidelity_score": 82.9, "overall_score": 92.3, "overall_raw_deductions": 7.72, "score_per_judge": { "deepseek_flash": 92.3 }, "per_group_deductions": { "meaning-flip": { "deepseek_flash": 140 }, "entity": { "deepseek_flash": 168 }, "cultural": { "deepseek_flash": 8 }, "fidelity": { "deepseek_flash": 456 } }, "per_bucket": { "general": { "n_samples": 40, "mean_raw": 10.6, "mean_score": 89.4, "total_raw": 424.0, "idxs": [ 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40 ], "worst_idx": 36 }, "idiom-test": { "n_samples": 5, "mean_raw": 3.8, "mean_score": 96.2, "total_raw": 19.0, "idxs": [ 41, 42, 43, 44, 45 ], "worst_idx": 41 }, "long-mixed": { "n_samples": 5, "mean_raw": 15.0, "mean_score": 85.0, "total_raw": 75.0, "idxs": [ 46, 47, 48, 49, 50 ], "worst_idx": 47 } }, "per_group_per_100char": { "meaning-flip": 0.619, "entity": 0.743, "cultural": 0.035, "fidelity": 2.015 }, "per_group_fidelity": { "meaning-flip": 85.2, "entity": 86.8, "cultural": 97.9, "fidelity": 76.4 } }, "gemini_stt": { "scores_by_judge": { "deepseek_flash": [ 100, 100, 99, 95, 92, 85, 97, 99, 96, 96, 85, 86, 78, 77, 95, 74, 70, 67, 55, 50, 100, 86, 98, 89, 89, 99, 90, 78, 84, 90, 95, 88, 79, 89, 71, 95, 79, 70, 46, 14, 93, 72, 72, 62, 29, 87, 85, 87, 81, 64, 80, 87, 91, 95, 91, 98, 54, 47, 88, 33, 82, 98, 79, 98, 98, 87, 93, 89, 85, 83, 78, 87, 67, 99, 69, 64, 62, 90, 83, 98, 91, 92, 95, 95, 89, 97, 77, 94, 92, 84, 93, 97, 91, 90, 93, 99, 95, 97, 96, 97 ] }, "per_category_deductions": { "polarity": { "deepseek_flash": 105 }, "semantic-inversion": { "deepseek_flash": 100 }, "hallucination": { "deepseek_flash": 30 }, "numeric": { "deepseek_flash": 20 }, "brand": { "deepseek_flash": 150 }, "tech": { "deepseek_flash": 184 }, "culture": { "deepseek_flash": 136 }, "proper-noun": { "deepseek_flash": 90 }, "dialect": { "deepseek_flash": 30 }, "mishear-severe": { "deepseek_flash": 340 }, "content-drop": { "deepseek_flash": 12 }, "paraphrase": { "deepseek_flash": 27 }, "mishear-minor": { "deepseek_flash": 122 }, "filler": { "deepseek_flash": 289 }, "format": { "deepseek_flash": 1 }, "unscoreable": { "deepseek_flash": 0 } }, "per_tier_deductions": { "tier1": { "deepseek_flash": 255 }, "tier2": { "deepseek_flash": 590 }, "tier3": { "deepseek_flash": 501 }, "tier4": { "deepseek_flash": 290 }, "unscoreable": { "deepseek_flash": 0 } }, "raw_deductions_by_judge": { "deepseek_flash": [ 0, 0, 1, 5, 8, 15, 3, 1, 4, 4, 15, 14, 22, 23, 5, 26, 30, 33, 45, 50, 0, 14, 2, 11, 11, 1, 10, 22, 16, 10, 5, 12, 21, 11, 29, 5, 21, 30, 54, 86, 7, 28, 28, 38, 71, 13, 15, 13, 19, 36, 20, 13, 9, 5, 9, 2, 46, 53, 12, 67, 18, 2, 21, 2, 2, 13, 7, 11, 15, 17, 22, 13, 33, 1, 31, 36, 38, 10, 17, 2, 9, 8, 5, 5, 11, 3, 23, 6, 8, 16, 7, 3, 9, 10, 7, 1, 5, 3, 4, 3 ] }, "sample_details": [ { "idx": 1, "duration_s": 5, "mean_score": 100.0, "mean_raw": 0.0, "scores_per_judge": [ 100 ], "deductions_per_judge": { "deepseek_flash": [] }, "judge_disagreement": 0 }, { "idx": 2, "duration_s": 7, "mean_score": 100.0, "mean_raw": 0.0, "scores_per_judge": [ 100 ], "deductions_per_judge": { "deepseek_flash": [] }, "judge_disagreement": 0 }, { "idx": 3, "duration_s": 8, "mean_score": 99.0, "mean_raw": 1.0, "scores_per_judge": [ 99 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "format", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 4, "duration_s": 11, "mean_score": 95.0, "mean_raw": 5.0, "scores_per_judge": [ 95 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 5, "duration_s": 16, "mean_score": 92.0, "mean_raw": 8.0, "scores_per_judge": [ 92 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "proper-noun", "points": 6 }, { "category": "filler", "points": 2 } ] }, "judge_disagreement": 0 }, { "idx": 6, "duration_s": 17, "mean_score": 85.0, "mean_raw": 15.0, "scores_per_judge": [ 85 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 1 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-minor", "points": 2 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 7, "duration_s": 20, "mean_score": 97.0, "mean_raw": 3.0, "scores_per_judge": [ 97 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 8, "duration_s": 30, "mean_score": 99.0, "mean_raw": 1.0, "scores_per_judge": [ 99 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 9, "duration_s": 32, "mean_score": 96.0, "mean_raw": 4.0, "scores_per_judge": [ 96 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-minor", "points": 2 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 10, "duration_s": 35, "mean_score": 96.0, "mean_raw": 4.0, "scores_per_judge": [ 96 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 11, "duration_s": 40, "mean_score": 85.0, "mean_raw": 15.0, "scores_per_judge": [ 85 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "tech", "points": 8 }, { "category": "mishear-minor", "points": 2 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 12, "duration_s": 46, "mean_score": 86.0, "mean_raw": 14.0, "scores_per_judge": [ 86 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 }, { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 2 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 13, "duration_s": 60, "mean_score": 78.0, "mean_raw": 22.0, "scores_per_judge": [ 78 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "brand", "points": 10 }, { "category": "brand", "points": 10 }, { "category": "mishear-minor", "points": 2 } ] }, "judge_disagreement": 0 }, { "idx": 14, "duration_s": 64, "mean_score": 77.0, "mean_raw": 23.0, "scores_per_judge": [ 77 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "proper-noun", "points": 6 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-minor", "points": 2 }, { "category": "mishear-minor", "points": 2 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 15, "duration_s": 69, "mean_score": 95.0, "mean_raw": 5.0, "scores_per_judge": [ 95 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 16, "duration_s": 81, "mean_score": 74.0, "mean_raw": 26.0, "scores_per_judge": [ 74 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "proper-noun", "points": 6 }, { "category": "proper-noun", "points": 6 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 17, "duration_s": 100, "mean_score": 70.0, "mean_raw": 30.0, "scores_per_judge": [ 70 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "brand", "points": 10 }, { "category": "tech", "points": 8 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-minor", "points": 2 }, { "category": "filler", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 18, "duration_s": 125, "mean_score": 67.0, "mean_raw": 33.0, "scores_per_judge": [ 67 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "brand", "points": 10 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-minor", "points": 2 }, { "category": "paraphrase", "points": 3 }, { "category": "filler", "points": 3 } ] }, "judge_disagreement": 0 }, { "idx": 19, "duration_s": 133, "mean_score": 55.0, "mean_raw": 45.0, "scores_per_judge": [ 55 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "brand", "points": 10 }, { "category": "brand", "points": 10 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-minor", "points": 2 }, { "category": "filler", "points": 3 } ] }, "judge_disagreement": 0 }, { "idx": 20, "duration_s": 164, "mean_score": 50.0, "mean_raw": 50.0, "scores_per_judge": [ 50 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-severe", "points": 5 }, { "category": "culture", "points": 8 }, { "category": "mishear-minor", "points": 2 }, { "category": "mishear-minor", "points": 2 }, { "category": "mishear-minor", "points": 2 }, { "category": "mishear-minor", "points": 2 }, { "category": "mishear-minor", "points": 2 }, { "category": "mishear-minor", "points": 2 }, { "category": "filler", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 21, "duration_s": 5, "mean_score": 100.0, "mean_raw": 0.0, "scores_per_judge": [ 100 ], "deductions_per_judge": { "deepseek_flash": [] }, "judge_disagreement": 0 }, { "idx": 22, "duration_s": 7, "mean_score": 86.0, "mean_raw": 14.0, "scores_per_judge": [ 86 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "proper-noun", "points": 6 }, { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 23, "duration_s": 10, "mean_score": 98.0, "mean_raw": 2.0, "scores_per_judge": [ 98 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 24, "duration_s": 14, "mean_score": 89.0, "mean_raw": 11.0, "scores_per_judge": [ 89 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "tech", "points": 8 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 25, "duration_s": 16, "mean_score": 89.0, "mean_raw": 11.0, "scores_per_judge": [ 89 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "tech", "points": 8 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 26, "duration_s": 23, "mean_score": 99.0, "mean_raw": 1.0, "scores_per_judge": [ 99 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 27, "duration_s": 25, "mean_score": 90.0, "mean_raw": 10.0, "scores_per_judge": [ 90 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 }, { "category": "mishear-severe", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 28, "duration_s": 30, "mean_score": 78.0, "mean_raw": 22.0, "scores_per_judge": [ 78 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-minor", "points": 2 }, { "category": "filler", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 29, "duration_s": 34, "mean_score": 84.0, "mean_raw": 16.0, "scores_per_judge": [ 84 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "brand", "points": 10 }, { "category": "content-drop", "points": 3 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 30, "duration_s": 38, "mean_score": 90.0, "mean_raw": 10.0, "scores_per_judge": [ 90 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 }, { "category": "mishear-minor", "points": 2 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 31, "duration_s": 44, "mean_score": 95.0, "mean_raw": 5.0, "scores_per_judge": [ 95 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 32, "duration_s": 50, "mean_score": 88.0, "mean_raw": 12.0, "scores_per_judge": [ 88 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 }, { "category": "mishear-minor", "points": 2 }, { "category": "filler", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 33, "duration_s": 62, "mean_score": 79.0, "mean_raw": 21.0, "scores_per_judge": [ 79 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "brand", "points": 10 }, { "category": "proper-noun", "points": 6 }, { "category": "mishear-minor", "points": 2 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 34, "duration_s": 66, "mean_score": 89.0, "mean_raw": 11.0, "scores_per_judge": [ 89 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "proper-noun", "points": 6 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 35, "duration_s": 77, "mean_score": 71.0, "mean_raw": 29.0, "scores_per_judge": [ 71 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "tech", "points": 8 }, { "category": "tech", "points": 8 }, { "category": "tech", "points": 8 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 36, "duration_s": 94, "mean_score": 95.0, "mean_raw": 5.0, "scores_per_judge": [ 95 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 37, "duration_s": 106, "mean_score": 79.0, "mean_raw": 21.0, "scores_per_judge": [ 79 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "tech", "points": 8 }, { "category": "proper-noun", "points": 6 }, { "category": "mishear-minor", "points": 2 }, { "category": "filler", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 38, "duration_s": 126, "mean_score": 70.0, "mean_raw": 30.0, "scores_per_judge": [ 70 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "proper-noun", "points": 6 }, { "category": "tech", "points": 8 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-minor", "points": 2 }, { "category": "mishear-minor", "points": 2 }, { "category": "mishear-minor", "points": 2 }, { "category": "mishear-minor", "points": 2 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 39, "duration_s": 158, "mean_score": 46.0, "mean_raw": 54.0, "scores_per_judge": [ 46 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "tech", "points": 8 }, { "category": "brand", "points": 10 }, { "category": "tech", "points": 8 }, { "category": "culture", "points": 8 }, { "category": "proper-noun", "points": 6 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-minor", "points": 2 }, { "category": "mishear-minor", "points": 2 }, { "category": "filler", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 40, "duration_s": 279, "mean_score": 14.0, "mean_raw": 86.0, "scores_per_judge": [ 14 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "semantic-inversion", "points": 20 }, { "category": "semantic-inversion", "points": 20 }, { "category": "numeric", "points": 10 }, { "category": "culture", "points": 8 }, { "category": "culture", "points": 8 }, { "category": "dialect", "points": 5 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-minor", "points": 2 }, { "category": "filler", "points": 3 } ] }, "judge_disagreement": 0 }, { "idx": 41, "duration_s": 34, "mean_score": 93.0, "mean_raw": 7.0, "scores_per_judge": [ 93 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-minor", "points": 2 }, { "category": "filler", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 42, "duration_s": 43, "mean_score": 72.0, "mean_raw": 28.0, "scores_per_judge": [ 72 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "culture", "points": 8 }, { "category": "culture", "points": 8 }, { "category": "culture", "points": 8 }, { "category": "mishear-minor", "points": 2 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 43, "duration_s": 29, "mean_score": 72.0, "mean_raw": 28.0, "scores_per_judge": [ 72 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "culture", "points": 8 }, { "category": "culture", "points": 8 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-minor", "points": 2 }, { "category": "filler", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 44, "duration_s": 71, "mean_score": 62.0, "mean_raw": 38.0, "scores_per_judge": [ 62 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "proper-noun", "points": 6 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-minor", "points": 2 }, { "category": "culture", "points": 8 }, { "category": "culture", "points": 8 }, { "category": "culture", "points": 8 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 45, "duration_s": 46, "mean_score": 29.0, "mean_raw": 71.0, "scores_per_judge": [ 29 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "culture", "points": 8 }, { "category": "culture", "points": 8 }, { "category": "culture", "points": 8 }, { "category": "dialect", "points": 5 }, { "category": "dialect", "points": 5 }, { "category": "dialect", "points": 5 }, { "category": "dialect", "points": 5 }, { "category": "dialect", "points": 5 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-minor", "points": 2 }, { "category": "filler", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 46, "duration_s": 62, "mean_score": 87.0, "mean_raw": 13.0, "scores_per_judge": [ 87 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 }, { "category": "paraphrase", "points": 3 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 47, "duration_s": 67, "mean_score": 85.0, "mean_raw": 15.0, "scores_per_judge": [ 85 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "brand", "points": 10 }, { "category": "mishear-minor", "points": 2 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 48, "duration_s": 72, "mean_score": 87.0, "mean_raw": 13.0, "scores_per_judge": [ 87 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "tech", "points": 8 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 49, "duration_s": 87, "mean_score": 81.0, "mean_raw": 19.0, "scores_per_judge": [ 81 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "tech", "points": 8 }, { "category": "paraphrase", "points": 3 }, { "category": "paraphrase", "points": 3 }, { "category": "filler", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 50, "duration_s": 106, "mean_score": 64.0, "mean_raw": 36.0, "scores_per_judge": [ 64 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "culture", "points": 8 }, { "category": "culture", "points": 8 }, { "category": "tech", "points": 8 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-minor", "points": 2 } ] }, "judge_disagreement": 0 }, { "idx": 51, "duration_s": 33, "mean_score": 80.0, "mean_raw": 20.0, "scores_per_judge": [ 80 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 52, "duration_s": 25, "mean_score": 87.0, "mean_raw": 13.0, "scores_per_judge": [ 87 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "proper-noun", "points": 6 }, { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 2 } ] }, "judge_disagreement": 0 }, { "idx": 53, "duration_s": 35, "mean_score": 91.0, "mean_raw": 9.0, "scores_per_judge": [ 91 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "proper-noun", "points": 6 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 54, "duration_s": 36, "mean_score": 95.0, "mean_raw": 5.0, "scores_per_judge": [ 95 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 55, "duration_s": 33, "mean_score": 91.0, "mean_raw": 9.0, "scores_per_judge": [ 91 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 5 }, { "category": "mishear-minor", "points": 2 }, { "category": "mishear-minor", "points": 2 } ] }, "judge_disagreement": 0 }, { "idx": 56, "duration_s": 15, "mean_score": 98.0, "mean_raw": 2.0, "scores_per_judge": [ 98 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 57, "duration_s": 38, "mean_score": 54.0, "mean_raw": 46.0, "scores_per_judge": [ 54 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "polarity", "points": 35 }, { "category": "mishear-minor", "points": 2 }, { "category": "mishear-minor", "points": 2 }, { "category": "paraphrase", "points": 3 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 58, "duration_s": 15, "mean_score": 47.0, "mean_raw": 53.0, "scores_per_judge": [ 47 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "polarity", "points": 35 }, { "category": "brand", "points": 10 }, { "category": "tech", "points": 8 } ] }, "judge_disagreement": 0 }, { "idx": 59, "duration_s": 28, "mean_score": 88.0, "mean_raw": 12.0, "scores_per_judge": [ 88 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "tech", "points": 8 }, { "category": "mishear-minor", "points": 2 }, { "category": "mishear-minor", "points": 2 } ] }, "judge_disagreement": 0 }, { "idx": 60, "duration_s": 18, "mean_score": 33.0, "mean_raw": 67.0, "scores_per_judge": [ 33 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "polarity", "points": 35 }, { "category": "semantic-inversion", "points": 20 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 61, "duration_s": 19, "mean_score": 82.0, "mean_raw": 18.0, "scores_per_judge": [ 82 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 }, { "category": "brand", "points": 10 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 62, "duration_s": 17, "mean_score": 98.0, "mean_raw": 2.0, "scores_per_judge": [ 98 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-minor", "points": 2 } ] }, "judge_disagreement": 0 }, { "idx": 63, "duration_s": 21, "mean_score": 79.0, "mean_raw": 21.0, "scores_per_judge": [ 79 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "hallucination", "points": 15 }, { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 64, "duration_s": 23, "mean_score": 98.0, "mean_raw": 2.0, "scores_per_judge": [ 98 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 65, "duration_s": 20, "mean_score": 98.0, "mean_raw": 2.0, "scores_per_judge": [ 98 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 66, "duration_s": 24, "mean_score": 87.0, "mean_raw": 13.0, "scores_per_judge": [ 87 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "tech", "points": 8 }, { "category": "mishear-severe", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 67, "duration_s": 16, "mean_score": 93.0, "mean_raw": 7.0, "scores_per_judge": [ 93 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 }, { "category": "mishear-minor", "points": 2 } ] }, "judge_disagreement": 0 }, { "idx": 68, "duration_s": 18, "mean_score": 89.0, "mean_raw": 11.0, "scores_per_judge": [ 89 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "brand", "points": 10 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 69, "duration_s": 18, "mean_score": 85.0, "mean_raw": 15.0, "scores_per_judge": [ 85 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "tech", "points": 8 }, { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 70, "duration_s": 17, "mean_score": 83.0, "mean_raw": 17.0, "scores_per_judge": [ 83 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 }, { "category": "tech", "points": 8 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 71, "duration_s": 51, "mean_score": 78.0, "mean_raw": 22.0, "scores_per_judge": [ 78 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "tech", "points": 8 }, { "category": "mishear-severe", "points": 5 }, { "category": "content-drop", "points": 3 }, { "category": "mishear-minor", "points": 2 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 72, "duration_s": 54, "mean_score": 87.0, "mean_raw": 13.0, "scores_per_judge": [ 87 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "tech", "points": 8 }, { "category": "mishear-minor", "points": 2 }, { "category": "mishear-minor", "points": 2 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 73, "duration_s": 56, "mean_score": 67.0, "mean_raw": 33.0, "scores_per_judge": [ 67 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 }, { "category": "mishear-minor", "points": 2 }, { "category": "brand", "points": 10 }, { "category": "mishear-minor", "points": 2 }, { "category": "mishear-minor", "points": 2 }, { "category": "mishear-minor", "points": 2 }, { "category": "proper-noun", "points": 6 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 74, "duration_s": 45, "mean_score": 99.0, "mean_raw": 1.0, "scores_per_judge": [ 99 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 75, "duration_s": 74, "mean_score": 69.0, "mean_raw": 31.0, "scores_per_judge": [ 69 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "hallucination", "points": 15 }, { "category": "tech", "points": 8 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-minor", "points": 2 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 76, "duration_s": 58, "mean_score": 64.0, "mean_raw": 36.0, "scores_per_judge": [ 64 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "semantic-inversion", "points": 20 }, { "category": "proper-noun", "points": 6 }, { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 77, "duration_s": 130, "mean_score": 62.0, "mean_raw": 38.0, "scores_per_judge": [ 62 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "semantic-inversion", "points": 20 }, { "category": "brand", "points": 10 }, { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 3 } ] }, "judge_disagreement": 0 }, { "idx": 78, "duration_s": 61, "mean_score": 90.0, "mean_raw": 10.0, "scores_per_judge": [ 90 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 79, "duration_s": 47, "mean_score": 83.0, "mean_raw": 17.0, "scores_per_judge": [ 83 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 }, { "category": "paraphrase", "points": 3 }, { "category": "mishear-minor", "points": 2 }, { "category": "mishear-minor", "points": 2 }, { "category": "filler", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 80, "duration_s": 42, "mean_score": 98.0, "mean_raw": 2.0, "scores_per_judge": [ 98 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 81, "duration_s": 38, "mean_score": 91.0, "mean_raw": 9.0, "scores_per_judge": [ 91 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "paraphrase", "points": 3 }, { "category": "mishear-minor", "points": 2 }, { "category": "mishear-minor", "points": 2 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 82, "duration_s": 38, "mean_score": 92.0, "mean_raw": 8.0, "scores_per_judge": [ 92 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "content-drop", "points": 3 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 83, "duration_s": 37, "mean_score": 95.0, "mean_raw": 5.0, "scores_per_judge": [ 95 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 84, "duration_s": 36, "mean_score": 95.0, "mean_raw": 5.0, "scores_per_judge": [ 95 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 85, "duration_s": 36, "mean_score": 89.0, "mean_raw": 11.0, "scores_per_judge": [ 89 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 }, { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 86, "duration_s": 35, "mean_score": 97.0, "mean_raw": 3.0, "scores_per_judge": [ 97 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 87, "duration_s": 34, "mean_score": 77.0, "mean_raw": 23.0, "scores_per_judge": [ 77 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "numeric", "points": 10 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 88, "duration_s": 34, "mean_score": 94.0, "mean_raw": 6.0, "scores_per_judge": [ 94 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 89, "duration_s": 34, "mean_score": 92.0, "mean_raw": 8.0, "scores_per_judge": [ 92 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 }, { "category": "mishear-minor", "points": 2 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 90, "duration_s": 33, "mean_score": 84.0, "mean_raw": 16.0, "scores_per_judge": [ 84 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "tech", "points": 8 }, { "category": "mishear-minor", "points": 2 }, { "category": "mishear-minor", "points": 2 }, { "category": "mishear-minor", "points": 2 }, { "category": "mishear-minor", "points": 2 } ] }, "judge_disagreement": 0 }, { "idx": 91, "duration_s": 33, "mean_score": 93.0, "mean_raw": 7.0, "scores_per_judge": [ 93 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 5 }, { "category": "mishear-minor", "points": 2 } ] }, "judge_disagreement": 0 }, { "idx": 92, "duration_s": 32, "mean_score": 97.0, "mean_raw": 3.0, "scores_per_judge": [ 97 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 3 } ] }, "judge_disagreement": 0 }, { "idx": 93, "duration_s": 32, "mean_score": 91.0, "mean_raw": 9.0, "scores_per_judge": [ 91 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "content-drop", "points": 3 }, { "category": "mishear-minor", "points": 2 }, { "category": "filler", "points": 4 } ] }, "judge_disagreement": 0 }, { "idx": 94, "duration_s": 32, "mean_score": 90.0, "mean_raw": 10.0, "scores_per_judge": [ 90 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 95, "duration_s": 32, "mean_score": 93.0, "mean_raw": 7.0, "scores_per_judge": [ 93 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "paraphrase", "points": 3 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 96, "duration_s": 31, "mean_score": 99.0, "mean_raw": 1.0, "scores_per_judge": [ 99 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 97, "duration_s": 30, "mean_score": 95.0, "mean_raw": 5.0, "scores_per_judge": [ 95 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 98, "duration_s": 30, "mean_score": 97.0, "mean_raw": 3.0, "scores_per_judge": [ 97 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 99, "duration_s": 30, "mean_score": 96.0, "mean_raw": 4.0, "scores_per_judge": [ 96 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "paraphrase", "points": 3 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 100, "duration_s": 30, "mean_score": 97.0, "mean_raw": 3.0, "scores_per_judge": [ 97 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 } ], "total_deductions": 1636, "avg_deductions_per_sample": 16.36, "mean_clamped_score": 83.6, "n_judge_sample_pairs": 100, "gt_chars": 22625, "ded_per_100char": 7.23, "fidelity_score": 63.8, "overall_score": 83.6, "overall_raw_deductions": 16.36, "score_per_judge": { "deepseek_flash": 83.6 }, "per_group_deductions": { "meaning-flip": { "deepseek_flash": 255 }, "entity": { "deepseek_flash": 424 }, "cultural": { "deepseek_flash": 166 }, "fidelity": { "deepseek_flash": 791 } }, "per_bucket": { "general": { "n_samples": 40, "mean_raw": 16.9, "mean_score": 83.1, "total_raw": 675.0, "idxs": [ 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40 ], "worst_idx": 40 }, "idiom-test": { "n_samples": 5, "mean_raw": 34.4, "mean_score": 65.6, "total_raw": 172.0, "idxs": [ 41, 42, 43, 44, 45 ], "worst_idx": 45 }, "long-mixed": { "n_samples": 5, "mean_raw": 19.2, "mean_score": 80.8, "total_raw": 96.0, "idxs": [ 46, 47, 48, 49, 50 ], "worst_idx": 50 } }, "per_group_per_100char": { "meaning-flip": 1.127, "entity": 1.874, "cultural": 0.734, "fidelity": 3.496 }, "per_group_fidelity": { "meaning-flip": 73.0, "entity": 66.8, "cultural": 55.6, "fidelity": 59.0 } }, "glm_asr_nano": { "scores_by_judge": { "deepseek_flash": [ 100, 100, 100, 100, 97, 99, 98, 100, 95, 99, 83, 75, 61, 92, 95, 79, 81, 87, 47, 94, 99, 90, 100, 99, 92, 100, 100, 89, 69, 65, 63, 99, 76, 62, 89, 96, 90, 70, 60, 87, 100, 90, 89, 84, 90, 82, 92, 0, 92, 89, 84, 96, 97, 97, 92, 90, 95, 99, 96, 48, 100, 100, 98, 96, 100, 90, 99, 90, 99, 64, 79, 95, 89, 100, 91, 71, 83, 97, 95, 98, 76, 75, 92, 63, 84, 82, 82, 96, 97, 90, 63, 100, 84, 96, 97, 100, 98, 99, 100, 95 ] }, "per_category_deductions": { "polarity": { "deepseek_flash": 245 }, "semantic-inversion": { "deepseek_flash": 60 }, "hallucination": { "deepseek_flash": 30 }, "numeric": { "deepseek_flash": 20 }, "brand": { "deepseek_flash": 170 }, "tech": { "deepseek_flash": 120 }, "culture": { "deepseek_flash": 48 }, "proper-noun": { "deepseek_flash": 36 }, "dialect": { "deepseek_flash": 0 }, "mishear-severe": { "deepseek_flash": 130 }, "content-drop": { "deepseek_flash": 75 }, "paraphrase": { "deepseek_flash": 12 }, "mishear-minor": { "deepseek_flash": 22 }, "filler": { "deepseek_flash": 149 }, "format": { "deepseek_flash": 1 }, "unscoreable": { "deepseek_flash": 100 } }, "per_tier_deductions": { "tier1": { "deepseek_flash": 355 }, "tier2": { "deepseek_flash": 374 }, "tier3": { "deepseek_flash": 239 }, "tier4": { "deepseek_flash": 150 }, "unscoreable": { "deepseek_flash": 100 } }, "raw_deductions_by_judge": { "deepseek_flash": [ 0, 0, 0, 0, 3, 1, 2, 0, 5, 1, 17, 25, 39, 8, 5, 21, 19, 13, 53, 6, 1, 10, 0, 1, 8, 0, 0, 11, 31, 35, 37, 1, 24, 38, 11, 4, 10, 30, 40, 13, 0, 10, 11, 16, 10, 18, 8, 100, 8, 11, 16, 4, 3, 3, 8, 10, 5, 1, 4, 52, 0, 0, 2, 4, 0, 10, 1, 10, 1, 36, 21, 5, 11, 0, 9, 29, 17, 3, 5, 2, 24, 25, 8, 37, 16, 18, 18, 4, 3, 10, 37, 0, 16, 4, 3, 0, 2, 1, 0, 5 ] }, "sample_details": [ { "idx": 1, "duration_s": 5, "mean_score": 100.0, "mean_raw": 0.0, "scores_per_judge": [ 100 ], "deductions_per_judge": { "deepseek_flash": [] }, "judge_disagreement": 0 }, { "idx": 2, "duration_s": 7, "mean_score": 100.0, "mean_raw": 0.0, "scores_per_judge": [ 100 ], "deductions_per_judge": { "deepseek_flash": [] }, "judge_disagreement": 0 }, { "idx": 3, "duration_s": 8, "mean_score": 100.0, "mean_raw": 0.0, "scores_per_judge": [ 100 ], "deductions_per_judge": { "deepseek_flash": [] }, "judge_disagreement": 0 }, { "idx": 4, "duration_s": 11, "mean_score": 100.0, "mean_raw": 0.0, "scores_per_judge": [ 100 ], "deductions_per_judge": { "deepseek_flash": [] }, "judge_disagreement": 0 }, { "idx": 5, "duration_s": 16, "mean_score": 97.0, "mean_raw": 3.0, "scores_per_judge": [ 97 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 6, "duration_s": 17, "mean_score": 99.0, "mean_raw": 1.0, "scores_per_judge": [ 99 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 7, "duration_s": 20, "mean_score": 98.0, "mean_raw": 2.0, "scores_per_judge": [ 98 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 8, "duration_s": 30, "mean_score": 100.0, "mean_raw": 0.0, "scores_per_judge": [ 100 ], "deductions_per_judge": { "deepseek_flash": [] }, "judge_disagreement": 0 }, { "idx": 9, "duration_s": 32, "mean_score": 95.0, "mean_raw": 5.0, "scores_per_judge": [ 95 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 10, "duration_s": 35, "mean_score": 99.0, "mean_raw": 1.0, "scores_per_judge": [ 99 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 11, "duration_s": 40, "mean_score": 83.0, "mean_raw": 17.0, "scores_per_judge": [ 83 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "tech", "points": 8 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-minor", "points": 2 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 12, "duration_s": 46, "mean_score": 75.0, "mean_raw": 25.0, "scores_per_judge": [ 75 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "semantic-inversion", "points": 20 }, { "category": "mishear-severe", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 13, "duration_s": 60, "mean_score": 61.0, "mean_raw": 39.0, "scores_per_judge": [ 61 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "tech", "points": 8 }, { "category": "tech", "points": 8 }, { "category": "brand", "points": 10 }, { "category": "brand", "points": 10 }, { "category": "mishear-minor", "points": 2 } ] }, "judge_disagreement": 0 }, { "idx": 14, "duration_s": 64, "mean_score": 92.0, "mean_raw": 8.0, "scores_per_judge": [ 92 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 }, { "category": "content-drop", "points": 3 } ] }, "judge_disagreement": 0 }, { "idx": 15, "duration_s": 69, "mean_score": 95.0, "mean_raw": 5.0, "scores_per_judge": [ 95 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 16, "duration_s": 81, "mean_score": 79.0, "mean_raw": 21.0, "scores_per_judge": [ 79 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "tech", "points": 8 }, { "category": "proper-noun", "points": 6 }, { "category": "content-drop", "points": 3 }, { "category": "content-drop", "points": 3 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 17, "duration_s": 100, "mean_score": 81.0, "mean_raw": 19.0, "scores_per_judge": [ 81 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "brand", "points": 10 }, { "category": "content-drop", "points": 3 }, { "category": "content-drop", "points": 3 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 18, "duration_s": 125, "mean_score": 87.0, "mean_raw": 13.0, "scores_per_judge": [ 87 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 }, { "category": "content-drop", "points": 3 }, { "category": "mishear-minor", "points": 2 }, { "category": "mishear-minor", "points": 2 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 19, "duration_s": 133, "mean_score": 47.0, "mean_raw": 53.0, "scores_per_judge": [ 47 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 5 }, { "category": "tech", "points": 8 }, { "category": "brand", "points": 10 }, { "category": "brand", "points": 10 }, { "category": "mishear-severe", "points": 5 }, { "category": "hallucination", "points": 15 } ] }, "judge_disagreement": 0 }, { "idx": 20, "duration_s": 164, "mean_score": 94.0, "mean_raw": 6.0, "scores_per_judge": [ 94 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 5 }, { "category": "format", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 21, "duration_s": 5, "mean_score": 99.0, "mean_raw": 1.0, "scores_per_judge": [ 99 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 22, "duration_s": 7, "mean_score": 90.0, "mean_raw": 10.0, "scores_per_judge": [ 90 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "brand", "points": 10 } ] }, "judge_disagreement": 0 }, { "idx": 23, "duration_s": 10, "mean_score": 100.0, "mean_raw": 0.0, "scores_per_judge": [ 100 ], "deductions_per_judge": { "deepseek_flash": [] }, "judge_disagreement": 0 }, { "idx": 24, "duration_s": 14, "mean_score": 99.0, "mean_raw": 1.0, "scores_per_judge": [ 99 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 25, "duration_s": 16, "mean_score": 92.0, "mean_raw": 8.0, "scores_per_judge": [ 92 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "tech", "points": 8 } ] }, "judge_disagreement": 0 }, { "idx": 26, "duration_s": 23, "mean_score": 100.0, "mean_raw": 0.0, "scores_per_judge": [ 100 ], "deductions_per_judge": { "deepseek_flash": [] }, "judge_disagreement": 0 }, { "idx": 27, "duration_s": 25, "mean_score": 100.0, "mean_raw": 0.0, "scores_per_judge": [ 100 ], "deductions_per_judge": { "deepseek_flash": [] }, "judge_disagreement": 0 }, { "idx": 28, "duration_s": 30, "mean_score": 89.0, "mean_raw": 11.0, "scores_per_judge": [ 89 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 }, { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 29, "duration_s": 34, "mean_score": 69.0, "mean_raw": 31.0, "scores_per_judge": [ 69 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "brand", "points": 10 }, { "category": "brand", "points": 10 }, { "category": "brand", "points": 10 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 30, "duration_s": 38, "mean_score": 65.0, "mean_raw": 35.0, "scores_per_judge": [ 65 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "polarity", "points": 35 } ] }, "judge_disagreement": 0 }, { "idx": 31, "duration_s": 44, "mean_score": 63.0, "mean_raw": 37.0, "scores_per_judge": [ 63 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "polarity", "points": 35 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 32, "duration_s": 50, "mean_score": 99.0, "mean_raw": 1.0, "scores_per_judge": [ 99 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 33, "duration_s": 62, "mean_score": 76.0, "mean_raw": 24.0, "scores_per_judge": [ 76 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "brand", "points": 10 }, { "category": "proper-noun", "points": 6 }, { "category": "content-drop", "points": 3 }, { "category": "paraphrase", "points": 3 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 34, "duration_s": 66, "mean_score": 62.0, "mean_raw": 38.0, "scores_per_judge": [ 62 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "polarity", "points": 35 }, { "category": "content-drop", "points": 3 } ] }, "judge_disagreement": 0 }, { "idx": 35, "duration_s": 77, "mean_score": 89.0, "mean_raw": 11.0, "scores_per_judge": [ 89 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "tech", "points": 8 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 36, "duration_s": 94, "mean_score": 96.0, "mean_raw": 4.0, "scores_per_judge": [ 96 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "content-drop", "points": 3 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 37, "duration_s": 106, "mean_score": 90.0, "mean_raw": 10.0, "scores_per_judge": [ 90 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 38, "duration_s": 126, "mean_score": 70.0, "mean_raw": 30.0, "scores_per_judge": [ 70 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "culture", "points": 8 }, { "category": "tech", "points": 8 }, { "category": "tech", "points": 8 }, { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 39, "duration_s": 158, "mean_score": 60.0, "mean_raw": 40.0, "scores_per_judge": [ 60 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "brand", "points": 10 }, { "category": "brand", "points": 10 }, { "category": "brand", "points": 10 }, { "category": "proper-noun", "points": 6 }, { "category": "content-drop", "points": 3 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 40, "duration_s": 279, "mean_score": 87.0, "mean_raw": 13.0, "scores_per_judge": [ 87 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 }, { "category": "content-drop", "points": 3 }, { "category": "mishear-severe", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 41, "duration_s": 34, "mean_score": 100.0, "mean_raw": 0.0, "scores_per_judge": [ 100 ], "deductions_per_judge": { "deepseek_flash": [] }, "judge_disagreement": 0 }, { "idx": 42, "duration_s": 43, "mean_score": 90.0, "mean_raw": 10.0, "scores_per_judge": [ 90 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "culture", "points": 8 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 43, "duration_s": 29, "mean_score": 89.0, "mean_raw": 11.0, "scores_per_judge": [ 89 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "culture", "points": 8 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 44, "duration_s": 71, "mean_score": 84.0, "mean_raw": 16.0, "scores_per_judge": [ 84 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "proper-noun", "points": 6 }, { "category": "mishear-minor", "points": 2 }, { "category": "mishear-severe", "points": 5 }, { "category": "content-drop", "points": 3 } ] }, "judge_disagreement": 0 }, { "idx": 45, "duration_s": 46, "mean_score": 90.0, "mean_raw": 10.0, "scores_per_judge": [ 90 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "content-drop", "points": 3 }, { "category": "paraphrase", "points": 3 }, { "category": "mishear-minor", "points": 2 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 46, "duration_s": 62, "mean_score": 82.0, "mean_raw": 18.0, "scores_per_judge": [ 82 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "culture", "points": 8 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-minor", "points": 2 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 47, "duration_s": 67, "mean_score": 92.0, "mean_raw": 8.0, "scores_per_judge": [ 92 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "tech", "points": 8 } ] }, "judge_disagreement": 0 }, { "idx": 48, "duration_s": 72, "mean_score": 0.0, "mean_raw": 100.0, "scores_per_judge": [ 0 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "unscoreable", "points": 100 } ] }, "judge_disagreement": 0 }, { "idx": 49, "duration_s": 87, "mean_score": 92.0, "mean_raw": 8.0, "scores_per_judge": [ 92 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 5 }, { "category": "paraphrase", "points": 3 } ] }, "judge_disagreement": 0 }, { "idx": 50, "duration_s": 106, "mean_score": 89.0, "mean_raw": 11.0, "scores_per_judge": [ 89 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "culture", "points": 8 }, { "category": "content-drop", "points": 3 } ] }, "judge_disagreement": 0 }, { "idx": 51, "duration_s": 33, "mean_score": 84.0, "mean_raw": 16.0, "scores_per_judge": [ 84 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-severe", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 52, "duration_s": 25, "mean_score": 96.0, "mean_raw": 4.0, "scores_per_judge": [ 96 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-minor", "points": 2 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 53, "duration_s": 35, "mean_score": 97.0, "mean_raw": 3.0, "scores_per_judge": [ 97 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 54, "duration_s": 36, "mean_score": 97.0, "mean_raw": 3.0, "scores_per_judge": [ 97 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 55, "duration_s": 33, "mean_score": 92.0, "mean_raw": 8.0, "scores_per_judge": [ 92 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "content-drop", "points": 3 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 56, "duration_s": 15, "mean_score": 90.0, "mean_raw": 10.0, "scores_per_judge": [ 90 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "brand", "points": 10 } ] }, "judge_disagreement": 0 }, { "idx": 57, "duration_s": 38, "mean_score": 95.0, "mean_raw": 5.0, "scores_per_judge": [ 95 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "content-drop", "points": 3 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 58, "duration_s": 15, "mean_score": 99.0, "mean_raw": 1.0, "scores_per_judge": [ 99 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 59, "duration_s": 28, "mean_score": 96.0, "mean_raw": 4.0, "scores_per_judge": [ 96 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-minor", "points": 2 }, { "category": "filler", "points": 2 } ] }, "judge_disagreement": 0 }, { "idx": 60, "duration_s": 18, "mean_score": 48.0, "mean_raw": 52.0, "scores_per_judge": [ 48 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "polarity", "points": 35 }, { "category": "hallucination", "points": 15 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 61, "duration_s": 19, "mean_score": 100.0, "mean_raw": 0.0, "scores_per_judge": [ 100 ], "deductions_per_judge": { "deepseek_flash": [] }, "judge_disagreement": 0 }, { "idx": 62, "duration_s": 17, "mean_score": 100.0, "mean_raw": 0.0, "scores_per_judge": [ 100 ], "deductions_per_judge": { "deepseek_flash": [] }, "judge_disagreement": 0 }, { "idx": 63, "duration_s": 21, "mean_score": 98.0, "mean_raw": 2.0, "scores_per_judge": [ 98 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 64, "duration_s": 23, "mean_score": 96.0, "mean_raw": 4.0, "scores_per_judge": [ 96 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 65, "duration_s": 20, "mean_score": 100.0, "mean_raw": 0.0, "scores_per_judge": [ 100 ], "deductions_per_judge": { "deepseek_flash": [] }, "judge_disagreement": 0 }, { "idx": 66, "duration_s": 24, "mean_score": 90.0, "mean_raw": 10.0, "scores_per_judge": [ 90 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "culture", "points": 8 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 67, "duration_s": 16, "mean_score": 99.0, "mean_raw": 1.0, "scores_per_judge": [ 99 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 68, "duration_s": 18, "mean_score": 90.0, "mean_raw": 10.0, "scores_per_judge": [ 90 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "brand", "points": 10 } ] }, "judge_disagreement": 0 }, { "idx": 69, "duration_s": 18, "mean_score": 99.0, "mean_raw": 1.0, "scores_per_judge": [ 99 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 70, "duration_s": 17, "mean_score": 64.0, "mean_raw": 36.0, "scores_per_judge": [ 64 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "polarity", "points": 35 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 71, "duration_s": 51, "mean_score": 79.0, "mean_raw": 21.0, "scores_per_judge": [ 79 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "brand", "points": 10 }, { "category": "tech", "points": 8 }, { "category": "filler", "points": 3 } ] }, "judge_disagreement": 0 }, { "idx": 72, "duration_s": 54, "mean_score": 95.0, "mean_raw": 5.0, "scores_per_judge": [ 95 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "content-drop", "points": 3 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 73, "duration_s": 56, "mean_score": 89.0, "mean_raw": 11.0, "scores_per_judge": [ 89 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 }, { "category": "content-drop", "points": 3 }, { "category": "paraphrase", "points": 3 } ] }, "judge_disagreement": 0 }, { "idx": 74, "duration_s": 45, "mean_score": 100.0, "mean_raw": 0.0, "scores_per_judge": [ 100 ], "deductions_per_judge": { "deepseek_flash": [] }, "judge_disagreement": 0 }, { "idx": 75, "duration_s": 74, "mean_score": 91.0, "mean_raw": 9.0, "scores_per_judge": [ 91 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "content-drop", "points": 3 }, { "category": "proper-noun", "points": 6 } ] }, "judge_disagreement": 0 }, { "idx": 76, "duration_s": 58, "mean_score": 71.0, "mean_raw": 29.0, "scores_per_judge": [ 71 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "semantic-inversion", "points": 20 }, { "category": "proper-noun", "points": 6 }, { "category": "filler", "points": 3 } ] }, "judge_disagreement": 0 }, { "idx": 77, "duration_s": 130, "mean_score": 83.0, "mean_raw": 17.0, "scores_per_judge": [ 83 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "brand", "points": 10 }, { "category": "content-drop", "points": 3 }, { "category": "filler", "points": 4 } ] }, "judge_disagreement": 0 }, { "idx": 78, "duration_s": 61, "mean_score": 97.0, "mean_raw": 3.0, "scores_per_judge": [ 97 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "content-drop", "points": 3 } ] }, "judge_disagreement": 0 }, { "idx": 79, "duration_s": 47, "mean_score": 95.0, "mean_raw": 5.0, "scores_per_judge": [ 95 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 80, "duration_s": 42, "mean_score": 98.0, "mean_raw": 2.0, "scores_per_judge": [ 98 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 81, "duration_s": 38, "mean_score": 76.0, "mean_raw": 24.0, "scores_per_judge": [ 76 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "semantic-inversion", "points": 20 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 82, "duration_s": 38, "mean_score": 75.0, "mean_raw": 25.0, "scores_per_judge": [ 75 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "numeric", "points": 10 }, { "category": "tech", "points": 8 }, { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 83, "duration_s": 37, "mean_score": 92.0, "mean_raw": 8.0, "scores_per_judge": [ 92 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 84, "duration_s": 36, "mean_score": 63.0, "mean_raw": 37.0, "scores_per_judge": [ 63 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "polarity", "points": 35 }, { "category": "mishear-minor", "points": 2 } ] }, "judge_disagreement": 0 }, { "idx": 85, "duration_s": 36, "mean_score": 84.0, "mean_raw": 16.0, "scores_per_judge": [ 84 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 }, { "category": "mishear-minor", "points": 2 }, { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 86, "duration_s": 35, "mean_score": 82.0, "mean_raw": 18.0, "scores_per_judge": [ 82 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "tech", "points": 8 }, { "category": "tech", "points": 8 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 87, "duration_s": 34, "mean_score": 82.0, "mean_raw": 18.0, "scores_per_judge": [ 82 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "numeric", "points": 10 }, { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 88, "duration_s": 34, "mean_score": 96.0, "mean_raw": 4.0, "scores_per_judge": [ 96 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "content-drop", "points": 3 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 89, "duration_s": 34, "mean_score": 97.0, "mean_raw": 3.0, "scores_per_judge": [ 97 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "content-drop", "points": 3 } ] }, "judge_disagreement": 0 }, { "idx": 90, "duration_s": 33, "mean_score": 90.0, "mean_raw": 10.0, "scores_per_judge": [ 90 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "tech", "points": 8 }, { "category": "filler", "points": 2 } ] }, "judge_disagreement": 0 }, { "idx": 91, "duration_s": 33, "mean_score": 63.0, "mean_raw": 37.0, "scores_per_judge": [ 63 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "polarity", "points": 35 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 92, "duration_s": 32, "mean_score": 100.0, "mean_raw": 0.0, "scores_per_judge": [ 100 ], "deductions_per_judge": { "deepseek_flash": [] }, "judge_disagreement": 0 }, { "idx": 93, "duration_s": 32, "mean_score": 84.0, "mean_raw": 16.0, "scores_per_judge": [ 84 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "content-drop", "points": 3 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-severe", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 94, "duration_s": 32, "mean_score": 96.0, "mean_raw": 4.0, "scores_per_judge": [ 96 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "content-drop", "points": 3 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 95, "duration_s": 32, "mean_score": 97.0, "mean_raw": 3.0, "scores_per_judge": [ 97 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 96, "duration_s": 31, "mean_score": 100.0, "mean_raw": 0.0, "scores_per_judge": [ 100 ], "deductions_per_judge": { "deepseek_flash": [] }, "judge_disagreement": 0 }, { "idx": 97, "duration_s": 30, "mean_score": 98.0, "mean_raw": 2.0, "scores_per_judge": [ 98 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 98, "duration_s": 30, "mean_score": 99.0, "mean_raw": 1.0, "scores_per_judge": [ 99 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 99, "duration_s": 30, "mean_score": 100.0, "mean_raw": 0.0, "scores_per_judge": [ 100 ], "deductions_per_judge": { "deepseek_flash": [] }, "judge_disagreement": 0 }, { "idx": 100, "duration_s": 30, "mean_score": 95.0, "mean_raw": 5.0, "scores_per_judge": [ 95 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 } ] }, "judge_disagreement": 0 } ], "total_deductions": 1218, "avg_deductions_per_sample": 12.18, "mean_clamped_score": 87.8, "n_judge_sample_pairs": 100, "gt_chars": 22625, "ded_per_100char": 5.38, "fidelity_score": 73.1, "overall_score": 87.8, "overall_raw_deductions": 12.18, "score_per_judge": { "deepseek_flash": 87.8 }, "per_group_deductions": { "meaning-flip": { "deepseek_flash": 355 }, "entity": { "deepseek_flash": 326 }, "cultural": { "deepseek_flash": 48 }, "fidelity": { "deepseek_flash": 389 } }, "per_bucket": { "general": { "n_samples": 40, "mean_raw": 13.1, "mean_score": 86.9, "total_raw": 523.0, "idxs": [ 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40 ], "worst_idx": 19 }, "idiom-test": { "n_samples": 5, "mean_raw": 9.4, "mean_score": 90.6, "total_raw": 47.0, "idxs": [ 41, 42, 43, 44, 45 ], "worst_idx": 44 }, "long-mixed": { "n_samples": 5, "mean_raw": 29.0, "mean_score": 71.0, "total_raw": 145.0, "idxs": [ 46, 47, 48, 49, 50 ], "worst_idx": 48 } }, "per_group_per_100char": { "meaning-flip": 1.569, "entity": 1.441, "cultural": 0.212, "fidelity": 1.719 }, "per_group_fidelity": { "meaning-flip": 62.4, "entity": 74.5, "cultural": 87.2, "fidelity": 79.9 } }, "gpt_transcribe": { "scores_by_judge": { "deepseek_flash": [ 100, 100, 99, 98, 97, 97, 79, 100, 98, 97, 90, 85, 83, 82, 95, 89, 95, 84, 77, 80, 100, 94, 100, 89, 96, 97, 100, 93, 97, 96, 95, 82, 89, 89, 88, 95, 95, 85, 61, 87, 92, 95, 89, 89, 44, 84, 98, 87, 95, 87, 92, 98, 98, 96, 95, 87, 89, 95, 88, 63, 100, 100, 98, 95, 100, 87, 98, 85, 98, 98, 77, 80, 87, 97, 93, 68, 65, 96, 97, 98, 91, 85, 93, 95, 89, 84, 95, 100, 100, 98, 97, 100, 95, 97, 95, 99, 97, 99, 100, 97 ] }, "per_category_deductions": { "polarity": { "deepseek_flash": 35 }, "semantic-inversion": { "deepseek_flash": 80 }, "hallucination": { "deepseek_flash": 15 }, "numeric": { "deepseek_flash": 0 }, "brand": { "deepseek_flash": 60 }, "tech": { "deepseek_flash": 80 }, "culture": { "deepseek_flash": 40 }, "proper-noun": { "deepseek_flash": 54 }, "dialect": { "deepseek_flash": 15 }, "mishear-severe": { "deepseek_flash": 130 }, "content-drop": { "deepseek_flash": 18 }, "paraphrase": { "deepseek_flash": 6 }, "mishear-minor": { "deepseek_flash": 40 }, "filler": { "deepseek_flash": 290 }, "format": { "deepseek_flash": 0 }, "unscoreable": { "deepseek_flash": 0 } }, "per_tier_deductions": { "tier1": { "deepseek_flash": 130 }, "tier2": { "deepseek_flash": 249 }, "tier3": { "deepseek_flash": 194 }, "tier4": { "deepseek_flash": 290 }, "unscoreable": { "deepseek_flash": 0 } }, "raw_deductions_by_judge": { "deepseek_flash": [ 0, 0, 1, 2, 3, 3, 21, 0, 2, 3, 10, 15, 17, 18, 5, 11, 5, 16, 23, 20, 0, 6, 0, 11, 4, 3, 0, 7, 3, 4, 5, 18, 11, 11, 12, 5, 5, 15, 39, 13, 8, 5, 11, 11, 56, 16, 2, 13, 5, 13, 8, 2, 2, 4, 5, 13, 11, 5, 12, 37, 0, 0, 2, 5, 0, 13, 2, 15, 2, 2, 23, 20, 13, 3, 7, 32, 35, 4, 3, 2, 9, 15, 7, 5, 11, 16, 5, 0, 0, 2, 3, 0, 5, 3, 5, 1, 3, 1, 0, 3 ] }, "sample_details": [ { "idx": 1, "duration_s": 5, "mean_score": 100.0, "mean_raw": 0.0, "scores_per_judge": [ 100 ], "deductions_per_judge": { "deepseek_flash": [] }, "judge_disagreement": 0 }, { "idx": 2, "duration_s": 7, "mean_score": 100.0, "mean_raw": 0.0, "scores_per_judge": [ 100 ], "deductions_per_judge": { "deepseek_flash": [] }, "judge_disagreement": 0 }, { "idx": 3, "duration_s": 8, "mean_score": 99.0, "mean_raw": 1.0, "scores_per_judge": [ 99 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 4, "duration_s": 11, "mean_score": 98.0, "mean_raw": 2.0, "scores_per_judge": [ 98 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 5, "duration_s": 16, "mean_score": 97.0, "mean_raw": 3.0, "scores_per_judge": [ 97 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 6, "duration_s": 17, "mean_score": 97.0, "mean_raw": 3.0, "scores_per_judge": [ 97 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "paraphrase", "points": 3 } ] }, "judge_disagreement": 0 }, { "idx": 7, "duration_s": 20, "mean_score": 79.0, "mean_raw": 21.0, "scores_per_judge": [ 79 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "semantic-inversion", "points": 20 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 8, "duration_s": 30, "mean_score": 100.0, "mean_raw": 0.0, "scores_per_judge": [ 100 ], "deductions_per_judge": { "deepseek_flash": [] }, "judge_disagreement": 0 }, { "idx": 9, "duration_s": 32, "mean_score": 98.0, "mean_raw": 2.0, "scores_per_judge": [ 98 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 10, "duration_s": 35, "mean_score": 97.0, "mean_raw": 3.0, "scores_per_judge": [ 97 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 11, "duration_s": 40, "mean_score": 90.0, "mean_raw": 10.0, "scores_per_judge": [ 90 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 12, "duration_s": 46, "mean_score": 85.0, "mean_raw": 15.0, "scores_per_judge": [ 85 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 }, { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 13, "duration_s": 60, "mean_score": 83.0, "mean_raw": 17.0, "scores_per_judge": [ 83 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "brand", "points": 10 }, { "category": "mishear-minor", "points": 2 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 14, "duration_s": 64, "mean_score": 82.0, "mean_raw": 18.0, "scores_per_judge": [ 82 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "proper-noun", "points": 6 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-minor", "points": 2 }, { "category": "filler", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 15, "duration_s": 69, "mean_score": 95.0, "mean_raw": 5.0, "scores_per_judge": [ 95 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 16, "duration_s": 81, "mean_score": 89.0, "mean_raw": 11.0, "scores_per_judge": [ 89 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "proper-noun", "points": 6 }, { "category": "filler", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 17, "duration_s": 100, "mean_score": 95.0, "mean_raw": 5.0, "scores_per_judge": [ 95 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 18, "duration_s": 125, "mean_score": 84.0, "mean_raw": 16.0, "scores_per_judge": [ 84 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 }, { "category": "mishear-minor", "points": 2 }, { "category": "mishear-minor", "points": 2 }, { "category": "mishear-minor", "points": 2 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 19, "duration_s": 133, "mean_score": 77.0, "mean_raw": 23.0, "scores_per_judge": [ 77 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "tech", "points": 8 }, { "category": "mishear-severe", "points": 5 }, { "category": "content-drop", "points": 3 }, { "category": "mishear-minor", "points": 2 }, { "category": "filler", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 20, "duration_s": 164, "mean_score": 80.0, "mean_raw": 20.0, "scores_per_judge": [ 80 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 21, "duration_s": 5, "mean_score": 100.0, "mean_raw": 0.0, "scores_per_judge": [ 100 ], "deductions_per_judge": { "deepseek_flash": [] }, "judge_disagreement": 0 }, { "idx": 22, "duration_s": 7, "mean_score": 94.0, "mean_raw": 6.0, "scores_per_judge": [ 94 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "proper-noun", "points": 6 } ] }, "judge_disagreement": 0 }, { "idx": 23, "duration_s": 10, "mean_score": 100.0, "mean_raw": 0.0, "scores_per_judge": [ 100 ], "deductions_per_judge": { "deepseek_flash": [] }, "judge_disagreement": 0 }, { "idx": 24, "duration_s": 14, "mean_score": 89.0, "mean_raw": 11.0, "scores_per_judge": [ 89 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "tech", "points": 8 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 25, "duration_s": 16, "mean_score": 96.0, "mean_raw": 4.0, "scores_per_judge": [ 96 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "content-drop", "points": 3 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 26, "duration_s": 23, "mean_score": 97.0, "mean_raw": 3.0, "scores_per_judge": [ 97 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-minor", "points": 2 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 27, "duration_s": 25, "mean_score": 100.0, "mean_raw": 0.0, "scores_per_judge": [ 100 ], "deductions_per_judge": { "deepseek_flash": [] }, "judge_disagreement": 0 }, { "idx": 28, "duration_s": 30, "mean_score": 93.0, "mean_raw": 7.0, "scores_per_judge": [ 93 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 29, "duration_s": 34, "mean_score": 97.0, "mean_raw": 3.0, "scores_per_judge": [ 97 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 30, "duration_s": 38, "mean_score": 96.0, "mean_raw": 4.0, "scores_per_judge": [ 96 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 31, "duration_s": 44, "mean_score": 95.0, "mean_raw": 5.0, "scores_per_judge": [ 95 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 32, "duration_s": 50, "mean_score": 82.0, "mean_raw": 18.0, "scores_per_judge": [ 82 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "tech", "points": 8 }, { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 33, "duration_s": 62, "mean_score": 89.0, "mean_raw": 11.0, "scores_per_judge": [ 89 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "proper-noun", "points": 6 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 34, "duration_s": 66, "mean_score": 89.0, "mean_raw": 11.0, "scores_per_judge": [ 89 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "proper-noun", "points": 6 }, { "category": "filler", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 35, "duration_s": 77, "mean_score": 88.0, "mean_raw": 12.0, "scores_per_judge": [ 88 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "tech", "points": 8 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 36, "duration_s": 94, "mean_score": 95.0, "mean_raw": 5.0, "scores_per_judge": [ 95 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 37, "duration_s": 106, "mean_score": 95.0, "mean_raw": 5.0, "scores_per_judge": [ 95 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 38, "duration_s": 126, "mean_score": 85.0, "mean_raw": 15.0, "scores_per_judge": [ 85 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "content-drop", "points": 3 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-minor", "points": 2 } ] }, "judge_disagreement": 0 }, { "idx": 39, "duration_s": 158, "mean_score": 61.0, "mean_raw": 39.0, "scores_per_judge": [ 61 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "tech", "points": 8 }, { "category": "brand", "points": 10 }, { "category": "proper-noun", "points": 6 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 40, "duration_s": 279, "mean_score": 87.0, "mean_raw": 13.0, "scores_per_judge": [ 87 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "culture", "points": 8 }, { "category": "filler", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 41, "duration_s": 34, "mean_score": 92.0, "mean_raw": 8.0, "scores_per_judge": [ 92 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "proper-noun", "points": 6 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 42, "duration_s": 43, "mean_score": 95.0, "mean_raw": 5.0, "scores_per_judge": [ 95 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 43, "duration_s": 29, "mean_score": 89.0, "mean_raw": 11.0, "scores_per_judge": [ 89 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "culture", "points": 8 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 44, "duration_s": 71, "mean_score": 89.0, "mean_raw": 11.0, "scores_per_judge": [ 89 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 }, { "category": "content-drop", "points": 3 }, { "category": "filler", "points": 3 } ] }, "judge_disagreement": 0 }, { "idx": 45, "duration_s": 46, "mean_score": 44.0, "mean_raw": 56.0, "scores_per_judge": [ 44 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "dialect", "points": 5 }, { "category": "dialect", "points": 5 }, { "category": "dialect", "points": 5 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-severe", "points": 5 }, { "category": "semantic-inversion", "points": 20 }, { "category": "proper-noun", "points": 6 }, { "category": "filler", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 46, "duration_s": 62, "mean_score": 84.0, "mean_raw": 16.0, "scores_per_judge": [ 84 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "culture", "points": 8 }, { "category": "content-drop", "points": 3 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 47, "duration_s": 67, "mean_score": 98.0, "mean_raw": 2.0, "scores_per_judge": [ 98 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 48, "duration_s": 72, "mean_score": 87.0, "mean_raw": 13.0, "scores_per_judge": [ 87 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "tech", "points": 8 }, { "category": "filler", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 49, "duration_s": 87, "mean_score": 95.0, "mean_raw": 5.0, "scores_per_judge": [ 95 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 50, "duration_s": 106, "mean_score": 87.0, "mean_raw": 13.0, "scores_per_judge": [ 87 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "culture", "points": 8 }, { "category": "mishear-minor", "points": 2 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 51, "duration_s": 33, "mean_score": 92.0, "mean_raw": 8.0, "scores_per_judge": [ 92 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 52, "duration_s": 25, "mean_score": 98.0, "mean_raw": 2.0, "scores_per_judge": [ 98 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 53, "duration_s": 35, "mean_score": 98.0, "mean_raw": 2.0, "scores_per_judge": [ 98 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 54, "duration_s": 36, "mean_score": 96.0, "mean_raw": 4.0, "scores_per_judge": [ 96 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 4 } ] }, "judge_disagreement": 0 }, { "idx": 55, "duration_s": 33, "mean_score": 95.0, "mean_raw": 5.0, "scores_per_judge": [ 95 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 56, "duration_s": 15, "mean_score": 87.0, "mean_raw": 13.0, "scores_per_judge": [ 87 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "brand", "points": 10 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 57, "duration_s": 38, "mean_score": 89.0, "mean_raw": 11.0, "scores_per_judge": [ 89 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-minor", "points": 2 }, { "category": "mishear-minor", "points": 2 }, { "category": "mishear-minor", "points": 2 }, { "category": "mishear-minor", "points": 2 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 58, "duration_s": 15, "mean_score": 95.0, "mean_raw": 5.0, "scores_per_judge": [ 95 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 59, "duration_s": 28, "mean_score": 88.0, "mean_raw": 12.0, "scores_per_judge": [ 88 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 }, { "category": "mishear-minor", "points": 2 }, { "category": "filler", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 60, "duration_s": 18, "mean_score": 63.0, "mean_raw": 37.0, "scores_per_judge": [ 63 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "polarity", "points": 35 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 61, "duration_s": 19, "mean_score": 100.0, "mean_raw": 0.0, "scores_per_judge": [ 100 ], "deductions_per_judge": { "deepseek_flash": [] }, "judge_disagreement": 0 }, { "idx": 62, "duration_s": 17, "mean_score": 100.0, "mean_raw": 0.0, "scores_per_judge": [ 100 ], "deductions_per_judge": { "deepseek_flash": [] }, "judge_disagreement": 0 }, { "idx": 63, "duration_s": 21, "mean_score": 98.0, "mean_raw": 2.0, "scores_per_judge": [ 98 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 64, "duration_s": 23, "mean_score": 95.0, "mean_raw": 5.0, "scores_per_judge": [ 95 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 65, "duration_s": 20, "mean_score": 100.0, "mean_raw": 0.0, "scores_per_judge": [ 100 ], "deductions_per_judge": { "deepseek_flash": [] }, "judge_disagreement": 0 }, { "idx": 66, "duration_s": 24, "mean_score": 87.0, "mean_raw": 13.0, "scores_per_judge": [ 87 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "culture", "points": 8 }, { "category": "mishear-severe", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 67, "duration_s": 16, "mean_score": 98.0, "mean_raw": 2.0, "scores_per_judge": [ 98 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 68, "duration_s": 18, "mean_score": 85.0, "mean_raw": 15.0, "scores_per_judge": [ 85 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "hallucination", "points": 15 } ] }, "judge_disagreement": 0 }, { "idx": 69, "duration_s": 18, "mean_score": 98.0, "mean_raw": 2.0, "scores_per_judge": [ 98 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 70, "duration_s": 17, "mean_score": 98.0, "mean_raw": 2.0, "scores_per_judge": [ 98 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 71, "duration_s": 51, "mean_score": 77.0, "mean_raw": 23.0, "scores_per_judge": [ 77 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "brand", "points": 10 }, { "category": "tech", "points": 8 }, { "category": "filler", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 72, "duration_s": 54, "mean_score": 80.0, "mean_raw": 20.0, "scores_per_judge": [ 80 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 }, { "category": "mishear-minor", "points": 2 }, { "category": "tech", "points": 8 }, { "category": "filler", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 73, "duration_s": 56, "mean_score": 87.0, "mean_raw": 13.0, "scores_per_judge": [ 87 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "brand", "points": 10 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 74, "duration_s": 45, "mean_score": 97.0, "mean_raw": 3.0, "scores_per_judge": [ 97 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 75, "duration_s": 74, "mean_score": 93.0, "mean_raw": 7.0, "scores_per_judge": [ 93 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 5 }, { "category": "mishear-minor", "points": 2 } ] }, "judge_disagreement": 0 }, { "idx": 76, "duration_s": 58, "mean_score": 68.0, "mean_raw": 32.0, "scores_per_judge": [ 68 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "semantic-inversion", "points": 20 }, { "category": "proper-noun", "points": 6 }, { "category": "mishear-minor", "points": 2 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 77, "duration_s": 130, "mean_score": 65.0, "mean_raw": 35.0, "scores_per_judge": [ 65 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "semantic-inversion", "points": 20 }, { "category": "brand", "points": 10 }, { "category": "filler", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 78, "duration_s": 61, "mean_score": 96.0, "mean_raw": 4.0, "scores_per_judge": [ 96 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 79, "duration_s": 47, "mean_score": 97.0, "mean_raw": 3.0, "scores_per_judge": [ 97 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 80, "duration_s": 42, "mean_score": 98.0, "mean_raw": 2.0, "scores_per_judge": [ 98 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 81, "duration_s": 38, "mean_score": 91.0, "mean_raw": 9.0, "scores_per_judge": [ 91 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 }, { "category": "mishear-minor", "points": 2 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 82, "duration_s": 38, "mean_score": 85.0, "mean_raw": 15.0, "scores_per_judge": [ 85 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "tech", "points": 8 }, { "category": "mishear-severe", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 83, "duration_s": 37, "mean_score": 93.0, "mean_raw": 7.0, "scores_per_judge": [ 93 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-minor", "points": 2 }, { "category": "filler", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 84, "duration_s": 36, "mean_score": 95.0, "mean_raw": 5.0, "scores_per_judge": [ 95 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 85, "duration_s": 36, "mean_score": 89.0, "mean_raw": 11.0, "scores_per_judge": [ 89 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 }, { "category": "mishear-minor", "points": 2 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 86, "duration_s": 35, "mean_score": 84.0, "mean_raw": 16.0, "scores_per_judge": [ 84 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "tech", "points": 8 }, { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 3 } ] }, "judge_disagreement": 0 }, { "idx": 87, "duration_s": 34, "mean_score": 95.0, "mean_raw": 5.0, "scores_per_judge": [ 95 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 88, "duration_s": 34, "mean_score": 100.0, "mean_raw": 0.0, "scores_per_judge": [ 100 ], "deductions_per_judge": { "deepseek_flash": [] }, "judge_disagreement": 0 }, { "idx": 89, "duration_s": 34, "mean_score": 100.0, "mean_raw": 0.0, "scores_per_judge": [ 100 ], "deductions_per_judge": { "deepseek_flash": [] }, "judge_disagreement": 0 }, { "idx": 90, "duration_s": 33, "mean_score": 98.0, "mean_raw": 2.0, "scores_per_judge": [ 98 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 91, "duration_s": 33, "mean_score": 97.0, "mean_raw": 3.0, "scores_per_judge": [ 97 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 92, "duration_s": 32, "mean_score": 100.0, "mean_raw": 0.0, "scores_per_judge": [ 100 ], "deductions_per_judge": { "deepseek_flash": [] }, "judge_disagreement": 0 }, { "idx": 93, "duration_s": 32, "mean_score": 95.0, "mean_raw": 5.0, "scores_per_judge": [ 95 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "content-drop", "points": 3 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 94, "duration_s": 32, "mean_score": 97.0, "mean_raw": 3.0, "scores_per_judge": [ 97 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 95, "duration_s": 32, "mean_score": 95.0, "mean_raw": 5.0, "scores_per_judge": [ 95 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "paraphrase", "points": 3 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 96, "duration_s": 31, "mean_score": 99.0, "mean_raw": 1.0, "scores_per_judge": [ 99 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 97, "duration_s": 30, "mean_score": 97.0, "mean_raw": 3.0, "scores_per_judge": [ 97 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 98, "duration_s": 30, "mean_score": 99.0, "mean_raw": 1.0, "scores_per_judge": [ 99 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 99, "duration_s": 30, "mean_score": 100.0, "mean_raw": 0.0, "scores_per_judge": [ 100 ], "deductions_per_judge": { "deepseek_flash": [] }, "judge_disagreement": 0 }, { "idx": 100, "duration_s": 30, "mean_score": 97.0, "mean_raw": 3.0, "scores_per_judge": [ 97 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 } ], "total_deductions": 863, "avg_deductions_per_sample": 8.63, "mean_clamped_score": 91.4, "n_judge_sample_pairs": 100, "gt_chars": 22625, "ded_per_100char": 3.81, "fidelity_score": 80.9, "overall_score": 91.4, "overall_raw_deductions": 8.63, "score_per_judge": { "deepseek_flash": 91.4 }, "per_group_deductions": { "meaning-flip": { "deepseek_flash": 130 }, "entity": { "deepseek_flash": 194 }, "cultural": { "deepseek_flash": 55 }, "fidelity": { "deepseek_flash": 484 } }, "per_bucket": { "general": { "n_samples": 40, "mean_raw": 8.7, "mean_score": 91.3, "total_raw": 347.0, "idxs": [ 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40 ], "worst_idx": 39 }, "idiom-test": { "n_samples": 5, "mean_raw": 18.2, "mean_score": 81.8, "total_raw": 91.0, "idxs": [ 41, 42, 43, 44, 45 ], "worst_idx": 45 }, "long-mixed": { "n_samples": 5, "mean_raw": 9.8, "mean_score": 90.2, "total_raw": 49.0, "idxs": [ 46, 47, 48, 49, 50 ], "worst_idx": 46 } }, "per_group_per_100char": { "meaning-flip": 0.575, "entity": 0.857, "cultural": 0.243, "fidelity": 2.139 }, "per_group_fidelity": { "meaning-flip": 86.2, "entity": 84.8, "cultural": 85.3, "fidelity": 74.9 } }, "mai_transcribe_2": { "scores_by_judge": { "deepseek_flash": [ 100, 99, 100, 100, 98, 99, 99, 100, 94, 100, 96, 97, 77, 95, 98, 94, 88, 90, 70, 100, 100, 100, 100, 92, 100, 100, 100, 91, 92, 99, 100, 99, 94, 97, 97, 100, 100, 95, 86, 80, 100, 100, 100, 85, 93, 90, 90, 100, 96, 80, 97, 97, 100, 99, 95, 100, 97, 100, 97, 100, 100, 100, 100, 97, 99, 99, 99, 100, 100, 99, 97, 99, 85, 97, 98, 78, 70, 98, 97, 97, 97, 99, 98, 64, 96, 87, 98, 100, 100, 98, 98, 99, 100, 99, 99, 100, 96, 99, 100, 98 ] }, "per_category_deductions": { "polarity": { "deepseek_flash": 35 }, "semantic-inversion": { "deepseek_flash": 80 }, "hallucination": { "deepseek_flash": 15 }, "numeric": { "deepseek_flash": 0 }, "brand": { "deepseek_flash": 70 }, "tech": { "deepseek_flash": 32 }, "culture": { "deepseek_flash": 8 }, "proper-noun": { "deepseek_flash": 24 }, "dialect": { "deepseek_flash": 0 }, "mishear-severe": { "deepseek_flash": 60 }, "content-drop": { "deepseek_flash": 0 }, "paraphrase": { "deepseek_flash": 0 }, "mishear-minor": { "deepseek_flash": 12 }, "filler": { "deepseek_flash": 98 }, "format": { "deepseek_flash": 0 }, "unscoreable": { "deepseek_flash": 0 } }, "per_tier_deductions": { "tier1": { "deepseek_flash": 130 }, "tier2": { "deepseek_flash": 134 }, "tier3": { "deepseek_flash": 72 }, "tier4": { "deepseek_flash": 98 }, "unscoreable": { "deepseek_flash": 0 } }, "raw_deductions_by_judge": { "deepseek_flash": [ 0, 1, 0, 0, 2, 1, 1, 0, 6, 0, 4, 3, 23, 5, 2, 6, 12, 10, 30, 0, 0, 0, 0, 8, 0, 0, 0, 9, 8, 1, 0, 1, 6, 3, 3, 0, 0, 5, 14, 20, 0, 0, 0, 15, 7, 10, 10, 0, 4, 20, 3, 3, 0, 1, 5, 0, 3, 0, 3, 0, 0, 0, 0, 3, 1, 1, 1, 0, 0, 1, 3, 1, 15, 3, 2, 22, 30, 2, 3, 3, 3, 1, 2, 36, 4, 13, 2, 0, 0, 2, 2, 1, 0, 1, 1, 0, 4, 1, 0, 2 ] }, "sample_details": [ { "idx": 1, "duration_s": 5, "mean_score": 100.0, "mean_raw": 0.0, "scores_per_judge": [ 100 ], "deductions_per_judge": { "deepseek_flash": [] }, "judge_disagreement": 0 }, { "idx": 2, "duration_s": 7, "mean_score": 99.0, "mean_raw": 1.0, "scores_per_judge": [ 99 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 3, "duration_s": 8, "mean_score": 100.0, "mean_raw": 0.0, "scores_per_judge": [ 100 ], "deductions_per_judge": { "deepseek_flash": [] }, "judge_disagreement": 0 }, { "idx": 4, "duration_s": 11, "mean_score": 100.0, "mean_raw": 0.0, "scores_per_judge": [ 100 ], "deductions_per_judge": { "deepseek_flash": [] }, "judge_disagreement": 0 }, { "idx": 5, "duration_s": 16, "mean_score": 98.0, "mean_raw": 2.0, "scores_per_judge": [ 98 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 6, "duration_s": 17, "mean_score": 99.0, "mean_raw": 1.0, "scores_per_judge": [ 99 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 7, "duration_s": 20, "mean_score": 99.0, "mean_raw": 1.0, "scores_per_judge": [ 99 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 8, "duration_s": 30, "mean_score": 100.0, "mean_raw": 0.0, "scores_per_judge": [ 100 ], "deductions_per_judge": { "deepseek_flash": [] }, "judge_disagreement": 0 }, { "idx": 9, "duration_s": 32, "mean_score": 94.0, "mean_raw": 6.0, "scores_per_judge": [ 94 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 10, "duration_s": 35, "mean_score": 100.0, "mean_raw": 0.0, "scores_per_judge": [ 100 ], "deductions_per_judge": { "deepseek_flash": [] }, "judge_disagreement": 0 }, { "idx": 11, "duration_s": 40, "mean_score": 96.0, "mean_raw": 4.0, "scores_per_judge": [ 96 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 12, "duration_s": 46, "mean_score": 97.0, "mean_raw": 3.0, "scores_per_judge": [ 97 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 3 } ] }, "judge_disagreement": 0 }, { "idx": 13, "duration_s": 60, "mean_score": 77.0, "mean_raw": 23.0, "scores_per_judge": [ 77 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "brand", "points": 10 }, { "category": "brand", "points": 10 }, { "category": "mishear-minor", "points": 2 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 14, "duration_s": 64, "mean_score": 95.0, "mean_raw": 5.0, "scores_per_judge": [ 95 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 15, "duration_s": 69, "mean_score": 98.0, "mean_raw": 2.0, "scores_per_judge": [ 98 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 16, "duration_s": 81, "mean_score": 94.0, "mean_raw": 6.0, "scores_per_judge": [ 94 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "proper-noun", "points": 6 } ] }, "judge_disagreement": 0 }, { "idx": 17, "duration_s": 100, "mean_score": 88.0, "mean_raw": 12.0, "scores_per_judge": [ 88 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "brand", "points": 10 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 18, "duration_s": 125, "mean_score": 90.0, "mean_raw": 10.0, "scores_per_judge": [ 90 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "brand", "points": 10 } ] }, "judge_disagreement": 0 }, { "idx": 19, "duration_s": 133, "mean_score": 70.0, "mean_raw": 30.0, "scores_per_judge": [ 70 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "brand", "points": 10 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 20, "duration_s": 164, "mean_score": 100.0, "mean_raw": 0.0, "scores_per_judge": [ 100 ], "deductions_per_judge": { "deepseek_flash": [] }, "judge_disagreement": 0 }, { "idx": 21, "duration_s": 5, "mean_score": 100.0, "mean_raw": 0.0, "scores_per_judge": [ 100 ], "deductions_per_judge": { "deepseek_flash": [] }, "judge_disagreement": 0 }, { "idx": 22, "duration_s": 7, "mean_score": 100.0, "mean_raw": 0.0, "scores_per_judge": [ 100 ], "deductions_per_judge": { "deepseek_flash": [] }, "judge_disagreement": 0 }, { "idx": 23, "duration_s": 10, "mean_score": 100.0, "mean_raw": 0.0, "scores_per_judge": [ 100 ], "deductions_per_judge": { "deepseek_flash": [] }, "judge_disagreement": 0 }, { "idx": 24, "duration_s": 14, "mean_score": 92.0, "mean_raw": 8.0, "scores_per_judge": [ 92 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "tech", "points": 8 } ] }, "judge_disagreement": 0 }, { "idx": 25, "duration_s": 16, "mean_score": 100.0, "mean_raw": 0.0, "scores_per_judge": [ 100 ], "deductions_per_judge": { "deepseek_flash": [] }, "judge_disagreement": 0 }, { "idx": 26, "duration_s": 23, "mean_score": 100.0, "mean_raw": 0.0, "scores_per_judge": [ 100 ], "deductions_per_judge": { "deepseek_flash": [] }, "judge_disagreement": 0 }, { "idx": 27, "duration_s": 25, "mean_score": 100.0, "mean_raw": 0.0, "scores_per_judge": [ 100 ], "deductions_per_judge": { "deepseek_flash": [] }, "judge_disagreement": 0 }, { "idx": 28, "duration_s": 30, "mean_score": 91.0, "mean_raw": 9.0, "scores_per_judge": [ 91 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 }, { "category": "mishear-minor", "points": 2 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 29, "duration_s": 34, "mean_score": 92.0, "mean_raw": 8.0, "scores_per_judge": [ 92 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "tech", "points": 8 } ] }, "judge_disagreement": 0 }, { "idx": 30, "duration_s": 38, "mean_score": 99.0, "mean_raw": 1.0, "scores_per_judge": [ 99 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 31, "duration_s": 44, "mean_score": 100.0, "mean_raw": 0.0, "scores_per_judge": [ 100 ], "deductions_per_judge": { "deepseek_flash": [] }, "judge_disagreement": 0 }, { "idx": 32, "duration_s": 50, "mean_score": 99.0, "mean_raw": 1.0, "scores_per_judge": [ 99 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 33, "duration_s": 62, "mean_score": 94.0, "mean_raw": 6.0, "scores_per_judge": [ 94 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "proper-noun", "points": 6 } ] }, "judge_disagreement": 0 }, { "idx": 34, "duration_s": 66, "mean_score": 97.0, "mean_raw": 3.0, "scores_per_judge": [ 97 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 35, "duration_s": 77, "mean_score": 97.0, "mean_raw": 3.0, "scores_per_judge": [ 97 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 3 } ] }, "judge_disagreement": 0 }, { "idx": 36, "duration_s": 94, "mean_score": 100.0, "mean_raw": 0.0, "scores_per_judge": [ 100 ], "deductions_per_judge": { "deepseek_flash": [] }, "judge_disagreement": 0 }, { "idx": 37, "duration_s": 106, "mean_score": 100.0, "mean_raw": 0.0, "scores_per_judge": [ 100 ], "deductions_per_judge": { "deepseek_flash": [] }, "judge_disagreement": 0 }, { "idx": 38, "duration_s": 126, "mean_score": 95.0, "mean_raw": 5.0, "scores_per_judge": [ 95 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 39, "duration_s": 158, "mean_score": 86.0, "mean_raw": 14.0, "scores_per_judge": [ 86 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "proper-noun", "points": 6 }, { "category": "tech", "points": 8 } ] }, "judge_disagreement": 0 }, { "idx": 40, "duration_s": 279, "mean_score": 80.0, "mean_raw": 20.0, "scores_per_judge": [ 80 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "semantic-inversion", "points": 20 } ] }, "judge_disagreement": 0 }, { "idx": 41, "duration_s": 34, "mean_score": 100.0, "mean_raw": 0.0, "scores_per_judge": [ 100 ], "deductions_per_judge": { "deepseek_flash": [] }, "judge_disagreement": 0 }, { "idx": 42, "duration_s": 43, "mean_score": 100.0, "mean_raw": 0.0, "scores_per_judge": [ 100 ], "deductions_per_judge": { "deepseek_flash": [] }, "judge_disagreement": 0 }, { "idx": 43, "duration_s": 29, "mean_score": 100.0, "mean_raw": 0.0, "scores_per_judge": [ 100 ], "deductions_per_judge": { "deepseek_flash": [] }, "judge_disagreement": 0 }, { "idx": 44, "duration_s": 71, "mean_score": 85.0, "mean_raw": 15.0, "scores_per_judge": [ 85 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "proper-noun", "points": 6 }, { "category": "culture", "points": 8 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 45, "duration_s": 46, "mean_score": 93.0, "mean_raw": 7.0, "scores_per_judge": [ 93 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 }, { "category": "mishear-minor", "points": 2 } ] }, "judge_disagreement": 0 }, { "idx": 46, "duration_s": 62, "mean_score": 90.0, "mean_raw": 10.0, "scores_per_judge": [ 90 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 }, { "category": "mishear-severe", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 47, "duration_s": 67, "mean_score": 90.0, "mean_raw": 10.0, "scores_per_judge": [ 90 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "brand", "points": 10 } ] }, "judge_disagreement": 0 }, { "idx": 48, "duration_s": 72, "mean_score": 100.0, "mean_raw": 0.0, "scores_per_judge": [ 100 ], "deductions_per_judge": { "deepseek_flash": [] }, "judge_disagreement": 0 }, { "idx": 49, "duration_s": 87, "mean_score": 96.0, "mean_raw": 4.0, "scores_per_judge": [ 96 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 4 } ] }, "judge_disagreement": 0 }, { "idx": 50, "duration_s": 106, "mean_score": 80.0, "mean_raw": 20.0, "scores_per_judge": [ 80 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "semantic-inversion", "points": 20 } ] }, "judge_disagreement": 0 }, { "idx": 51, "duration_s": 33, "mean_score": 97.0, "mean_raw": 3.0, "scores_per_judge": [ 97 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 52, "duration_s": 25, "mean_score": 97.0, "mean_raw": 3.0, "scores_per_judge": [ 97 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 53, "duration_s": 35, "mean_score": 100.0, "mean_raw": 0.0, "scores_per_judge": [ 100 ], "deductions_per_judge": { "deepseek_flash": [] }, "judge_disagreement": 0 }, { "idx": 54, "duration_s": 36, "mean_score": 99.0, "mean_raw": 1.0, "scores_per_judge": [ 99 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 55, "duration_s": 33, "mean_score": 95.0, "mean_raw": 5.0, "scores_per_judge": [ 95 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 56, "duration_s": 15, "mean_score": 100.0, "mean_raw": 0.0, "scores_per_judge": [ 100 ], "deductions_per_judge": { "deepseek_flash": [] }, "judge_disagreement": 0 }, { "idx": 57, "duration_s": 38, "mean_score": 97.0, "mean_raw": 3.0, "scores_per_judge": [ 97 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 58, "duration_s": 15, "mean_score": 100.0, "mean_raw": 0.0, "scores_per_judge": [ 100 ], "deductions_per_judge": { "deepseek_flash": [] }, "judge_disagreement": 0 }, { "idx": 59, "duration_s": 28, "mean_score": 97.0, "mean_raw": 3.0, "scores_per_judge": [ 97 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-minor", "points": 2 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 60, "duration_s": 18, "mean_score": 100.0, "mean_raw": 0.0, "scores_per_judge": [ 100 ], "deductions_per_judge": { "deepseek_flash": [] }, "judge_disagreement": 0 }, { "idx": 61, "duration_s": 19, "mean_score": 100.0, "mean_raw": 0.0, "scores_per_judge": [ 100 ], "deductions_per_judge": { "deepseek_flash": [] }, "judge_disagreement": 0 }, { "idx": 62, "duration_s": 17, "mean_score": 100.0, "mean_raw": 0.0, "scores_per_judge": [ 100 ], "deductions_per_judge": { "deepseek_flash": [] }, "judge_disagreement": 0 }, { "idx": 63, "duration_s": 21, "mean_score": 100.0, "mean_raw": 0.0, "scores_per_judge": [ 100 ], "deductions_per_judge": { "deepseek_flash": [] }, "judge_disagreement": 0 }, { "idx": 64, "duration_s": 23, "mean_score": 97.0, "mean_raw": 3.0, "scores_per_judge": [ 97 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 65, "duration_s": 20, "mean_score": 99.0, "mean_raw": 1.0, "scores_per_judge": [ 99 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 66, "duration_s": 24, "mean_score": 99.0, "mean_raw": 1.0, "scores_per_judge": [ 99 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 67, "duration_s": 16, "mean_score": 99.0, "mean_raw": 1.0, "scores_per_judge": [ 99 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 68, "duration_s": 18, "mean_score": 100.0, "mean_raw": 0.0, "scores_per_judge": [ 100 ], "deductions_per_judge": { "deepseek_flash": [] }, "judge_disagreement": 0 }, { "idx": 69, "duration_s": 18, "mean_score": 100.0, "mean_raw": 0.0, "scores_per_judge": [ 100 ], "deductions_per_judge": { "deepseek_flash": [] }, "judge_disagreement": 0 }, { "idx": 70, "duration_s": 17, "mean_score": 99.0, "mean_raw": 1.0, "scores_per_judge": [ 99 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 71, "duration_s": 51, "mean_score": 97.0, "mean_raw": 3.0, "scores_per_judge": [ 97 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 72, "duration_s": 54, "mean_score": 99.0, "mean_raw": 1.0, "scores_per_judge": [ 99 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 73, "duration_s": 56, "mean_score": 85.0, "mean_raw": 15.0, "scores_per_judge": [ 85 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "hallucination", "points": 15 } ] }, "judge_disagreement": 0 }, { "idx": 74, "duration_s": 45, "mean_score": 97.0, "mean_raw": 3.0, "scores_per_judge": [ 97 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 75, "duration_s": 74, "mean_score": 98.0, "mean_raw": 2.0, "scores_per_judge": [ 98 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 76, "duration_s": 58, "mean_score": 78.0, "mean_raw": 22.0, "scores_per_judge": [ 78 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "semantic-inversion", "points": 20 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 77, "duration_s": 130, "mean_score": 70.0, "mean_raw": 30.0, "scores_per_judge": [ 70 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "semantic-inversion", "points": 20 }, { "category": "brand", "points": 10 } ] }, "judge_disagreement": 0 }, { "idx": 78, "duration_s": 61, "mean_score": 98.0, "mean_raw": 2.0, "scores_per_judge": [ 98 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-minor", "points": 2 } ] }, "judge_disagreement": 0 }, { "idx": 79, "duration_s": 47, "mean_score": 97.0, "mean_raw": 3.0, "scores_per_judge": [ 97 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 80, "duration_s": 42, "mean_score": 97.0, "mean_raw": 3.0, "scores_per_judge": [ 97 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 81, "duration_s": 38, "mean_score": 97.0, "mean_raw": 3.0, "scores_per_judge": [ 97 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 82, "duration_s": 38, "mean_score": 99.0, "mean_raw": 1.0, "scores_per_judge": [ 99 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 83, "duration_s": 37, "mean_score": 98.0, "mean_raw": 2.0, "scores_per_judge": [ 98 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 84, "duration_s": 36, "mean_score": 64.0, "mean_raw": 36.0, "scores_per_judge": [ 64 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "polarity", "points": 35 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 85, "duration_s": 36, "mean_score": 96.0, "mean_raw": 4.0, "scores_per_judge": [ 96 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-minor", "points": 2 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 86, "duration_s": 35, "mean_score": 87.0, "mean_raw": 13.0, "scores_per_judge": [ 87 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "tech", "points": 8 }, { "category": "mishear-severe", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 87, "duration_s": 34, "mean_score": 98.0, "mean_raw": 2.0, "scores_per_judge": [ 98 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 88, "duration_s": 34, "mean_score": 100.0, "mean_raw": 0.0, "scores_per_judge": [ 100 ], "deductions_per_judge": { "deepseek_flash": [] }, "judge_disagreement": 0 }, { "idx": 89, "duration_s": 34, "mean_score": 100.0, "mean_raw": 0.0, "scores_per_judge": [ 100 ], "deductions_per_judge": { "deepseek_flash": [] }, "judge_disagreement": 0 }, { "idx": 90, "duration_s": 33, "mean_score": 98.0, "mean_raw": 2.0, "scores_per_judge": [ 98 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 2 } ] }, "judge_disagreement": 0 }, { "idx": 91, "duration_s": 33, "mean_score": 98.0, "mean_raw": 2.0, "scores_per_judge": [ 98 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 92, "duration_s": 32, "mean_score": 99.0, "mean_raw": 1.0, "scores_per_judge": [ 99 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 93, "duration_s": 32, "mean_score": 100.0, "mean_raw": 0.0, "scores_per_judge": [ 100 ], "deductions_per_judge": { "deepseek_flash": [] }, "judge_disagreement": 0 }, { "idx": 94, "duration_s": 32, "mean_score": 99.0, "mean_raw": 1.0, "scores_per_judge": [ 99 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 95, "duration_s": 32, "mean_score": 99.0, "mean_raw": 1.0, "scores_per_judge": [ 99 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 96, "duration_s": 31, "mean_score": 100.0, "mean_raw": 0.0, "scores_per_judge": [ 100 ], "deductions_per_judge": { "deepseek_flash": [] }, "judge_disagreement": 0 }, { "idx": 97, "duration_s": 30, "mean_score": 96.0, "mean_raw": 4.0, "scores_per_judge": [ 96 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 98, "duration_s": 30, "mean_score": 99.0, "mean_raw": 1.0, "scores_per_judge": [ 99 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 99, "duration_s": 30, "mean_score": 100.0, "mean_raw": 0.0, "scores_per_judge": [ 100 ], "deductions_per_judge": { "deepseek_flash": [] }, "judge_disagreement": 0 }, { "idx": 100, "duration_s": 30, "mean_score": 98.0, "mean_raw": 2.0, "scores_per_judge": [ 98 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 } ], "total_deductions": 434, "avg_deductions_per_sample": 4.34, "mean_clamped_score": 95.7, "n_judge_sample_pairs": 100, "gt_chars": 22625, "ded_per_100char": 1.92, "fidelity_score": 90.4, "overall_score": 95.7, "overall_raw_deductions": 4.34, "score_per_judge": { "deepseek_flash": 95.7 }, "per_group_deductions": { "meaning-flip": { "deepseek_flash": 130 }, "entity": { "deepseek_flash": 126 }, "cultural": { "deepseek_flash": 8 }, "fidelity": { "deepseek_flash": 170 } }, "per_bucket": { "general": { "n_samples": 40, "mean_raw": 4.6, "mean_score": 95.4, "total_raw": 184.0, "idxs": [ 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40 ], "worst_idx": 19 }, "idiom-test": { "n_samples": 5, "mean_raw": 4.4, "mean_score": 95.6, "total_raw": 22.0, "idxs": [ 41, 42, 43, 44, 45 ], "worst_idx": 44 }, "long-mixed": { "n_samples": 5, "mean_raw": 8.8, "mean_score": 91.2, "total_raw": 44.0, "idxs": [ 46, 47, 48, 49, 50 ], "worst_idx": 50 } }, "per_group_per_100char": { "meaning-flip": 0.575, "entity": 0.557, "cultural": 0.035, "fidelity": 0.751 }, "per_group_fidelity": { "meaning-flip": 86.2, "entity": 90.1, "cultural": 97.9, "fidelity": 91.2 } }, "mimo": { "scores_by_judge": { "deepseek_flash": [ 100, 100, 100, 100, 98, 99, 99, 100, 98, 99, 86, 100, 70, 84, 97, 93, 95, 87, 59, 91, 99, 94, 100, 91, 92, 100, 100, 97, 99, 100, 97, 98, 83, 98, 95, 89, 98, 78, 95, 77, 100, 94, 91, 100, 93, 97, 89, 81, 95, 91, 97, 99, 100, 99, 95, 89, 99, 98, 97, 63, 85, 100, 100, 96, 100, 96, 100, 100, 100, 99, 95, 99, 87, 98, 96, 94, 65, 96, 97, 97, 97, 90, 97, 97, 96, 92, 94, 98, 100, 97, 99, 100, 93, 93, 97, 99, 98, 99, 100, 98 ] }, "per_category_deductions": { "polarity": { "deepseek_flash": 35 }, "semantic-inversion": { "deepseek_flash": 20 }, "hallucination": { "deepseek_flash": 0 }, "numeric": { "deepseek_flash": 10 }, "brand": { "deepseek_flash": 100 }, "tech": { "deepseek_flash": 88 }, "culture": { "deepseek_flash": 24 }, "proper-noun": { "deepseek_flash": 24 }, "dialect": { "deepseek_flash": 0 }, "mishear-severe": { "deepseek_flash": 65 }, "content-drop": { "deepseek_flash": 3 }, "paraphrase": { "deepseek_flash": 18 }, "mishear-minor": { "deepseek_flash": 12 }, "filler": { "deepseek_flash": 165 }, "format": { "deepseek_flash": 0 }, "unscoreable": { "deepseek_flash": 0 } }, "per_tier_deductions": { "tier1": { "deepseek_flash": 65 }, "tier2": { "deepseek_flash": 236 }, "tier3": { "deepseek_flash": 98 }, "tier4": { "deepseek_flash": 165 }, "unscoreable": { "deepseek_flash": 0 } }, "raw_deductions_by_judge": { "deepseek_flash": [ 0, 0, 0, 0, 2, 1, 1, 0, 2, 1, 14, 0, 30, 16, 3, 7, 5, 13, 41, 9, 1, 6, 0, 9, 8, 0, 0, 3, 1, 0, 3, 2, 17, 2, 5, 11, 2, 22, 5, 23, 0, 6, 9, 0, 7, 3, 11, 19, 5, 9, 3, 1, 0, 1, 5, 11, 1, 2, 3, 37, 15, 0, 0, 4, 0, 4, 0, 0, 0, 1, 5, 1, 13, 2, 4, 6, 35, 4, 3, 3, 3, 10, 3, 3, 4, 8, 6, 2, 0, 3, 1, 0, 7, 7, 3, 1, 2, 1, 0, 2 ] }, "sample_details": [ { "idx": 1, "duration_s": 5, "mean_score": 100.0, "mean_raw": 0.0, "scores_per_judge": [ 100 ], "deductions_per_judge": { "deepseek_flash": [] }, "judge_disagreement": 0 }, { "idx": 2, "duration_s": 7, "mean_score": 100.0, "mean_raw": 0.0, "scores_per_judge": [ 100 ], "deductions_per_judge": { "deepseek_flash": [] }, "judge_disagreement": 0 }, { "idx": 3, "duration_s": 8, "mean_score": 100.0, "mean_raw": 0.0, "scores_per_judge": [ 100 ], "deductions_per_judge": { "deepseek_flash": [] }, "judge_disagreement": 0 }, { "idx": 4, "duration_s": 11, "mean_score": 100.0, "mean_raw": 0.0, "scores_per_judge": [ 100 ], "deductions_per_judge": { "deepseek_flash": [] }, "judge_disagreement": 0 }, { "idx": 5, "duration_s": 16, "mean_score": 98.0, "mean_raw": 2.0, "scores_per_judge": [ 98 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 6, "duration_s": 17, "mean_score": 99.0, "mean_raw": 1.0, "scores_per_judge": [ 99 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 7, "duration_s": 20, "mean_score": 99.0, "mean_raw": 1.0, "scores_per_judge": [ 99 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 8, "duration_s": 30, "mean_score": 100.0, "mean_raw": 0.0, "scores_per_judge": [ 100 ], "deductions_per_judge": { "deepseek_flash": [] }, "judge_disagreement": 0 }, { "idx": 9, "duration_s": 32, "mean_score": 98.0, "mean_raw": 2.0, "scores_per_judge": [ 98 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 10, "duration_s": 35, "mean_score": 99.0, "mean_raw": 1.0, "scores_per_judge": [ 99 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 11, "duration_s": 40, "mean_score": 86.0, "mean_raw": 14.0, "scores_per_judge": [ 86 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "tech", "points": 8 }, { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 12, "duration_s": 46, "mean_score": 100.0, "mean_raw": 0.0, "scores_per_judge": [ 100 ], "deductions_per_judge": { "deepseek_flash": [] }, "judge_disagreement": 0 }, { "idx": 13, "duration_s": 60, "mean_score": 70.0, "mean_raw": 30.0, "scores_per_judge": [ 70 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "brand", "points": 10 }, { "category": "brand", "points": 10 }, { "category": "tech", "points": 8 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 14, "duration_s": 64, "mean_score": 84.0, "mean_raw": 16.0, "scores_per_judge": [ 84 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "proper-noun", "points": 6 }, { "category": "content-drop", "points": 3 }, { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 15, "duration_s": 69, "mean_score": 97.0, "mean_raw": 3.0, "scores_per_judge": [ 97 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 16, "duration_s": 81, "mean_score": 93.0, "mean_raw": 7.0, "scores_per_judge": [ 93 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "proper-noun", "points": 6 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 17, "duration_s": 100, "mean_score": 95.0, "mean_raw": 5.0, "scores_per_judge": [ 95 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 18, "duration_s": 125, "mean_score": 87.0, "mean_raw": 13.0, "scores_per_judge": [ 87 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "brand", "points": 10 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 19, "duration_s": 133, "mean_score": 59.0, "mean_raw": 41.0, "scores_per_judge": [ 59 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "tech", "points": 8 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-severe", "points": 5 }, { "category": "brand", "points": 10 }, { "category": "tech", "points": 8 }, { "category": "filler", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 20, "duration_s": 164, "mean_score": 91.0, "mean_raw": 9.0, "scores_per_judge": [ 91 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "mishear-minor", "points": 2 }, { "category": "mishear-minor", "points": 2 } ] }, "judge_disagreement": 0 }, { "idx": 21, "duration_s": 5, "mean_score": 99.0, "mean_raw": 1.0, "scores_per_judge": [ 99 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 22, "duration_s": 7, "mean_score": 94.0, "mean_raw": 6.0, "scores_per_judge": [ 94 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "proper-noun", "points": 6 } ] }, "judge_disagreement": 0 }, { "idx": 23, "duration_s": 10, "mean_score": 100.0, "mean_raw": 0.0, "scores_per_judge": [ 100 ], "deductions_per_judge": { "deepseek_flash": [] }, "judge_disagreement": 0 }, { "idx": 24, "duration_s": 14, "mean_score": 91.0, "mean_raw": 9.0, "scores_per_judge": [ 91 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "tech", "points": 8 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 25, "duration_s": 16, "mean_score": 92.0, "mean_raw": 8.0, "scores_per_judge": [ 92 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "tech", "points": 8 } ] }, "judge_disagreement": 0 }, { "idx": 26, "duration_s": 23, "mean_score": 100.0, "mean_raw": 0.0, "scores_per_judge": [ 100 ], "deductions_per_judge": { "deepseek_flash": [] }, "judge_disagreement": 0 }, { "idx": 27, "duration_s": 25, "mean_score": 100.0, "mean_raw": 0.0, "scores_per_judge": [ 100 ], "deductions_per_judge": { "deepseek_flash": [] }, "judge_disagreement": 0 }, { "idx": 28, "duration_s": 30, "mean_score": 97.0, "mean_raw": 3.0, "scores_per_judge": [ 97 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 29, "duration_s": 34, "mean_score": 99.0, "mean_raw": 1.0, "scores_per_judge": [ 99 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 30, "duration_s": 38, "mean_score": 100.0, "mean_raw": 0.0, "scores_per_judge": [ 100 ], "deductions_per_judge": { "deepseek_flash": [] }, "judge_disagreement": 0 }, { "idx": 31, "duration_s": 44, "mean_score": 97.0, "mean_raw": 3.0, "scores_per_judge": [ 97 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-minor", "points": 2 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 32, "duration_s": 50, "mean_score": 98.0, "mean_raw": 2.0, "scores_per_judge": [ 98 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 33, "duration_s": 62, "mean_score": 83.0, "mean_raw": 17.0, "scores_per_judge": [ 83 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "brand", "points": 10 }, { "category": "proper-noun", "points": 6 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 34, "duration_s": 66, "mean_score": 98.0, "mean_raw": 2.0, "scores_per_judge": [ 98 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 35, "duration_s": 77, "mean_score": 95.0, "mean_raw": 5.0, "scores_per_judge": [ 95 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 36, "duration_s": 94, "mean_score": 89.0, "mean_raw": 11.0, "scores_per_judge": [ 89 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 }, { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 37, "duration_s": 106, "mean_score": 98.0, "mean_raw": 2.0, "scores_per_judge": [ 98 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 38, "duration_s": 126, "mean_score": 78.0, "mean_raw": 22.0, "scores_per_judge": [ 78 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "tech", "points": 8 }, { "category": "tech", "points": 8 }, { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 39, "duration_s": 158, "mean_score": 95.0, "mean_raw": 5.0, "scores_per_judge": [ 95 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 40, "duration_s": 279, "mean_score": 77.0, "mean_raw": 23.0, "scores_per_judge": [ 77 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "numeric", "points": 10 }, { "category": "culture", "points": 8 }, { "category": "paraphrase", "points": 3 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 41, "duration_s": 34, "mean_score": 100.0, "mean_raw": 0.0, "scores_per_judge": [ 100 ], "deductions_per_judge": { "deepseek_flash": [] }, "judge_disagreement": 0 }, { "idx": 42, "duration_s": 43, "mean_score": 94.0, "mean_raw": 6.0, "scores_per_judge": [ 94 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 43, "duration_s": 29, "mean_score": 91.0, "mean_raw": 9.0, "scores_per_judge": [ 91 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "culture", "points": 8 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 44, "duration_s": 71, "mean_score": 100.0, "mean_raw": 0.0, "scores_per_judge": [ 100 ], "deductions_per_judge": { "deepseek_flash": [] }, "judge_disagreement": 0 }, { "idx": 45, "duration_s": 46, "mean_score": 93.0, "mean_raw": 7.0, "scores_per_judge": [ 93 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "paraphrase", "points": 3 }, { "category": "paraphrase", "points": 3 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 46, "duration_s": 62, "mean_score": 97.0, "mean_raw": 3.0, "scores_per_judge": [ 97 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 47, "duration_s": 67, "mean_score": 89.0, "mean_raw": 11.0, "scores_per_judge": [ 89 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "brand", "points": 10 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 48, "duration_s": 72, "mean_score": 81.0, "mean_raw": 19.0, "scores_per_judge": [ 81 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "tech", "points": 8 }, { "category": "brand", "points": 10 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 49, "duration_s": 87, "mean_score": 95.0, "mean_raw": 5.0, "scores_per_judge": [ 95 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 50, "duration_s": 106, "mean_score": 91.0, "mean_raw": 9.0, "scores_per_judge": [ 91 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "culture", "points": 8 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 51, "duration_s": 33, "mean_score": 97.0, "mean_raw": 3.0, "scores_per_judge": [ 97 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 52, "duration_s": 25, "mean_score": 99.0, "mean_raw": 1.0, "scores_per_judge": [ 99 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 53, "duration_s": 35, "mean_score": 100.0, "mean_raw": 0.0, "scores_per_judge": [ 100 ], "deductions_per_judge": { "deepseek_flash": [] }, "judge_disagreement": 0 }, { "idx": 54, "duration_s": 36, "mean_score": 99.0, "mean_raw": 1.0, "scores_per_judge": [ 99 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 55, "duration_s": 33, "mean_score": 95.0, "mean_raw": 5.0, "scores_per_judge": [ 95 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 56, "duration_s": 15, "mean_score": 89.0, "mean_raw": 11.0, "scores_per_judge": [ 89 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "brand", "points": 10 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 57, "duration_s": 38, "mean_score": 99.0, "mean_raw": 1.0, "scores_per_judge": [ 99 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 58, "duration_s": 15, "mean_score": 98.0, "mean_raw": 2.0, "scores_per_judge": [ 98 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 59, "duration_s": 28, "mean_score": 97.0, "mean_raw": 3.0, "scores_per_judge": [ 97 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 60, "duration_s": 18, "mean_score": 63.0, "mean_raw": 37.0, "scores_per_judge": [ 63 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "polarity", "points": 35 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 61, "duration_s": 19, "mean_score": 85.0, "mean_raw": 15.0, "scores_per_judge": [ 85 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 }, { "category": "brand", "points": 10 } ] }, "judge_disagreement": 0 }, { "idx": 62, "duration_s": 17, "mean_score": 100.0, "mean_raw": 0.0, "scores_per_judge": [ 100 ], "deductions_per_judge": { "deepseek_flash": [] }, "judge_disagreement": 0 }, { "idx": 63, "duration_s": 21, "mean_score": 100.0, "mean_raw": 0.0, "scores_per_judge": [ 100 ], "deductions_per_judge": { "deepseek_flash": [] }, "judge_disagreement": 0 }, { "idx": 64, "duration_s": 23, "mean_score": 96.0, "mean_raw": 4.0, "scores_per_judge": [ 96 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 65, "duration_s": 20, "mean_score": 100.0, "mean_raw": 0.0, "scores_per_judge": [ 100 ], "deductions_per_judge": { "deepseek_flash": [] }, "judge_disagreement": 0 }, { "idx": 66, "duration_s": 24, "mean_score": 96.0, "mean_raw": 4.0, "scores_per_judge": [ 96 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "paraphrase", "points": 3 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 67, "duration_s": 16, "mean_score": 100.0, "mean_raw": 0.0, "scores_per_judge": [ 100 ], "deductions_per_judge": { "deepseek_flash": [] }, "judge_disagreement": 0 }, { "idx": 68, "duration_s": 18, "mean_score": 100.0, "mean_raw": 0.0, "scores_per_judge": [ 100 ], "deductions_per_judge": { "deepseek_flash": [] }, "judge_disagreement": 0 }, { "idx": 69, "duration_s": 18, "mean_score": 100.0, "mean_raw": 0.0, "scores_per_judge": [ 100 ], "deductions_per_judge": { "deepseek_flash": [] }, "judge_disagreement": 0 }, { "idx": 70, "duration_s": 17, "mean_score": 99.0, "mean_raw": 1.0, "scores_per_judge": [ 99 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 71, "duration_s": 51, "mean_score": 95.0, "mean_raw": 5.0, "scores_per_judge": [ 95 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "mishear-minor", "points": 2 } ] }, "judge_disagreement": 0 }, { "idx": 72, "duration_s": 54, "mean_score": 99.0, "mean_raw": 1.0, "scores_per_judge": [ 99 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 73, "duration_s": 56, "mean_score": 87.0, "mean_raw": 13.0, "scores_per_judge": [ 87 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 }, { "category": "paraphrase", "points": 3 }, { "category": "mishear-severe", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 74, "duration_s": 45, "mean_score": 98.0, "mean_raw": 2.0, "scores_per_judge": [ 98 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 75, "duration_s": 74, "mean_score": 96.0, "mean_raw": 4.0, "scores_per_judge": [ 96 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 76, "duration_s": 58, "mean_score": 94.0, "mean_raw": 6.0, "scores_per_judge": [ 94 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "paraphrase", "points": 3 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 77, "duration_s": 130, "mean_score": 65.0, "mean_raw": 35.0, "scores_per_judge": [ 65 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "semantic-inversion", "points": 20 }, { "category": "brand", "points": 10 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 78, "duration_s": 61, "mean_score": 96.0, "mean_raw": 4.0, "scores_per_judge": [ 96 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 79, "duration_s": 47, "mean_score": 97.0, "mean_raw": 3.0, "scores_per_judge": [ 97 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 80, "duration_s": 42, "mean_score": 97.0, "mean_raw": 3.0, "scores_per_judge": [ 97 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 81, "duration_s": 38, "mean_score": 97.0, "mean_raw": 3.0, "scores_per_judge": [ 97 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 82, "duration_s": 38, "mean_score": 90.0, "mean_raw": 10.0, "scores_per_judge": [ 90 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "tech", "points": 8 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 83, "duration_s": 37, "mean_score": 97.0, "mean_raw": 3.0, "scores_per_judge": [ 97 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 84, "duration_s": 36, "mean_score": 97.0, "mean_raw": 3.0, "scores_per_judge": [ 97 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 85, "duration_s": 36, "mean_score": 96.0, "mean_raw": 4.0, "scores_per_judge": [ 96 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "mishear-minor", "points": 2 } ] }, "judge_disagreement": 0 }, { "idx": 86, "duration_s": 35, "mean_score": 92.0, "mean_raw": 8.0, "scores_per_judge": [ 92 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "tech", "points": 8 } ] }, "judge_disagreement": 0 }, { "idx": 87, "duration_s": 34, "mean_score": 94.0, "mean_raw": 6.0, "scores_per_judge": [ 94 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-minor", "points": 2 }, { "category": "filler", "points": 4 } ] }, "judge_disagreement": 0 }, { "idx": 88, "duration_s": 34, "mean_score": 98.0, "mean_raw": 2.0, "scores_per_judge": [ 98 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 89, "duration_s": 34, "mean_score": 100.0, "mean_raw": 0.0, "scores_per_judge": [ 100 ], "deductions_per_judge": { "deepseek_flash": [] }, "judge_disagreement": 0 }, { "idx": 90, "duration_s": 33, "mean_score": 97.0, "mean_raw": 3.0, "scores_per_judge": [ 97 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 91, "duration_s": 33, "mean_score": 99.0, "mean_raw": 1.0, "scores_per_judge": [ 99 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 92, "duration_s": 32, "mean_score": 100.0, "mean_raw": 0.0, "scores_per_judge": [ 100 ], "deductions_per_judge": { "deepseek_flash": [] }, "judge_disagreement": 0 }, { "idx": 93, "duration_s": 32, "mean_score": 93.0, "mean_raw": 7.0, "scores_per_judge": [ 93 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 94, "duration_s": 32, "mean_score": 93.0, "mean_raw": 7.0, "scores_per_judge": [ 93 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 95, "duration_s": 32, "mean_score": 97.0, "mean_raw": 3.0, "scores_per_judge": [ 97 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 96, "duration_s": 31, "mean_score": 99.0, "mean_raw": 1.0, "scores_per_judge": [ 99 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 97, "duration_s": 30, "mean_score": 98.0, "mean_raw": 2.0, "scores_per_judge": [ 98 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 98, "duration_s": 30, "mean_score": 99.0, "mean_raw": 1.0, "scores_per_judge": [ 99 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 99, "duration_s": 30, "mean_score": 100.0, "mean_raw": 0.0, "scores_per_judge": [ 100 ], "deductions_per_judge": { "deepseek_flash": [] }, "judge_disagreement": 0 }, { "idx": 100, "duration_s": 30, "mean_score": 98.0, "mean_raw": 2.0, "scores_per_judge": [ 98 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 } ], "total_deductions": 564, "avg_deductions_per_sample": 5.64, "mean_clamped_score": 94.4, "n_judge_sample_pairs": 100, "gt_chars": 22625, "ded_per_100char": 2.49, "fidelity_score": 87.5, "overall_score": 94.4, "overall_raw_deductions": 5.64, "score_per_judge": { "deepseek_flash": 94.4 }, "per_group_deductions": { "meaning-flip": { "deepseek_flash": 65 }, "entity": { "deepseek_flash": 212 }, "cultural": { "deepseek_flash": 24 }, "fidelity": { "deepseek_flash": 263 } }, "per_bucket": { "general": { "n_samples": 40, "mean_raw": 6.6, "mean_score": 93.4, "total_raw": 265.0, "idxs": [ 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40 ], "worst_idx": 19 }, "idiom-test": { "n_samples": 5, "mean_raw": 4.4, "mean_score": 95.6, "total_raw": 22.0, "idxs": [ 41, 42, 43, 44, 45 ], "worst_idx": 43 }, "long-mixed": { "n_samples": 5, "mean_raw": 9.4, "mean_score": 90.6, "total_raw": 47.0, "idxs": [ 46, 47, 48, 49, 50 ], "worst_idx": 48 } }, "per_group_per_100char": { "meaning-flip": 0.287, "entity": 0.937, "cultural": 0.106, "fidelity": 1.162 }, "per_group_fidelity": { "meaning-flip": 93.1, "entity": 83.4, "cultural": 93.6, "fidelity": 86.4 } }, "mimo_stt": { "scores_by_judge": { "deepseek_flash": [ 99, 99, 84, 91, 98, 99, 99, 99, 98, 92, 84, 91, 70, 78, 98, 90, 90, 86, 47, 91, 99, 88, 100, 87, 89, 92, 98, 95, 89, 93, 98, 96, 89, 58, 81, 96, 95, 93, 93, 86, 97, 98, 87, 83, 51, 96, 99, 79, 82, 87, 96, 98, 96, 96, 95, 98, 98, 93, 94, 64, 97, 100, 96, 90, 99, 90, 100, 100, 96, 62, 94, 93, 88, 99, 67, 77, 83, 96, 91, 97, 93, 91, 93, 63, 91, 88, 96, 98, 99, 90, 97, 100, 93, 98, 99, 100, 95, 99, 98, 94 ] }, "per_category_deductions": { "polarity": { "deepseek_flash": 175 }, "semantic-inversion": { "deepseek_flash": 20 }, "hallucination": { "deepseek_flash": 30 }, "numeric": { "deepseek_flash": 0 }, "brand": { "deepseek_flash": 120 }, "tech": { "deepseek_flash": 64 }, "culture": { "deepseek_flash": 56 }, "proper-noun": { "deepseek_flash": 18 }, "dialect": { "deepseek_flash": 0 }, "mishear-severe": { "deepseek_flash": 115 }, "content-drop": { "deepseek_flash": 12 }, "paraphrase": { "deepseek_flash": 15 }, "mishear-minor": { "deepseek_flash": 52 }, "filler": { "deepseek_flash": 248 }, "format": { "deepseek_flash": 0 }, "unscoreable": { "deepseek_flash": 0 } }, "per_tier_deductions": { "tier1": { "deepseek_flash": 225 }, "tier2": { "deepseek_flash": 258 }, "tier3": { "deepseek_flash": 194 }, "tier4": { "deepseek_flash": 248 }, "unscoreable": { "deepseek_flash": 0 } }, "raw_deductions_by_judge": { "deepseek_flash": [ 1, 1, 16, 9, 2, 1, 1, 1, 2, 8, 16, 9, 30, 22, 2, 10, 10, 14, 53, 9, 1, 12, 0, 13, 11, 8, 2, 5, 11, 7, 2, 4, 11, 42, 19, 4, 5, 7, 7, 14, 3, 2, 13, 17, 49, 4, 1, 21, 18, 13, 4, 2, 4, 4, 5, 2, 2, 7, 6, 36, 3, 0, 4, 10, 1, 10, 0, 0, 4, 38, 6, 7, 12, 1, 33, 23, 17, 4, 9, 3, 7, 9, 7, 37, 9, 12, 4, 2, 1, 10, 3, 0, 7, 2, 1, 0, 5, 1, 2, 6 ] }, "sample_details": [ { "idx": 1, "duration_s": 5, "mean_score": 99.0, "mean_raw": 1.0, "scores_per_judge": [ 99 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 2, "duration_s": 7, "mean_score": 99.0, "mean_raw": 1.0, "scores_per_judge": [ 99 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 3, "duration_s": 8, "mean_score": 84.0, "mean_raw": 16.0, "scores_per_judge": [ 84 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "hallucination", "points": 15 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 4, "duration_s": 11, "mean_score": 91.0, "mean_raw": 9.0, "scores_per_judge": [ 91 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "culture", "points": 8 } ] }, "judge_disagreement": 0 }, { "idx": 5, "duration_s": 16, "mean_score": 98.0, "mean_raw": 2.0, "scores_per_judge": [ 98 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 6, "duration_s": 17, "mean_score": 99.0, "mean_raw": 1.0, "scores_per_judge": [ 99 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 7, "duration_s": 20, "mean_score": 99.0, "mean_raw": 1.0, "scores_per_judge": [ 99 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 8, "duration_s": 30, "mean_score": 99.0, "mean_raw": 1.0, "scores_per_judge": [ 99 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 9, "duration_s": 32, "mean_score": 98.0, "mean_raw": 2.0, "scores_per_judge": [ 98 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 10, "duration_s": 35, "mean_score": 92.0, "mean_raw": 8.0, "scores_per_judge": [ 92 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 5 }, { "category": "content-drop", "points": 3 } ] }, "judge_disagreement": 0 }, { "idx": 11, "duration_s": 40, "mean_score": 84.0, "mean_raw": 16.0, "scores_per_judge": [ 84 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "content-drop", "points": 3 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 12, "duration_s": 46, "mean_score": 91.0, "mean_raw": 9.0, "scores_per_judge": [ 91 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "culture", "points": 8 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 13, "duration_s": 60, "mean_score": 70.0, "mean_raw": 30.0, "scores_per_judge": [ 70 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "brand", "points": 10 }, { "category": "brand", "points": 10 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-minor", "points": 2 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 14, "duration_s": 64, "mean_score": 78.0, "mean_raw": 22.0, "scores_per_judge": [ 78 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-minor", "points": 2 }, { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 5 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-severe", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 15, "duration_s": 69, "mean_score": 98.0, "mean_raw": 2.0, "scores_per_judge": [ 98 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 2 } ] }, "judge_disagreement": 0 }, { "idx": 16, "duration_s": 81, "mean_score": 90.0, "mean_raw": 10.0, "scores_per_judge": [ 90 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "proper-noun", "points": 6 }, { "category": "filler", "points": 4 } ] }, "judge_disagreement": 0 }, { "idx": 17, "duration_s": 100, "mean_score": 90.0, "mean_raw": 10.0, "scores_per_judge": [ 90 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 18, "duration_s": 125, "mean_score": 86.0, "mean_raw": 14.0, "scores_per_judge": [ 86 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "brand", "points": 10 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 19, "duration_s": 133, "mean_score": 47.0, "mean_raw": 53.0, "scores_per_judge": [ 47 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "brand", "points": 10 }, { "category": "brand", "points": 10 }, { "category": "brand", "points": 10 }, { "category": "brand", "points": 10 }, { "category": "content-drop", "points": 3 }, { "category": "paraphrase", "points": 3 }, { "category": "mishear-minor", "points": 2 }, { "category": "filler", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 20, "duration_s": 164, "mean_score": 91.0, "mean_raw": 9.0, "scores_per_judge": [ 91 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 5 }, { "category": "mishear-minor", "points": 2 }, { "category": "mishear-minor", "points": 2 } ] }, "judge_disagreement": 0 }, { "idx": 21, "duration_s": 5, "mean_score": 99.0, "mean_raw": 1.0, "scores_per_judge": [ 99 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 22, "duration_s": 7, "mean_score": 88.0, "mean_raw": 12.0, "scores_per_judge": [ 88 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "brand", "points": 10 }, { "category": "mishear-minor", "points": 2 } ] }, "judge_disagreement": 0 }, { "idx": 23, "duration_s": 10, "mean_score": 100.0, "mean_raw": 0.0, "scores_per_judge": [ 100 ], "deductions_per_judge": { "deepseek_flash": [] }, "judge_disagreement": 0 }, { "idx": 24, "duration_s": 14, "mean_score": 87.0, "mean_raw": 13.0, "scores_per_judge": [ 87 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "tech", "points": 8 }, { "category": "filler", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 25, "duration_s": 16, "mean_score": 89.0, "mean_raw": 11.0, "scores_per_judge": [ 89 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "tech", "points": 8 }, { "category": "filler", "points": 3 } ] }, "judge_disagreement": 0 }, { "idx": 26, "duration_s": 23, "mean_score": 92.0, "mean_raw": 8.0, "scores_per_judge": [ 92 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 27, "duration_s": 25, "mean_score": 98.0, "mean_raw": 2.0, "scores_per_judge": [ 98 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 28, "duration_s": 30, "mean_score": 95.0, "mean_raw": 5.0, "scores_per_judge": [ 95 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 29, "duration_s": 34, "mean_score": 89.0, "mean_raw": 11.0, "scores_per_judge": [ 89 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "brand", "points": 10 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 30, "duration_s": 38, "mean_score": 93.0, "mean_raw": 7.0, "scores_per_judge": [ 93 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 31, "duration_s": 44, "mean_score": 98.0, "mean_raw": 2.0, "scores_per_judge": [ 98 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 32, "duration_s": 50, "mean_score": 96.0, "mean_raw": 4.0, "scores_per_judge": [ 96 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 33, "duration_s": 62, "mean_score": 89.0, "mean_raw": 11.0, "scores_per_judge": [ 89 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "proper-noun", "points": 6 }, { "category": "mishear-minor", "points": 2 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 34, "duration_s": 66, "mean_score": 58.0, "mean_raw": 42.0, "scores_per_judge": [ 58 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "polarity", "points": 35 }, { "category": "mishear-minor", "points": 2 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 35, "duration_s": 77, "mean_score": 81.0, "mean_raw": 19.0, "scores_per_judge": [ 81 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "mishear-minor", "points": 2 }, { "category": "filler", "points": 1 }, { "category": "mishear-minor", "points": 2 }, { "category": "filler", "points": 1 }, { "category": "tech", "points": 8 }, { "category": "filler", "points": 1 }, { "category": "mishear-minor", "points": 2 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 36, "duration_s": 94, "mean_score": 96.0, "mean_raw": 4.0, "scores_per_judge": [ 96 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-minor", "points": 2 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 37, "duration_s": 106, "mean_score": 95.0, "mean_raw": 5.0, "scores_per_judge": [ 95 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 38, "duration_s": 126, "mean_score": 93.0, "mean_raw": 7.0, "scores_per_judge": [ 93 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 39, "duration_s": 158, "mean_score": 93.0, "mean_raw": 7.0, "scores_per_judge": [ 93 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 5 }, { "category": "mishear-minor", "points": 2 } ] }, "judge_disagreement": 0 }, { "idx": 40, "duration_s": 279, "mean_score": 86.0, "mean_raw": 14.0, "scores_per_judge": [ 86 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "paraphrase", "points": 3 }, { "category": "paraphrase", "points": 3 }, { "category": "content-drop", "points": 3 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 41, "duration_s": 34, "mean_score": 97.0, "mean_raw": 3.0, "scores_per_judge": [ 97 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 42, "duration_s": 43, "mean_score": 98.0, "mean_raw": 2.0, "scores_per_judge": [ 98 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-minor", "points": 2 } ] }, "judge_disagreement": 0 }, { "idx": 43, "duration_s": 29, "mean_score": 87.0, "mean_raw": 13.0, "scores_per_judge": [ 87 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "culture", "points": 8 }, { "category": "filler", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 44, "duration_s": 71, "mean_score": 83.0, "mean_raw": 17.0, "scores_per_judge": [ 83 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "culture", "points": 8 }, { "category": "culture", "points": 8 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 45, "duration_s": 46, "mean_score": 51.0, "mean_raw": 49.0, "scores_per_judge": [ 51 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "polarity", "points": 35 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 46, "duration_s": 62, "mean_score": 96.0, "mean_raw": 4.0, "scores_per_judge": [ 96 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 47, "duration_s": 67, "mean_score": 99.0, "mean_raw": 1.0, "scores_per_judge": [ 99 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 48, "duration_s": 72, "mean_score": 79.0, "mean_raw": 21.0, "scores_per_judge": [ 79 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "brand", "points": 10 }, { "category": "brand", "points": 10 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 49, "duration_s": 87, "mean_score": 82.0, "mean_raw": 18.0, "scores_per_judge": [ 82 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "tech", "points": 8 }, { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 50, "duration_s": 106, "mean_score": 87.0, "mean_raw": 13.0, "scores_per_judge": [ 87 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "culture", "points": 8 }, { "category": "filler", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 51, "duration_s": 33, "mean_score": 96.0, "mean_raw": 4.0, "scores_per_judge": [ 96 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 52, "duration_s": 25, "mean_score": 98.0, "mean_raw": 2.0, "scores_per_judge": [ 98 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 53, "duration_s": 35, "mean_score": 96.0, "mean_raw": 4.0, "scores_per_judge": [ 96 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 54, "duration_s": 36, "mean_score": 96.0, "mean_raw": 4.0, "scores_per_judge": [ 96 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 55, "duration_s": 33, "mean_score": 95.0, "mean_raw": 5.0, "scores_per_judge": [ 95 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "paraphrase", "points": 3 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 56, "duration_s": 15, "mean_score": 98.0, "mean_raw": 2.0, "scores_per_judge": [ 98 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 57, "duration_s": 38, "mean_score": 98.0, "mean_raw": 2.0, "scores_per_judge": [ 98 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 2 } ] }, "judge_disagreement": 0 }, { "idx": 58, "duration_s": 15, "mean_score": 93.0, "mean_raw": 7.0, "scores_per_judge": [ 93 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 59, "duration_s": 28, "mean_score": 94.0, "mean_raw": 6.0, "scores_per_judge": [ 94 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "mishear-minor", "points": 2 } ] }, "judge_disagreement": 0 }, { "idx": 60, "duration_s": 18, "mean_score": 64.0, "mean_raw": 36.0, "scores_per_judge": [ 64 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "polarity", "points": 35 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 61, "duration_s": 19, "mean_score": 97.0, "mean_raw": 3.0, "scores_per_judge": [ 97 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "mishear-minor", "points": 2 } ] }, "judge_disagreement": 0 }, { "idx": 62, "duration_s": 17, "mean_score": 100.0, "mean_raw": 0.0, "scores_per_judge": [ 100 ], "deductions_per_judge": { "deepseek_flash": [] }, "judge_disagreement": 0 }, { "idx": 63, "duration_s": 21, "mean_score": 96.0, "mean_raw": 4.0, "scores_per_judge": [ 96 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 4 } ] }, "judge_disagreement": 0 }, { "idx": 64, "duration_s": 23, "mean_score": 90.0, "mean_raw": 10.0, "scores_per_judge": [ 90 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 65, "duration_s": 20, "mean_score": 99.0, "mean_raw": 1.0, "scores_per_judge": [ 99 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 66, "duration_s": 24, "mean_score": 90.0, "mean_raw": 10.0, "scores_per_judge": [ 90 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "culture", "points": 8 }, { "category": "mishear-minor", "points": 2 } ] }, "judge_disagreement": 0 }, { "idx": 67, "duration_s": 16, "mean_score": 100.0, "mean_raw": 0.0, "scores_per_judge": [ 100 ], "deductions_per_judge": { "deepseek_flash": [] }, "judge_disagreement": 0 }, { "idx": 68, "duration_s": 18, "mean_score": 100.0, "mean_raw": 0.0, "scores_per_judge": [ 100 ], "deductions_per_judge": { "deepseek_flash": [] }, "judge_disagreement": 0 }, { "idx": 69, "duration_s": 18, "mean_score": 96.0, "mean_raw": 4.0, "scores_per_judge": [ 96 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-minor", "points": 2 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 70, "duration_s": 17, "mean_score": 62.0, "mean_raw": 38.0, "scores_per_judge": [ 62 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "polarity", "points": 35 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 71, "duration_s": 51, "mean_score": 94.0, "mean_raw": 6.0, "scores_per_judge": [ 94 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 72, "duration_s": 54, "mean_score": 93.0, "mean_raw": 7.0, "scores_per_judge": [ 93 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 73, "duration_s": 56, "mean_score": 88.0, "mean_raw": 12.0, "scores_per_judge": [ 88 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "proper-noun", "points": 6 }, { "category": "paraphrase", "points": 3 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 74, "duration_s": 45, "mean_score": 99.0, "mean_raw": 1.0, "scores_per_judge": [ 99 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 75, "duration_s": 74, "mean_score": 67.0, "mean_raw": 33.0, "scores_per_judge": [ 67 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "tech", "points": 8 }, { "category": "mishear-severe", "points": 5 }, { "category": "hallucination", "points": 15 }, { "category": "filler", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 76, "duration_s": 58, "mean_score": 77.0, "mean_raw": 23.0, "scores_per_judge": [ 77 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "semantic-inversion", "points": 20 }, { "category": "filler", "points": 3 } ] }, "judge_disagreement": 0 }, { "idx": 77, "duration_s": 130, "mean_score": 83.0, "mean_raw": 17.0, "scores_per_judge": [ 83 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "brand", "points": 10 }, { "category": "mishear-minor", "points": 2 }, { "category": "filler", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 78, "duration_s": 61, "mean_score": 96.0, "mean_raw": 4.0, "scores_per_judge": [ 96 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-minor", "points": 2 }, { "category": "mishear-minor", "points": 2 } ] }, "judge_disagreement": 0 }, { "idx": 79, "duration_s": 47, "mean_score": 91.0, "mean_raw": 9.0, "scores_per_judge": [ 91 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-minor", "points": 2 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "mishear-severe", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 80, "duration_s": 42, "mean_score": 97.0, "mean_raw": 3.0, "scores_per_judge": [ 97 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 81, "duration_s": 38, "mean_score": 93.0, "mean_raw": 7.0, "scores_per_judge": [ 93 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 }, { "category": "mishear-minor", "points": 2 } ] }, "judge_disagreement": 0 }, { "idx": 82, "duration_s": 38, "mean_score": 91.0, "mean_raw": 9.0, "scores_per_judge": [ 91 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "tech", "points": 8 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 83, "duration_s": 37, "mean_score": 93.0, "mean_raw": 7.0, "scores_per_judge": [ 93 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "mishear-minor", "points": 2 } ] }, "judge_disagreement": 0 }, { "idx": 84, "duration_s": 36, "mean_score": 63.0, "mean_raw": 37.0, "scores_per_judge": [ 63 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "polarity", "points": 35 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 85, "duration_s": 36, "mean_score": 91.0, "mean_raw": 9.0, "scores_per_judge": [ 91 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 }, { "category": "mishear-minor", "points": 2 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 86, "duration_s": 35, "mean_score": 88.0, "mean_raw": 12.0, "scores_per_judge": [ 88 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "tech", "points": 8 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 87, "duration_s": 34, "mean_score": 96.0, "mean_raw": 4.0, "scores_per_judge": [ 96 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 88, "duration_s": 34, "mean_score": 98.0, "mean_raw": 2.0, "scores_per_judge": [ 98 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 89, "duration_s": 34, "mean_score": 99.0, "mean_raw": 1.0, "scores_per_judge": [ 99 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 90, "duration_s": 33, "mean_score": 90.0, "mean_raw": 10.0, "scores_per_judge": [ 90 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "tech", "points": 8 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 91, "duration_s": 33, "mean_score": 97.0, "mean_raw": 3.0, "scores_per_judge": [ 97 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "mishear-minor", "points": 2 } ] }, "judge_disagreement": 0 }, { "idx": 92, "duration_s": 32, "mean_score": 100.0, "mean_raw": 0.0, "scores_per_judge": [ 100 ], "deductions_per_judge": { "deepseek_flash": [] }, "judge_disagreement": 0 }, { "idx": 93, "duration_s": 32, "mean_score": 93.0, "mean_raw": 7.0, "scores_per_judge": [ 93 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 94, "duration_s": 32, "mean_score": 98.0, "mean_raw": 2.0, "scores_per_judge": [ 98 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 95, "duration_s": 32, "mean_score": 99.0, "mean_raw": 1.0, "scores_per_judge": [ 99 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 96, "duration_s": 31, "mean_score": 100.0, "mean_raw": 0.0, "scores_per_judge": [ 100 ], "deductions_per_judge": { "deepseek_flash": [] }, "judge_disagreement": 0 }, { "idx": 97, "duration_s": 30, "mean_score": 95.0, "mean_raw": 5.0, "scores_per_judge": [ 95 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 98, "duration_s": 30, "mean_score": 99.0, "mean_raw": 1.0, "scores_per_judge": [ 99 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 99, "duration_s": 30, "mean_score": 98.0, "mean_raw": 2.0, "scores_per_judge": [ 98 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 100, "duration_s": 30, "mean_score": 94.0, "mean_raw": 6.0, "scores_per_judge": [ 94 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 } ], "total_deductions": 925, "avg_deductions_per_sample": 9.25, "mean_clamped_score": 90.8, "n_judge_sample_pairs": 100, "gt_chars": 22625, "ded_per_100char": 4.09, "fidelity_score": 79.6, "overall_score": 90.8, "overall_raw_deductions": 9.25, "score_per_judge": { "deepseek_flash": 90.8 }, "per_group_deductions": { "meaning-flip": { "deepseek_flash": 225 }, "entity": { "deepseek_flash": 202 }, "cultural": { "deepseek_flash": 56 }, "fidelity": { "deepseek_flash": 442 } }, "per_bucket": { "general": { "n_samples": 40, "mean_raw": 10.1, "mean_score": 90.0, "total_raw": 402.0, "idxs": [ 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40 ], "worst_idx": 19 }, "idiom-test": { "n_samples": 5, "mean_raw": 16.8, "mean_score": 83.2, "total_raw": 84.0, "idxs": [ 41, 42, 43, 44, 45 ], "worst_idx": 45 }, "long-mixed": { "n_samples": 5, "mean_raw": 11.4, "mean_score": 88.6, "total_raw": 57.0, "idxs": [ 46, 47, 48, 49, 50 ], "worst_idx": 48 } }, "per_group_per_100char": { "meaning-flip": 0.994, "entity": 0.893, "cultural": 0.248, "fidelity": 1.954 }, "per_group_fidelity": { "meaning-flip": 76.2, "entity": 84.2, "cultural": 85.0, "fidelity": 77.1 } }, "moss_audio": { "scores_by_judge": { "deepseek_flash": [ 100, 100, 100, 100, 97, 91, 97, 99, 93, 95, 0, 53, 57, 70, 5, 50, 60, 82, null, 82, 99, 89, 100, 90, 92, 64, 100, 97, 54, 82, 47, 58, 64, 48, 51, 57, null, 65, 72, 36, 98, 51, 80, 68, 25, 49, 6, 82, 82, 56, 86, 81, 93, 91, 75, 76, 55, 89, 97, 63, 85, 100, 100, 93, 100, 90, 99, 88, 99, 55, 60, 42, 59, 75, 78, 47, 72, 50, 56, 82, 54, 59, 84, 92, 82, 79, 80, 96, 76, 89, 93, 95, 100, 92, 95, 100, 95, 99, 100, 94 ] }, "per_category_deductions": { "polarity": { "deepseek_flash": 420 }, "semantic-inversion": { "deepseek_flash": 180 }, "hallucination": { "deepseek_flash": 300 }, "numeric": { "deepseek_flash": 20 }, "brand": { "deepseek_flash": 160 }, "tech": { "deepseek_flash": 128 }, "culture": { "deepseek_flash": 104 }, "proper-noun": { "deepseek_flash": 48 }, "dialect": { "deepseek_flash": 10 }, "mishear-severe": { "deepseek_flash": 350 }, "content-drop": { "deepseek_flash": 222 }, "paraphrase": { "deepseek_flash": 30 }, "mishear-minor": { "deepseek_flash": 50 }, "filler": { "deepseek_flash": 195 }, "format": { "deepseek_flash": 0 }, "unscoreable": { "deepseek_flash": 100 } }, "per_tier_deductions": { "tier1": { "deepseek_flash": 920 }, "tier2": { "deepseek_flash": 450 }, "tier3": { "deepseek_flash": 652 }, "tier4": { "deepseek_flash": 195 }, "unscoreable": { "deepseek_flash": 100 } }, "raw_deductions_by_judge": { "deepseek_flash": [ 0, 0, 0, 0, 3, 9, 3, 1, 7, 5, 100, 47, 43, 30, 95, 50, 40, 18, null, 18, 1, 11, 0, 10, 8, 36, 0, 3, 46, 18, 53, 42, 36, 52, 49, 43, null, 35, 28, 64, 2, 49, 20, 32, 75, 51, 94, 18, 18, 44, 14, 19, 7, 9, 25, 24, 45, 11, 3, 37, 15, 0, 0, 7, 0, 10, 1, 12, 1, 45, 40, 58, 41, 25, 22, 53, 28, 50, 44, 18, 46, 41, 16, 8, 18, 21, 20, 4, 24, 11, 7, 5, 0, 8, 5, 0, 5, 1, 0, 6 ] }, "sample_details": [ { "idx": 1, "duration_s": 5, "mean_score": 100.0, "mean_raw": 0.0, "scores_per_judge": [ 100 ], "deductions_per_judge": { "deepseek_flash": [] }, "judge_disagreement": 0 }, { "idx": 2, "duration_s": 7, "mean_score": 100.0, "mean_raw": 0.0, "scores_per_judge": [ 100 ], "deductions_per_judge": { "deepseek_flash": [] }, "judge_disagreement": 0 }, { "idx": 3, "duration_s": 8, "mean_score": 100.0, "mean_raw": 0.0, "scores_per_judge": [ 100 ], "deductions_per_judge": { "deepseek_flash": [] }, "judge_disagreement": 0 }, { "idx": 4, "duration_s": 11, "mean_score": 100.0, "mean_raw": 0.0, "scores_per_judge": [ 100 ], "deductions_per_judge": { "deepseek_flash": [] }, "judge_disagreement": 0 }, { "idx": 5, "duration_s": 16, "mean_score": 97.0, "mean_raw": 3.0, "scores_per_judge": [ 97 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 6, "duration_s": 17, "mean_score": 91.0, "mean_raw": 9.0, "scores_per_judge": [ 91 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "content-drop", "points": 3 }, { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 7, "duration_s": 20, "mean_score": 97.0, "mean_raw": 3.0, "scores_per_judge": [ 97 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 8, "duration_s": 30, "mean_score": 99.0, "mean_raw": 1.0, "scores_per_judge": [ 99 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 9, "duration_s": 32, "mean_score": 93.0, "mean_raw": 7.0, "scores_per_judge": [ 93 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 5 }, { "category": "mishear-minor", "points": 2 } ] }, "judge_disagreement": 0 }, { "idx": 10, "duration_s": 35, "mean_score": 95.0, "mean_raw": 5.0, "scores_per_judge": [ 95 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 11, "duration_s": 40, "mean_score": 0.0, "mean_raw": 100.0, "scores_per_judge": [ 0 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "unscoreable", "points": 100 } ] }, "judge_disagreement": 0 }, { "idx": 12, "duration_s": 46, "mean_score": 53.0, "mean_raw": 47.0, "scores_per_judge": [ 53 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "tech", "points": 8 }, { "category": "semantic-inversion", "points": 20 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-severe", "points": 5 }, { "category": "content-drop", "points": 3 }, { "category": "content-drop", "points": 3 }, { "category": "paraphrase", "points": 3 } ] }, "judge_disagreement": 0 }, { "idx": 13, "duration_s": 60, "mean_score": 57.0, "mean_raw": 43.0, "scores_per_judge": [ 57 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "brand", "points": 10 }, { "category": "brand", "points": 10 }, { "category": "tech", "points": 8 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-minor", "points": 2 }, { "category": "content-drop", "points": 3 }, { "category": "filler", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 14, "duration_s": 64, "mean_score": 70.0, "mean_raw": 30.0, "scores_per_judge": [ 70 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "paraphrase", "points": 3 }, { "category": "content-drop", "points": 3 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-severe", "points": 5 }, { "category": "content-drop", "points": 3 }, { "category": "paraphrase", "points": 3 }, { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 3 } ] }, "judge_disagreement": 0 }, { "idx": 15, "duration_s": 69, "mean_score": 5.0, "mean_raw": 95.0, "scores_per_judge": [ 5 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "polarity", "points": 35 }, { "category": "semantic-inversion", "points": 20 }, { "category": "culture", "points": 8 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-severe", "points": 5 }, { "category": "content-drop", "points": 3 }, { "category": "content-drop", "points": 3 }, { "category": "content-drop", "points": 3 }, { "category": "content-drop", "points": 3 }, { "category": "content-drop", "points": 3 }, { "category": "mishear-minor", "points": 2 } ] }, "judge_disagreement": 0 }, { "idx": 16, "duration_s": 81, "mean_score": 50.0, "mean_raw": 50.0, "scores_per_judge": [ 50 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "proper-noun", "points": 6 }, { "category": "content-drop", "points": 3 }, { "category": "mishear-severe", "points": 5 }, { "category": "tech", "points": 8 }, { "category": "content-drop", "points": 3 }, { "category": "tech", "points": 8 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-severe", "points": 5 }, { "category": "content-drop", "points": 3 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 17, "duration_s": 100, "mean_score": 60.0, "mean_raw": 40.0, "scores_per_judge": [ 60 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "mishear-minor", "points": 2 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "content-drop", "points": 3 }, { "category": "mishear-minor", "points": 2 }, { "category": "brand", "points": 10 }, { "category": "tech", "points": 8 }, { "category": "content-drop", "points": 3 }, { "category": "content-drop", "points": 3 }, { "category": "mishear-severe", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 18, "duration_s": 125, "mean_score": 82.0, "mean_raw": 18.0, "scores_per_judge": [ 82 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "content-drop", "points": 3 }, { "category": "hallucination", "points": 15 } ] }, "judge_disagreement": 0 }, { "idx": 20, "duration_s": 164, "mean_score": 82.0, "mean_raw": 18.0, "scores_per_judge": [ 82 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "hallucination", "points": 15 }, { "category": "content-drop", "points": 3 } ] }, "judge_disagreement": 0 }, { "idx": 21, "duration_s": 5, "mean_score": 99.0, "mean_raw": 1.0, "scores_per_judge": [ 99 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 22, "duration_s": 7, "mean_score": 89.0, "mean_raw": 11.0, "scores_per_judge": [ 89 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "brand", "points": 10 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 23, "duration_s": 10, "mean_score": 100.0, "mean_raw": 0.0, "scores_per_judge": [ 100 ], "deductions_per_judge": { "deepseek_flash": [] }, "judge_disagreement": 0 }, { "idx": 24, "duration_s": 14, "mean_score": 90.0, "mean_raw": 10.0, "scores_per_judge": [ 90 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "tech", "points": 8 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 25, "duration_s": 16, "mean_score": 92.0, "mean_raw": 8.0, "scores_per_judge": [ 92 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "tech", "points": 8 } ] }, "judge_disagreement": 0 }, { "idx": 26, "duration_s": 23, "mean_score": 64.0, "mean_raw": 36.0, "scores_per_judge": [ 64 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "polarity", "points": 35 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 27, "duration_s": 25, "mean_score": 100.0, "mean_raw": 0.0, "scores_per_judge": [ 100 ], "deductions_per_judge": { "deepseek_flash": [] }, "judge_disagreement": 0 }, { "idx": 28, "duration_s": 30, "mean_score": 97.0, "mean_raw": 3.0, "scores_per_judge": [ 97 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 29, "duration_s": 34, "mean_score": 54.0, "mean_raw": 46.0, "scores_per_judge": [ 54 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "brand", "points": 10 }, { "category": "brand", "points": 10 }, { "category": "brand", "points": 10 }, { "category": "tech", "points": 8 }, { "category": "paraphrase", "points": 3 }, { "category": "filler", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 30, "duration_s": 38, "mean_score": 82.0, "mean_raw": 18.0, "scores_per_judge": [ 82 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 31, "duration_s": 44, "mean_score": 47.0, "mean_raw": 53.0, "scores_per_judge": [ 47 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "polarity", "points": 35 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-severe", "points": 5 }, { "category": "paraphrase", "points": 3 }, { "category": "filler", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 32, "duration_s": 50, "mean_score": 58.0, "mean_raw": 42.0, "scores_per_judge": [ 58 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "semantic-inversion", "points": 20 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 2 } ] }, "judge_disagreement": 0 }, { "idx": 33, "duration_s": 62, "mean_score": 64.0, "mean_raw": 36.0, "scores_per_judge": [ 64 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "brand", "points": 10 }, { "category": "hallucination", "points": 15 }, { "category": "content-drop", "points": 3 }, { "category": "content-drop", "points": 3 }, { "category": "mishear-minor", "points": 2 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 34, "duration_s": 66, "mean_score": 48.0, "mean_raw": 52.0, "scores_per_judge": [ 48 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "polarity", "points": 35 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-minor", "points": 2 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 35, "duration_s": 77, "mean_score": 51.0, "mean_raw": 49.0, "scores_per_judge": [ 51 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 }, { "category": "tech", "points": 8 }, { "category": "mishear-severe", "points": 5 }, { "category": "tech", "points": 8 }, { "category": "hallucination", "points": 15 }, { "category": "content-drop", "points": 3 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 36, "duration_s": 94, "mean_score": 57.0, "mean_raw": 43.0, "scores_per_judge": [ 57 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-severe", "points": 5 }, { "category": "tech", "points": 8 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 38, "duration_s": 126, "mean_score": 65.0, "mean_raw": 35.0, "scores_per_judge": [ 65 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "proper-noun", "points": 6 }, { "category": "culture", "points": 8 }, { "category": "mishear-severe", "points": 5 }, { "category": "content-drop", "points": 3 }, { "category": "content-drop", "points": 3 }, { "category": "content-drop", "points": 3 }, { "category": "content-drop", "points": 3 }, { "category": "mishear-minor", "points": 2 }, { "category": "filler", "points": 2 } ] }, "judge_disagreement": 0 }, { "idx": 39, "duration_s": 158, "mean_score": 72.0, "mean_raw": 28.0, "scores_per_judge": [ 72 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "hallucination", "points": 15 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-severe", "points": 5 }, { "category": "content-drop", "points": 3 } ] }, "judge_disagreement": 0 }, { "idx": 40, "duration_s": 279, "mean_score": 36.0, "mean_raw": 64.0, "scores_per_judge": [ 36 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "semantic-inversion", "points": 20 }, { "category": "culture", "points": 8 }, { "category": "hallucination", "points": 15 }, { "category": "hallucination", "points": 15 }, { "category": "content-drop", "points": 3 }, { "category": "content-drop", "points": 3 } ] }, "judge_disagreement": 0 }, { "idx": 41, "duration_s": 34, "mean_score": 98.0, "mean_raw": 2.0, "scores_per_judge": [ 98 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 42, "duration_s": 43, "mean_score": 51.0, "mean_raw": 49.0, "scores_per_judge": [ 51 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "semantic-inversion", "points": 20 }, { "category": "culture", "points": 8 }, { "category": "culture", "points": 8 }, { "category": "mishear-severe", "points": 5 }, { "category": "paraphrase", "points": 3 }, { "category": "filler", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 43, "duration_s": 29, "mean_score": 80.0, "mean_raw": 20.0, "scores_per_judge": [ 80 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "culture", "points": 8 }, { "category": "proper-noun", "points": 6 }, { "category": "content-drop", "points": 3 }, { "category": "filler", "points": 3 } ] }, "judge_disagreement": 0 }, { "idx": 44, "duration_s": 71, "mean_score": 68.0, "mean_raw": 32.0, "scores_per_judge": [ 68 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "proper-noun", "points": 6 }, { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 1 }, { "category": "content-drop", "points": 3 }, { "category": "filler", "points": 1 }, { "category": "mishear-severe", "points": 5 }, { "category": "content-drop", "points": 3 }, { "category": "culture", "points": 8 } ] }, "judge_disagreement": 0 }, { "idx": 45, "duration_s": 46, "mean_score": 25.0, "mean_raw": 75.0, "scores_per_judge": [ 25 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "dialect", "points": 5 }, { "category": "dialect", "points": 5 }, { "category": "mishear-minor", "points": 2 }, { "category": "semantic-inversion", "points": 20 }, { "category": "polarity", "points": 35 }, { "category": "content-drop", "points": 3 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 46, "duration_s": 62, "mean_score": 49.0, "mean_raw": 51.0, "scores_per_judge": [ 49 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "content-drop", "points": 3 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-severe", "points": 5 }, { "category": "content-drop", "points": 3 }, { "category": "mishear-severe", "points": 5 }, { "category": "semantic-inversion", "points": 20 }, { "category": "content-drop", "points": 3 }, { "category": "mishear-severe", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 47, "duration_s": 67, "mean_score": 6.0, "mean_raw": 94.0, "scores_per_judge": [ 6 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 }, { "category": "brand", "points": 10 }, { "category": "filler", "points": 1 }, { "category": "content-drop", "points": 3 }, { "category": "mishear-severe", "points": 5 }, { "category": "polarity", "points": 35 }, { "category": "hallucination", "points": 15 }, { "category": "semantic-inversion", "points": 20 } ] }, "judge_disagreement": 0 }, { "idx": 48, "duration_s": 72, "mean_score": 82.0, "mean_raw": 18.0, "scores_per_judge": [ 82 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "hallucination", "points": 15 }, { "category": "content-drop", "points": 3 } ] }, "judge_disagreement": 0 }, { "idx": 49, "duration_s": 87, "mean_score": 82.0, "mean_raw": 18.0, "scores_per_judge": [ 82 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "hallucination", "points": 15 }, { "category": "content-drop", "points": 3 } ] }, "judge_disagreement": 0 }, { "idx": 50, "duration_s": 106, "mean_score": 56.0, "mean_raw": 44.0, "scores_per_judge": [ 56 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "culture", "points": 8 }, { "category": "culture", "points": 8 }, { "category": "hallucination", "points": 15 }, { "category": "mishear-severe", "points": 5 }, { "category": "content-drop", "points": 3 }, { "category": "mishear-severe", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 51, "duration_s": 33, "mean_score": 86.0, "mean_raw": 14.0, "scores_per_judge": [ 86 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "culture", "points": 8 }, { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 52, "duration_s": 25, "mean_score": 81.0, "mean_raw": 19.0, "scores_per_judge": [ 81 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "proper-noun", "points": 6 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-minor", "points": 2 }, { "category": "mishear-minor", "points": 2 }, { "category": "filler", "points": 4 } ] }, "judge_disagreement": 0 }, { "idx": 53, "duration_s": 35, "mean_score": 93.0, "mean_raw": 7.0, "scores_per_judge": [ 93 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 54, "duration_s": 36, "mean_score": 91.0, "mean_raw": 9.0, "scores_per_judge": [ 91 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 4 } ] }, "judge_disagreement": 0 }, { "idx": 55, "duration_s": 33, "mean_score": 75.0, "mean_raw": 25.0, "scores_per_judge": [ 75 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "proper-noun", "points": 6 }, { "category": "mishear-minor", "points": 2 }, { "category": "hallucination", "points": 15 }, { "category": "filler", "points": 2 } ] }, "judge_disagreement": 0 }, { "idx": 56, "duration_s": 15, "mean_score": 76.0, "mean_raw": 24.0, "scores_per_judge": [ 76 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "brand", "points": 10 }, { "category": "brand", "points": 10 }, { "category": "content-drop", "points": 3 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 57, "duration_s": 38, "mean_score": 55.0, "mean_raw": 45.0, "scores_per_judge": [ 55 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "polarity", "points": 35 }, { "category": "proper-noun", "points": 6 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 58, "duration_s": 15, "mean_score": 89.0, "mean_raw": 11.0, "scores_per_judge": [ 89 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "brand", "points": 10 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 59, "duration_s": 28, "mean_score": 97.0, "mean_raw": 3.0, "scores_per_judge": [ 97 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-minor", "points": 2 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 60, "duration_s": 18, "mean_score": 63.0, "mean_raw": 37.0, "scores_per_judge": [ 63 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "polarity", "points": 35 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 61, "duration_s": 19, "mean_score": 85.0, "mean_raw": 15.0, "scores_per_judge": [ 85 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "brand", "points": 10 }, { "category": "mishear-severe", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 62, "duration_s": 17, "mean_score": 100.0, "mean_raw": 0.0, "scores_per_judge": [ 100 ], "deductions_per_judge": { "deepseek_flash": [] }, "judge_disagreement": 0 }, { "idx": 63, "duration_s": 21, "mean_score": 100.0, "mean_raw": 0.0, "scores_per_judge": [ 100 ], "deductions_per_judge": { "deepseek_flash": [] }, "judge_disagreement": 0 }, { "idx": 64, "duration_s": 23, "mean_score": 93.0, "mean_raw": 7.0, "scores_per_judge": [ 93 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "paraphrase", "points": 3 }, { "category": "filler", "points": 4 } ] }, "judge_disagreement": 0 }, { "idx": 65, "duration_s": 20, "mean_score": 100.0, "mean_raw": 0.0, "scores_per_judge": [ 100 ], "deductions_per_judge": { "deepseek_flash": [] }, "judge_disagreement": 0 }, { "idx": 66, "duration_s": 24, "mean_score": 90.0, "mean_raw": 10.0, "scores_per_judge": [ 90 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "culture", "points": 8 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 67, "duration_s": 16, "mean_score": 99.0, "mean_raw": 1.0, "scores_per_judge": [ 99 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 68, "duration_s": 18, "mean_score": 88.0, "mean_raw": 12.0, "scores_per_judge": [ 88 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "brand", "points": 10 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 69, "duration_s": 18, "mean_score": 99.0, "mean_raw": 1.0, "scores_per_judge": [ 99 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 70, "duration_s": 17, "mean_score": 55.0, "mean_raw": 45.0, "scores_per_judge": [ 55 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "polarity", "points": 35 }, { "category": "tech", "points": 8 }, { "category": "mishear-minor", "points": 2 } ] }, "judge_disagreement": 0 }, { "idx": 71, "duration_s": 51, "mean_score": 60.0, "mean_raw": 40.0, "scores_per_judge": [ 60 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "tech", "points": 8 }, { "category": "content-drop", "points": 3 }, { "category": "mishear-minor", "points": 2 }, { "category": "brand", "points": 10 }, { "category": "mishear-severe", "points": 5 }, { "category": "content-drop", "points": 3 }, { "category": "mishear-severe", "points": 5 }, { "category": "content-drop", "points": 3 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 72, "duration_s": 54, "mean_score": 42.0, "mean_raw": 58.0, "scores_per_judge": [ 42 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "polarity", "points": 35 }, { "category": "tech", "points": 8 }, { "category": "content-drop", "points": 3 }, { "category": "content-drop", "points": 3 }, { "category": "content-drop", "points": 3 }, { "category": "mishear-minor", "points": 2 }, { "category": "filler", "points": 4 } ] }, "judge_disagreement": 0 }, { "idx": 73, "duration_s": 56, "mean_score": 59.0, "mean_raw": 41.0, "scores_per_judge": [ 59 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-severe", "points": 5 }, { "category": "brand", "points": 10 }, { "category": "paraphrase", "points": 3 }, { "category": "mishear-minor", "points": 2 }, { "category": "mishear-minor", "points": 2 }, { "category": "mishear-minor", "points": 2 }, { "category": "mishear-severe", "points": 5 }, { "category": "proper-noun", "points": 6 } ] }, "judge_disagreement": 0 }, { "idx": 74, "duration_s": 45, "mean_score": 75.0, "mean_raw": 25.0, "scores_per_judge": [ 75 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "hallucination", "points": 15 }, { "category": "content-drop", "points": 3 }, { "category": "content-drop", "points": 3 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 75, "duration_s": 74, "mean_score": 78.0, "mean_raw": 22.0, "scores_per_judge": [ 78 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "content-drop", "points": 3 }, { "category": "culture", "points": 8 }, { "category": "mishear-severe", "points": 5 }, { "category": "content-drop", "points": 3 }, { "category": "content-drop", "points": 3 } ] }, "judge_disagreement": 0 }, { "idx": 76, "duration_s": 58, "mean_score": 47.0, "mean_raw": 53.0, "scores_per_judge": [ 47 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "hallucination", "points": 15 }, { "category": "culture", "points": 8 }, { "category": "semantic-inversion", "points": 20 }, { "category": "content-drop", "points": 3 }, { "category": "content-drop", "points": 3 }, { "category": "content-drop", "points": 3 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 77, "duration_s": 130, "mean_score": 72.0, "mean_raw": 28.0, "scores_per_judge": [ 72 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "hallucination", "points": 15 }, { "category": "content-drop", "points": 3 }, { "category": "content-drop", "points": 3 }, { "category": "content-drop", "points": 3 }, { "category": "content-drop", "points": 3 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 78, "duration_s": 61, "mean_score": 50.0, "mean_raw": 50.0, "scores_per_judge": [ 50 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "hallucination", "points": 15 }, { "category": "hallucination", "points": 15 }, { "category": "content-drop", "points": 3 }, { "category": "content-drop", "points": 3 }, { "category": "content-drop", "points": 3 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 79, "duration_s": 47, "mean_score": 56.0, "mean_raw": 44.0, "scores_per_judge": [ 56 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "polarity", "points": 35 }, { "category": "content-drop", "points": 3 }, { "category": "content-drop", "points": 3 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 80, "duration_s": 42, "mean_score": 82.0, "mean_raw": 18.0, "scores_per_judge": [ 82 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "hallucination", "points": 15 }, { "category": "content-drop", "points": 3 } ] }, "judge_disagreement": 0 }, { "idx": 81, "duration_s": 38, "mean_score": 54.0, "mean_raw": 46.0, "scores_per_judge": [ 54 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "polarity", "points": 35 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-minor", "points": 2 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 82, "duration_s": 38, "mean_score": 59.0, "mean_raw": 41.0, "scores_per_judge": [ 59 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "hallucination", "points": 15 }, { "category": "numeric", "points": 10 }, { "category": "content-drop", "points": 3 }, { "category": "content-drop", "points": 3 }, { "category": "content-drop", "points": 3 }, { "category": "paraphrase", "points": 3 }, { "category": "filler", "points": 2 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 83, "duration_s": 37, "mean_score": 84.0, "mean_raw": 16.0, "scores_per_judge": [ 84 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-minor", "points": 2 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-severe", "points": 5 }, { "category": "content-drop", "points": 3 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 84, "duration_s": 36, "mean_score": 92.0, "mean_raw": 8.0, "scores_per_judge": [ 92 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 3 } ] }, "judge_disagreement": 0 }, { "idx": 85, "duration_s": 36, "mean_score": 82.0, "mean_raw": 18.0, "scores_per_judge": [ 82 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 }, { "category": "mishear-minor", "points": 2 }, { "category": "content-drop", "points": 3 }, { "category": "content-drop", "points": 3 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 86, "duration_s": 35, "mean_score": 79.0, "mean_raw": 21.0, "scores_per_judge": [ 79 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "tech", "points": 8 }, { "category": "mishear-severe", "points": 5 }, { "category": "content-drop", "points": 3 }, { "category": "content-drop", "points": 3 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 87, "duration_s": 34, "mean_score": 80.0, "mean_raw": 20.0, "scores_per_judge": [ 80 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "numeric", "points": 10 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-minor", "points": 2 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 88, "duration_s": 34, "mean_score": 96.0, "mean_raw": 4.0, "scores_per_judge": [ 96 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 89, "duration_s": 34, "mean_score": 76.0, "mean_raw": 24.0, "scores_per_judge": [ 76 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "hallucination", "points": 15 }, { "category": "content-drop", "points": 3 }, { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 90, "duration_s": 33, "mean_score": 89.0, "mean_raw": 11.0, "scores_per_judge": [ 89 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "tech", "points": 8 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 91, "duration_s": 33, "mean_score": 93.0, "mean_raw": 7.0, "scores_per_judge": [ 93 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "mishear-minor", "points": 2 } ] }, "judge_disagreement": 0 }, { "idx": 92, "duration_s": 32, "mean_score": 95.0, "mean_raw": 5.0, "scores_per_judge": [ 95 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 93, "duration_s": 32, "mean_score": 100.0, "mean_raw": 0.0, "scores_per_judge": [ 100 ], "deductions_per_judge": { "deepseek_flash": [] }, "judge_disagreement": 0 }, { "idx": 94, "duration_s": 32, "mean_score": 92.0, "mean_raw": 8.0, "scores_per_judge": [ 92 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 95, "duration_s": 32, "mean_score": 95.0, "mean_raw": 5.0, "scores_per_judge": [ 95 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "paraphrase", "points": 3 }, { "category": "filler", "points": 2 } ] }, "judge_disagreement": 0 }, { "idx": 96, "duration_s": 31, "mean_score": 100.0, "mean_raw": 0.0, "scores_per_judge": [ 100 ], "deductions_per_judge": { "deepseek_flash": [] }, "judge_disagreement": 0 }, { "idx": 97, "duration_s": 30, "mean_score": 95.0, "mean_raw": 5.0, "scores_per_judge": [ 95 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 98, "duration_s": 30, "mean_score": 99.0, "mean_raw": 1.0, "scores_per_judge": [ 99 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 99, "duration_s": 30, "mean_score": 100.0, "mean_raw": 0.0, "scores_per_judge": [ 100 ], "deductions_per_judge": { "deepseek_flash": [] }, "judge_disagreement": 0 }, { "idx": 100, "duration_s": 30, "mean_score": 94.0, "mean_raw": 6.0, "scores_per_judge": [ 94 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-minor", "points": 2 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 } ], "total_deductions": 2317, "avg_deductions_per_sample": 23.64, "mean_clamped_score": 76.4, "n_judge_sample_pairs": 98, "gt_chars": 21434, "ded_per_100char": 10.81, "fidelity_score": 46.0, "overall_score": 76.4, "overall_raw_deductions": 23.64, "score_per_judge": { "deepseek_flash": 76.4 }, "per_group_deductions": { "meaning-flip": { "deepseek_flash": 920 }, "entity": { "deepseek_flash": 336 }, "cultural": { "deepseek_flash": 114 }, "fidelity": { "deepseek_flash": 847 } }, "per_bucket": { "general": { "n_samples": 38, "mean_raw": 26.4, "mean_score": 73.6, "total_raw": 1004.0, "idxs": [ 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 38, 39, 40 ], "worst_idx": 11 }, "idiom-test": { "n_samples": 5, "mean_raw": 35.6, "mean_score": 64.4, "total_raw": 178.0, "idxs": [ 41, 42, 43, 44, 45 ], "worst_idx": 45 }, "long-mixed": { "n_samples": 5, "mean_raw": 45.0, "mean_score": 55.0, "total_raw": 225.0, "idxs": [ 46, 47, 48, 49, 50 ], "worst_idx": 47 } }, "per_group_per_100char": { "meaning-flip": 4.292, "entity": 1.568, "cultural": 0.532, "fidelity": 3.952 }, "per_group_fidelity": { "meaning-flip": 0.0, "entity": 72.2, "cultural": 67.8, "fidelity": 53.7 } }, "qwen3_oss": { "scores_by_judge": { "deepseek_flash": [ 100, 99, 100, 100, 97, 100, 99, 100, 96, 99, 83, 89, 59, 80, 98, 74, 77, 80, 63, 80, null, 94, 100, 90, 92, 100, 100, 99, 89, 56, 99, 93, 75, 78, 67, 100, 98, 76, 71, 13, 100, 79, 92, 89, 82, 88, 88, 90, 85, 80, 98, 96, 98, 99, 89, 89, 98, 98, 95, 44, 89, 100, 98, 95, 100, 91, 91, 89, 100, 56, 77, 91, 97, 93, 95, 68, 52, 96, 94, 97, 92, 97, 94, 96, 97, 81, 91, 100, 100, 93, 99, 100, 96, 92, 93, 100, 97, 98, 100, 95 ] }, "per_category_deductions": { "polarity": { "deepseek_flash": 105 }, "semantic-inversion": { "deepseek_flash": 120 }, "hallucination": { "deepseek_flash": 0 }, "numeric": { "deepseek_flash": 20 }, "brand": { "deepseek_flash": 140 }, "tech": { "deepseek_flash": 160 }, "culture": { "deepseek_flash": 72 }, "proper-noun": { "deepseek_flash": 42 }, "dialect": { "deepseek_flash": 15 }, "mishear-severe": { "deepseek_flash": 155 }, "content-drop": { "deepseek_flash": 6 }, "paraphrase": { "deepseek_flash": 15 }, "mishear-minor": { "deepseek_flash": 30 }, "filler": { "deepseek_flash": 199 }, "format": { "deepseek_flash": 1 }, "unscoreable": { "deepseek_flash": 0 } }, "per_tier_deductions": { "tier1": { "deepseek_flash": 245 }, "tier2": { "deepseek_flash": 429 }, "tier3": { "deepseek_flash": 206 }, "tier4": { "deepseek_flash": 200 }, "unscoreable": { "deepseek_flash": 0 } }, "raw_deductions_by_judge": { "deepseek_flash": [ 0, 1, 0, 0, 3, 0, 1, 0, 4, 1, 17, 11, 41, 20, 2, 26, 23, 20, 37, 20, null, 6, 0, 10, 8, 0, 0, 1, 11, 44, 1, 7, 25, 22, 33, 0, 2, 24, 29, 87, 0, 21, 8, 11, 18, 12, 12, 10, 15, 20, 2, 4, 2, 1, 11, 11, 2, 2, 5, 56, 11, 0, 2, 5, 0, 9, 9, 11, 0, 44, 23, 9, 3, 7, 5, 32, 48, 4, 6, 3, 8, 3, 6, 4, 3, 19, 9, 0, 0, 7, 1, 0, 4, 8, 7, 0, 3, 2, 0, 5 ] }, "sample_details": [ { "idx": 1, "duration_s": 5, "mean_score": 100.0, "mean_raw": 0.0, "scores_per_judge": [ 100 ], "deductions_per_judge": { "deepseek_flash": [] }, "judge_disagreement": 0 }, { "idx": 2, "duration_s": 7, "mean_score": 99.0, "mean_raw": 1.0, "scores_per_judge": [ 99 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 3, "duration_s": 8, "mean_score": 100.0, "mean_raw": 0.0, "scores_per_judge": [ 100 ], "deductions_per_judge": { "deepseek_flash": [] }, "judge_disagreement": 0 }, { "idx": 4, "duration_s": 11, "mean_score": 100.0, "mean_raw": 0.0, "scores_per_judge": [ 100 ], "deductions_per_judge": { "deepseek_flash": [] }, "judge_disagreement": 0 }, { "idx": 5, "duration_s": 16, "mean_score": 97.0, "mean_raw": 3.0, "scores_per_judge": [ 97 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 6, "duration_s": 17, "mean_score": 100.0, "mean_raw": 0.0, "scores_per_judge": [ 100 ], "deductions_per_judge": { "deepseek_flash": [] }, "judge_disagreement": 0 }, { "idx": 7, "duration_s": 20, "mean_score": 99.0, "mean_raw": 1.0, "scores_per_judge": [ 99 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 8, "duration_s": 30, "mean_score": 100.0, "mean_raw": 0.0, "scores_per_judge": [ 100 ], "deductions_per_judge": { "deepseek_flash": [] }, "judge_disagreement": 0 }, { "idx": 9, "duration_s": 32, "mean_score": 96.0, "mean_raw": 4.0, "scores_per_judge": [ 96 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "paraphrase", "points": 3 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 10, "duration_s": 35, "mean_score": 99.0, "mean_raw": 1.0, "scores_per_judge": [ 99 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 11, "duration_s": 40, "mean_score": 83.0, "mean_raw": 17.0, "scores_per_judge": [ 83 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "tech", "points": 8 }, { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 12, "duration_s": 46, "mean_score": 89.0, "mean_raw": 11.0, "scores_per_judge": [ 89 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 }, { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 13, "duration_s": 60, "mean_score": 59.0, "mean_raw": 41.0, "scores_per_judge": [ 59 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "brand", "points": 10 }, { "category": "brand", "points": 10 }, { "category": "tech", "points": 8 }, { "category": "tech", "points": 8 }, { "category": "mishear-minor", "points": 2 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 14, "duration_s": 64, "mean_score": 80.0, "mean_raw": 20.0, "scores_per_judge": [ 80 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 }, { "category": "proper-noun", "points": 6 }, { "category": "proper-noun", "points": 6 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 15, "duration_s": 69, "mean_score": 98.0, "mean_raw": 2.0, "scores_per_judge": [ 98 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 16, "duration_s": 81, "mean_score": 74.0, "mean_raw": 26.0, "scores_per_judge": [ 74 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "proper-noun", "points": 6 }, { "category": "numeric", "points": 10 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-minor", "points": 2 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 17, "duration_s": 100, "mean_score": 77.0, "mean_raw": 23.0, "scores_per_judge": [ 77 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "brand", "points": 10 }, { "category": "tech", "points": 8 }, { "category": "filler", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 18, "duration_s": 125, "mean_score": 80.0, "mean_raw": 20.0, "scores_per_judge": [ 80 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "brand", "points": 10 }, { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 19, "duration_s": 133, "mean_score": 63.0, "mean_raw": 37.0, "scores_per_judge": [ 63 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "brand", "points": 10 }, { "category": "tech", "points": 8 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 2 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 20, "duration_s": 164, "mean_score": 80.0, "mean_raw": 20.0, "scores_per_judge": [ 80 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "proper-noun", "points": 6 }, { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 3 }, { "category": "mishear-minor", "points": 2 }, { "category": "mishear-minor", "points": 2 }, { "category": "mishear-minor", "points": 2 } ] }, "judge_disagreement": 0 }, { "idx": 22, "duration_s": 7, "mean_score": 94.0, "mean_raw": 6.0, "scores_per_judge": [ 94 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "proper-noun", "points": 6 } ] }, "judge_disagreement": 0 }, { "idx": 23, "duration_s": 10, "mean_score": 100.0, "mean_raw": 0.0, "scores_per_judge": [ 100 ], "deductions_per_judge": { "deepseek_flash": [] }, "judge_disagreement": 0 }, { "idx": 24, "duration_s": 14, "mean_score": 90.0, "mean_raw": 10.0, "scores_per_judge": [ 90 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "tech", "points": 8 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 25, "duration_s": 16, "mean_score": 92.0, "mean_raw": 8.0, "scores_per_judge": [ 92 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "tech", "points": 8 } ] }, "judge_disagreement": 0 }, { "idx": 26, "duration_s": 23, "mean_score": 100.0, "mean_raw": 0.0, "scores_per_judge": [ 100 ], "deductions_per_judge": { "deepseek_flash": [] }, "judge_disagreement": 0 }, { "idx": 27, "duration_s": 25, "mean_score": 100.0, "mean_raw": 0.0, "scores_per_judge": [ 100 ], "deductions_per_judge": { "deepseek_flash": [] }, "judge_disagreement": 0 }, { "idx": 28, "duration_s": 30, "mean_score": 99.0, "mean_raw": 1.0, "scores_per_judge": [ 99 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 29, "duration_s": 34, "mean_score": 89.0, "mean_raw": 11.0, "scores_per_judge": [ 89 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "brand", "points": 10 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 30, "duration_s": 38, "mean_score": 56.0, "mean_raw": 44.0, "scores_per_judge": [ 56 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "polarity", "points": 35 }, { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 4 } ] }, "judge_disagreement": 0 }, { "idx": 31, "duration_s": 44, "mean_score": 99.0, "mean_raw": 1.0, "scores_per_judge": [ 99 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 32, "duration_s": 50, "mean_score": 93.0, "mean_raw": 7.0, "scores_per_judge": [ 93 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 2 } ] }, "judge_disagreement": 0 }, { "idx": 33, "duration_s": 62, "mean_score": 75.0, "mean_raw": 25.0, "scores_per_judge": [ 75 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "brand", "points": 10 }, { "category": "proper-noun", "points": 6 }, { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 34, "duration_s": 66, "mean_score": 78.0, "mean_raw": 22.0, "scores_per_judge": [ 78 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "semantic-inversion", "points": 20 }, { "category": "filler", "points": 2 } ] }, "judge_disagreement": 0 }, { "idx": 35, "duration_s": 77, "mean_score": 67.0, "mean_raw": 33.0, "scores_per_judge": [ 67 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "tech", "points": 8 }, { "category": "tech", "points": 8 }, { "category": "tech", "points": 8 }, { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 36, "duration_s": 94, "mean_score": 100.0, "mean_raw": 0.0, "scores_per_judge": [ 100 ], "deductions_per_judge": { "deepseek_flash": [] }, "judge_disagreement": 0 }, { "idx": 37, "duration_s": 106, "mean_score": 98.0, "mean_raw": 2.0, "scores_per_judge": [ 98 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 38, "duration_s": 126, "mean_score": 76.0, "mean_raw": 24.0, "scores_per_judge": [ 76 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "tech", "points": 8 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-minor", "points": 2 }, { "category": "mishear-minor", "points": 2 }, { "category": "filler", "points": 2 } ] }, "judge_disagreement": 0 }, { "idx": 39, "duration_s": 158, "mean_score": 71.0, "mean_raw": 29.0, "scores_per_judge": [ 71 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "brand", "points": 10 }, { "category": "tech", "points": 8 }, { "category": "tech", "points": 8 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 40, "duration_s": 279, "mean_score": 13.0, "mean_raw": 87.0, "scores_per_judge": [ 13 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "semantic-inversion", "points": 20 }, { "category": "numeric", "points": 10 }, { "category": "culture", "points": 8 }, { "category": "culture", "points": 8 }, { "category": "tech", "points": 8 }, { "category": "dialect", "points": 5 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-severe", "points": 5 }, { "category": "content-drop", "points": 3 } ] }, "judge_disagreement": 0 }, { "idx": 41, "duration_s": 34, "mean_score": 100.0, "mean_raw": 0.0, "scores_per_judge": [ 100 ], "deductions_per_judge": { "deepseek_flash": [] }, "judge_disagreement": 0 }, { "idx": 42, "duration_s": 43, "mean_score": 79.0, "mean_raw": 21.0, "scores_per_judge": [ 79 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "semantic-inversion", "points": 20 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 43, "duration_s": 29, "mean_score": 92.0, "mean_raw": 8.0, "scores_per_judge": [ 92 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "culture", "points": 8 } ] }, "judge_disagreement": 0 }, { "idx": 44, "duration_s": 71, "mean_score": 89.0, "mean_raw": 11.0, "scores_per_judge": [ 89 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "culture", "points": 8 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 45, "duration_s": 46, "mean_score": 82.0, "mean_raw": 18.0, "scores_per_judge": [ 82 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-minor", "points": 2 }, { "category": "dialect", "points": 5 }, { "category": "filler", "points": 1 }, { "category": "dialect", "points": 5 }, { "category": "mishear-severe", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 46, "duration_s": 62, "mean_score": 88.0, "mean_raw": 12.0, "scores_per_judge": [ 88 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "culture", "points": 8 }, { "category": "mishear-minor", "points": 2 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 47, "duration_s": 67, "mean_score": 88.0, "mean_raw": 12.0, "scores_per_judge": [ 88 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "brand", "points": 10 }, { "category": "mishear-minor", "points": 2 } ] }, "judge_disagreement": 0 }, { "idx": 48, "duration_s": 72, "mean_score": 90.0, "mean_raw": 10.0, "scores_per_judge": [ 90 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "tech", "points": 8 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 49, "duration_s": 87, "mean_score": 85.0, "mean_raw": 15.0, "scores_per_judge": [ 85 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "brand", "points": 10 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 50, "duration_s": 106, "mean_score": 80.0, "mean_raw": 20.0, "scores_per_judge": [ 80 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "culture", "points": 8 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 51, "duration_s": 33, "mean_score": 98.0, "mean_raw": 2.0, "scores_per_judge": [ 98 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 52, "duration_s": 25, "mean_score": 96.0, "mean_raw": 4.0, "scores_per_judge": [ 96 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 53, "duration_s": 35, "mean_score": 98.0, "mean_raw": 2.0, "scores_per_judge": [ 98 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 54, "duration_s": 36, "mean_score": 99.0, "mean_raw": 1.0, "scores_per_judge": [ 99 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 55, "duration_s": 33, "mean_score": 89.0, "mean_raw": 11.0, "scores_per_judge": [ 89 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "proper-noun", "points": 6 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 56, "duration_s": 15, "mean_score": 89.0, "mean_raw": 11.0, "scores_per_judge": [ 89 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "brand", "points": 10 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 57, "duration_s": 38, "mean_score": 98.0, "mean_raw": 2.0, "scores_per_judge": [ 98 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 58, "duration_s": 15, "mean_score": 98.0, "mean_raw": 2.0, "scores_per_judge": [ 98 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 59, "duration_s": 28, "mean_score": 95.0, "mean_raw": 5.0, "scores_per_judge": [ 95 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-minor", "points": 2 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 60, "duration_s": 18, "mean_score": 44.0, "mean_raw": 56.0, "scores_per_judge": [ 44 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "polarity", "points": 35 }, { "category": "semantic-inversion", "points": 20 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 61, "duration_s": 19, "mean_score": 89.0, "mean_raw": 11.0, "scores_per_judge": [ 89 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "brand", "points": 10 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 62, "duration_s": 17, "mean_score": 100.0, "mean_raw": 0.0, "scores_per_judge": [ 100 ], "deductions_per_judge": { "deepseek_flash": [] }, "judge_disagreement": 0 }, { "idx": 63, "duration_s": 21, "mean_score": 98.0, "mean_raw": 2.0, "scores_per_judge": [ 98 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 2 } ] }, "judge_disagreement": 0 }, { "idx": 64, "duration_s": 23, "mean_score": 95.0, "mean_raw": 5.0, "scores_per_judge": [ 95 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 65, "duration_s": 20, "mean_score": 100.0, "mean_raw": 0.0, "scores_per_judge": [ 100 ], "deductions_per_judge": { "deepseek_flash": [] }, "judge_disagreement": 0 }, { "idx": 66, "duration_s": 24, "mean_score": 91.0, "mean_raw": 9.0, "scores_per_judge": [ 91 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "culture", "points": 8 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 67, "duration_s": 16, "mean_score": 91.0, "mean_raw": 9.0, "scores_per_judge": [ 91 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "culture", "points": 8 } ] }, "judge_disagreement": 0 }, { "idx": 68, "duration_s": 18, "mean_score": 89.0, "mean_raw": 11.0, "scores_per_judge": [ 89 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "brand", "points": 10 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 69, "duration_s": 18, "mean_score": 100.0, "mean_raw": 0.0, "scores_per_judge": [ 100 ], "deductions_per_judge": { "deepseek_flash": [] }, "judge_disagreement": 0 }, { "idx": 70, "duration_s": 17, "mean_score": 56.0, "mean_raw": 44.0, "scores_per_judge": [ 56 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "polarity", "points": 35 }, { "category": "tech", "points": 8 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 71, "duration_s": 51, "mean_score": 77.0, "mean_raw": 23.0, "scores_per_judge": [ 77 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 }, { "category": "tech", "points": 8 }, { "category": "mishear-minor", "points": 2 }, { "category": "content-drop", "points": 3 }, { "category": "filler", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 72, "duration_s": 54, "mean_score": 91.0, "mean_raw": 9.0, "scores_per_judge": [ 91 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "tech", "points": 8 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 73, "duration_s": 56, "mean_score": 97.0, "mean_raw": 3.0, "scores_per_judge": [ 97 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "paraphrase", "points": 3 } ] }, "judge_disagreement": 0 }, { "idx": 74, "duration_s": 45, "mean_score": 93.0, "mean_raw": 7.0, "scores_per_judge": [ 93 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-minor", "points": 2 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 75, "duration_s": 74, "mean_score": 95.0, "mean_raw": 5.0, "scores_per_judge": [ 95 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 76, "duration_s": 58, "mean_score": 68.0, "mean_raw": 32.0, "scores_per_judge": [ 68 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "semantic-inversion", "points": 20 }, { "category": "culture", "points": 8 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 77, "duration_s": 130, "mean_score": 52.0, "mean_raw": 48.0, "scores_per_judge": [ 52 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "semantic-inversion", "points": 20 }, { "category": "brand", "points": 10 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-severe", "points": 5 }, { "category": "paraphrase", "points": 3 }, { "category": "filler", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 78, "duration_s": 61, "mean_score": 96.0, "mean_raw": 4.0, "scores_per_judge": [ 96 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 79, "duration_s": 47, "mean_score": 94.0, "mean_raw": 6.0, "scores_per_judge": [ 94 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "paraphrase", "points": 3 } ] }, "judge_disagreement": 0 }, { "idx": 80, "duration_s": 42, "mean_score": 97.0, "mean_raw": 3.0, "scores_per_judge": [ 97 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 81, "duration_s": 38, "mean_score": 92.0, "mean_raw": 8.0, "scores_per_judge": [ 92 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 2 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 82, "duration_s": 38, "mean_score": 97.0, "mean_raw": 3.0, "scores_per_judge": [ 97 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "format", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 83, "duration_s": 37, "mean_score": 94.0, "mean_raw": 6.0, "scores_per_judge": [ 94 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-minor", "points": 2 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 84, "duration_s": 36, "mean_score": 96.0, "mean_raw": 4.0, "scores_per_judge": [ 96 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 85, "duration_s": 36, "mean_score": 97.0, "mean_raw": 3.0, "scores_per_judge": [ 97 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 86, "duration_s": 35, "mean_score": 81.0, "mean_raw": 19.0, "scores_per_judge": [ 81 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "tech", "points": 8 }, { "category": "tech", "points": 8 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 87, "duration_s": 34, "mean_score": 91.0, "mean_raw": 9.0, "scores_per_judge": [ 91 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 88, "duration_s": 34, "mean_score": 100.0, "mean_raw": 0.0, "scores_per_judge": [ 100 ], "deductions_per_judge": { "deepseek_flash": [] }, "judge_disagreement": 0 }, { "idx": 89, "duration_s": 34, "mean_score": 100.0, "mean_raw": 0.0, "scores_per_judge": [ 100 ], "deductions_per_judge": { "deepseek_flash": [] }, "judge_disagreement": 0 }, { "idx": 90, "duration_s": 33, "mean_score": 93.0, "mean_raw": 7.0, "scores_per_judge": [ 93 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 2 } ] }, "judge_disagreement": 0 }, { "idx": 91, "duration_s": 33, "mean_score": 99.0, "mean_raw": 1.0, "scores_per_judge": [ 99 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 92, "duration_s": 32, "mean_score": 100.0, "mean_raw": 0.0, "scores_per_judge": [ 100 ], "deductions_per_judge": { "deepseek_flash": [] }, "judge_disagreement": 0 }, { "idx": 93, "duration_s": 32, "mean_score": 96.0, "mean_raw": 4.0, "scores_per_judge": [ 96 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-minor", "points": 2 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 94, "duration_s": 32, "mean_score": 92.0, "mean_raw": 8.0, "scores_per_judge": [ 92 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 95, "duration_s": 32, "mean_score": 93.0, "mean_raw": 7.0, "scores_per_judge": [ 93 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "paraphrase", "points": 3 }, { "category": "filler", "points": 4 } ] }, "judge_disagreement": 0 }, { "idx": 96, "duration_s": 31, "mean_score": 100.0, "mean_raw": 0.0, "scores_per_judge": [ 100 ], "deductions_per_judge": { "deepseek_flash": [] }, "judge_disagreement": 0 }, { "idx": 97, "duration_s": 30, "mean_score": 97.0, "mean_raw": 3.0, "scores_per_judge": [ 97 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 98, "duration_s": 30, "mean_score": 98.0, "mean_raw": 2.0, "scores_per_judge": [ 98 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 99, "duration_s": 30, "mean_score": 100.0, "mean_raw": 0.0, "scores_per_judge": [ 100 ], "deductions_per_judge": { "deepseek_flash": [] }, "judge_disagreement": 0 }, { "idx": 100, "duration_s": 30, "mean_score": 95.0, "mean_raw": 5.0, "scores_per_judge": [ 95 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 } ], "total_deductions": 1080, "avg_deductions_per_sample": 10.91, "mean_clamped_score": 89.1, "n_judge_sample_pairs": 99, "gt_chars": 22592, "ded_per_100char": 4.78, "fidelity_score": 76.1, "overall_score": 89.1, "overall_raw_deductions": 10.91, "score_per_judge": { "deepseek_flash": 89.1 }, "per_group_deductions": { "meaning-flip": { "deepseek_flash": 245 }, "entity": { "deepseek_flash": 342 }, "cultural": { "deepseek_flash": 87 }, "fidelity": { "deepseek_flash": 406 } }, "per_bucket": { "general": { "n_samples": 39, "mean_raw": 13.8, "mean_score": 86.2, "total_raw": 537.0, "idxs": [ 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40 ], "worst_idx": 40 }, "idiom-test": { "n_samples": 5, "mean_raw": 11.6, "mean_score": 88.4, "total_raw": 58.0, "idxs": [ 41, 42, 43, 44, 45 ], "worst_idx": 42 }, "long-mixed": { "n_samples": 5, "mean_raw": 13.8, "mean_score": 86.2, "total_raw": 69.0, "idxs": [ 46, 47, 48, 49, 50 ], "worst_idx": 50 } }, "per_group_per_100char": { "meaning-flip": 1.084, "entity": 1.514, "cultural": 0.385, "fidelity": 1.797 }, "per_group_fidelity": { "meaning-flip": 74.0, "entity": 73.2, "cultural": 76.7, "fidelity": 78.9 } }, "qwen_audio_3_1_asr": { "scores_by_judge": { "deepseek_flash": [ 100, 100, 100, 100, 98, 100, 100, 99, 98, 99, 81, 96, 71, 71, 98, 82, 100, 88, 54, 97, 100, 94, 100, 90, 100, 100, 100, 99, 100, 95, 64, 98, 89, 96, 94, 98, 96, 83, 94, 91, 98, 100, 91, 89, 95, 98, 99, 88, 90, 92, 94, 98, 94, 98, 93, 86, 99, 98, 97, 64, 100, 100, 97, 97, 99, 91, 92, 99, 100, 56, 96, 98, 99, 96, 92, 77, 60, 93, 96, 97, 94, 90, 97, 64, 89, 91, 85, 99, 100, 98, 99, 98, 98, 93, 94, 100, 97, 99, 100, 98 ] }, "per_category_deductions": { "polarity": { "deepseek_flash": 140 }, "semantic-inversion": { "deepseek_flash": 60 }, "hallucination": { "deepseek_flash": 0 }, "numeric": { "deepseek_flash": 10 }, "brand": { "deepseek_flash": 70 }, "tech": { "deepseek_flash": 88 }, "culture": { "deepseek_flash": 40 }, "proper-noun": { "deepseek_flash": 24 }, "dialect": { "deepseek_flash": 0 }, "mishear-severe": { "deepseek_flash": 60 }, "content-drop": { "deepseek_flash": 6 }, "paraphrase": { "deepseek_flash": 18 }, "mishear-minor": { "deepseek_flash": 16 }, "filler": { "deepseek_flash": 175 }, "format": { "deepseek_flash": 1 }, "unscoreable": { "deepseek_flash": 0 } }, "per_tier_deductions": { "tier1": { "deepseek_flash": 210 }, "tier2": { "deepseek_flash": 222 }, "tier3": { "deepseek_flash": 100 }, "tier4": { "deepseek_flash": 176 }, "unscoreable": { "deepseek_flash": 0 } }, "raw_deductions_by_judge": { "deepseek_flash": [ 0, 0, 0, 0, 2, 0, 0, 1, 2, 1, 19, 4, 29, 29, 2, 18, 0, 12, 46, 3, 0, 6, 0, 10, 0, 0, 0, 1, 0, 5, 36, 2, 11, 4, 6, 2, 4, 17, 6, 9, 2, 0, 9, 11, 5, 2, 1, 12, 10, 8, 6, 2, 6, 2, 7, 14, 1, 2, 3, 36, 0, 0, 3, 3, 1, 9, 8, 1, 0, 44, 4, 2, 1, 4, 8, 23, 40, 7, 4, 3, 6, 10, 3, 36, 11, 9, 15, 1, 0, 2, 1, 2, 2, 7, 6, 0, 3, 1, 0, 2 ] }, "sample_details": [ { "idx": 1, "duration_s": 5, "mean_score": 100.0, "mean_raw": 0.0, "scores_per_judge": [ 100 ], "deductions_per_judge": { "deepseek_flash": [] }, "judge_disagreement": 0 }, { "idx": 2, "duration_s": 7, "mean_score": 100.0, "mean_raw": 0.0, "scores_per_judge": [ 100 ], "deductions_per_judge": { "deepseek_flash": [] }, "judge_disagreement": 0 }, { "idx": 3, "duration_s": 8, "mean_score": 100.0, "mean_raw": 0.0, "scores_per_judge": [ 100 ], "deductions_per_judge": { "deepseek_flash": [] }, "judge_disagreement": 0 }, { "idx": 4, "duration_s": 11, "mean_score": 100.0, "mean_raw": 0.0, "scores_per_judge": [ 100 ], "deductions_per_judge": { "deepseek_flash": [] }, "judge_disagreement": 0 }, { "idx": 5, "duration_s": 16, "mean_score": 98.0, "mean_raw": 2.0, "scores_per_judge": [ 98 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 2 } ] }, "judge_disagreement": 0 }, { "idx": 6, "duration_s": 17, "mean_score": 100.0, "mean_raw": 0.0, "scores_per_judge": [ 100 ], "deductions_per_judge": { "deepseek_flash": [] }, "judge_disagreement": 0 }, { "idx": 7, "duration_s": 20, "mean_score": 100.0, "mean_raw": 0.0, "scores_per_judge": [ 100 ], "deductions_per_judge": { "deepseek_flash": [] }, "judge_disagreement": 0 }, { "idx": 8, "duration_s": 30, "mean_score": 99.0, "mean_raw": 1.0, "scores_per_judge": [ 99 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 9, "duration_s": 32, "mean_score": 98.0, "mean_raw": 2.0, "scores_per_judge": [ 98 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 2 } ] }, "judge_disagreement": 0 }, { "idx": 10, "duration_s": 35, "mean_score": 99.0, "mean_raw": 1.0, "scores_per_judge": [ 99 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 11, "duration_s": 40, "mean_score": 81.0, "mean_raw": 19.0, "scores_per_judge": [ 81 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "tech", "points": 8 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-minor", "points": 2 }, { "category": "filler", "points": 4 } ] }, "judge_disagreement": 0 }, { "idx": 12, "duration_s": 46, "mean_score": 96.0, "mean_raw": 4.0, "scores_per_judge": [ 96 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 4 } ] }, "judge_disagreement": 0 }, { "idx": 13, "duration_s": 60, "mean_score": 71.0, "mean_raw": 29.0, "scores_per_judge": [ 71 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "brand", "points": 10 }, { "category": "brand", "points": 10 }, { "category": "tech", "points": 8 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 14, "duration_s": 64, "mean_score": 71.0, "mean_raw": 29.0, "scores_per_judge": [ 71 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "semantic-inversion", "points": 20 }, { "category": "proper-noun", "points": 6 }, { "category": "filler", "points": 3 } ] }, "judge_disagreement": 0 }, { "idx": 15, "duration_s": 69, "mean_score": 98.0, "mean_raw": 2.0, "scores_per_judge": [ 98 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 2 } ] }, "judge_disagreement": 0 }, { "idx": 16, "duration_s": 81, "mean_score": 82.0, "mean_raw": 18.0, "scores_per_judge": [ 82 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "proper-noun", "points": 6 }, { "category": "brand", "points": 10 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 17, "duration_s": 100, "mean_score": 100.0, "mean_raw": 0.0, "scores_per_judge": [ 100 ], "deductions_per_judge": { "deepseek_flash": [] }, "judge_disagreement": 0 }, { "idx": 18, "duration_s": 125, "mean_score": 88.0, "mean_raw": 12.0, "scores_per_judge": [ 88 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "brand", "points": 10 }, { "category": "mishear-minor", "points": 2 } ] }, "judge_disagreement": 0 }, { "idx": 19, "duration_s": 133, "mean_score": 54.0, "mean_raw": 46.0, "scores_per_judge": [ 54 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "brand", "points": 10 }, { "category": "tech", "points": 8 }, { "category": "tech", "points": 8 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 20, "duration_s": 164, "mean_score": 97.0, "mean_raw": 3.0, "scores_per_judge": [ 97 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 3 } ] }, "judge_disagreement": 0 }, { "idx": 21, "duration_s": 5, "mean_score": 100.0, "mean_raw": 0.0, "scores_per_judge": [ 100 ], "deductions_per_judge": { "deepseek_flash": [] }, "judge_disagreement": 0 }, { "idx": 22, "duration_s": 7, "mean_score": 94.0, "mean_raw": 6.0, "scores_per_judge": [ 94 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "proper-noun", "points": 6 } ] }, "judge_disagreement": 0 }, { "idx": 23, "duration_s": 10, "mean_score": 100.0, "mean_raw": 0.0, "scores_per_judge": [ 100 ], "deductions_per_judge": { "deepseek_flash": [] }, "judge_disagreement": 0 }, { "idx": 24, "duration_s": 14, "mean_score": 90.0, "mean_raw": 10.0, "scores_per_judge": [ 90 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "tech", "points": 8 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 25, "duration_s": 16, "mean_score": 100.0, "mean_raw": 0.0, "scores_per_judge": [ 100 ], "deductions_per_judge": { "deepseek_flash": [] }, "judge_disagreement": 0 }, { "idx": 26, "duration_s": 23, "mean_score": 100.0, "mean_raw": 0.0, "scores_per_judge": [ 100 ], "deductions_per_judge": { "deepseek_flash": [] }, "judge_disagreement": 0 }, { "idx": 27, "duration_s": 25, "mean_score": 100.0, "mean_raw": 0.0, "scores_per_judge": [ 100 ], "deductions_per_judge": { "deepseek_flash": [] }, "judge_disagreement": 0 }, { "idx": 28, "duration_s": 30, "mean_score": 99.0, "mean_raw": 1.0, "scores_per_judge": [ 99 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 29, "duration_s": 34, "mean_score": 100.0, "mean_raw": 0.0, "scores_per_judge": [ 100 ], "deductions_per_judge": { "deepseek_flash": [] }, "judge_disagreement": 0 }, { "idx": 30, "duration_s": 38, "mean_score": 95.0, "mean_raw": 5.0, "scores_per_judge": [ 95 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 31, "duration_s": 44, "mean_score": 64.0, "mean_raw": 36.0, "scores_per_judge": [ 64 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "polarity", "points": 35 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 32, "duration_s": 50, "mean_score": 98.0, "mean_raw": 2.0, "scores_per_judge": [ 98 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 33, "duration_s": 62, "mean_score": 89.0, "mean_raw": 11.0, "scores_per_judge": [ 89 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "proper-noun", "points": 6 }, { "category": "paraphrase", "points": 3 }, { "category": "filler", "points": 2 } ] }, "judge_disagreement": 0 }, { "idx": 34, "duration_s": 66, "mean_score": 96.0, "mean_raw": 4.0, "scores_per_judge": [ 96 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 4 } ] }, "judge_disagreement": 0 }, { "idx": 35, "duration_s": 77, "mean_score": 94.0, "mean_raw": 6.0, "scores_per_judge": [ 94 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 4 }, { "category": "mishear-minor", "points": 2 } ] }, "judge_disagreement": 0 }, { "idx": 36, "duration_s": 94, "mean_score": 98.0, "mean_raw": 2.0, "scores_per_judge": [ 98 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 2 } ] }, "judge_disagreement": 0 }, { "idx": 37, "duration_s": 106, "mean_score": 96.0, "mean_raw": 4.0, "scores_per_judge": [ 96 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 4 } ] }, "judge_disagreement": 0 }, { "idx": 38, "duration_s": 126, "mean_score": 83.0, "mean_raw": 17.0, "scores_per_judge": [ 83 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "tech", "points": 8 }, { "category": "mishear-minor", "points": 2 }, { "category": "mishear-minor", "points": 2 }, { "category": "filler", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 39, "duration_s": 158, "mean_score": 94.0, "mean_raw": 6.0, "scores_per_judge": [ 94 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 40, "duration_s": 279, "mean_score": 91.0, "mean_raw": 9.0, "scores_per_judge": [ 91 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "culture", "points": 8 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 41, "duration_s": 34, "mean_score": 98.0, "mean_raw": 2.0, "scores_per_judge": [ 98 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 2 } ] }, "judge_disagreement": 0 }, { "idx": 42, "duration_s": 43, "mean_score": 100.0, "mean_raw": 0.0, "scores_per_judge": [ 100 ], "deductions_per_judge": { "deepseek_flash": [] }, "judge_disagreement": 0 }, { "idx": 43, "duration_s": 29, "mean_score": 91.0, "mean_raw": 9.0, "scores_per_judge": [ 91 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "culture", "points": 8 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 44, "duration_s": 71, "mean_score": 89.0, "mean_raw": 11.0, "scores_per_judge": [ 89 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "culture", "points": 8 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 45, "duration_s": 46, "mean_score": 95.0, "mean_raw": 5.0, "scores_per_judge": [ 95 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "paraphrase", "points": 3 }, { "category": "filler", "points": 2 } ] }, "judge_disagreement": 0 }, { "idx": 46, "duration_s": 62, "mean_score": 98.0, "mean_raw": 2.0, "scores_per_judge": [ 98 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 2 } ] }, "judge_disagreement": 0 }, { "idx": 47, "duration_s": 67, "mean_score": 99.0, "mean_raw": 1.0, "scores_per_judge": [ 99 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 48, "duration_s": 72, "mean_score": 88.0, "mean_raw": 12.0, "scores_per_judge": [ 88 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "tech", "points": 8 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 49, "duration_s": 87, "mean_score": 90.0, "mean_raw": 10.0, "scores_per_judge": [ 90 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "tech", "points": 8 }, { "category": "filler", "points": 2 } ] }, "judge_disagreement": 0 }, { "idx": 50, "duration_s": 106, "mean_score": 92.0, "mean_raw": 8.0, "scores_per_judge": [ 92 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "culture", "points": 8 } ] }, "judge_disagreement": 0 }, { "idx": 51, "duration_s": 33, "mean_score": 94.0, "mean_raw": 6.0, "scores_per_judge": [ 94 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "content-drop", "points": 3 }, { "category": "filler", "points": 3 } ] }, "judge_disagreement": 0 }, { "idx": 52, "duration_s": 25, "mean_score": 98.0, "mean_raw": 2.0, "scores_per_judge": [ 98 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 2 } ] }, "judge_disagreement": 0 }, { "idx": 53, "duration_s": 35, "mean_score": 94.0, "mean_raw": 6.0, "scores_per_judge": [ 94 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 5 }, { "category": "format", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 54, "duration_s": 36, "mean_score": 98.0, "mean_raw": 2.0, "scores_per_judge": [ 98 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 55, "duration_s": 33, "mean_score": 93.0, "mean_raw": 7.0, "scores_per_judge": [ 93 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 4 }, { "category": "paraphrase", "points": 3 } ] }, "judge_disagreement": 0 }, { "idx": 56, "duration_s": 15, "mean_score": 86.0, "mean_raw": 14.0, "scores_per_judge": [ 86 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "brand", "points": 10 }, { "category": "content-drop", "points": 3 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 57, "duration_s": 38, "mean_score": 99.0, "mean_raw": 1.0, "scores_per_judge": [ 99 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 58, "duration_s": 15, "mean_score": 98.0, "mean_raw": 2.0, "scores_per_judge": [ 98 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 59, "duration_s": 28, "mean_score": 97.0, "mean_raw": 3.0, "scores_per_judge": [ 97 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "mishear-minor", "points": 2 } ] }, "judge_disagreement": 0 }, { "idx": 60, "duration_s": 18, "mean_score": 64.0, "mean_raw": 36.0, "scores_per_judge": [ 64 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "polarity", "points": 35 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 61, "duration_s": 19, "mean_score": 100.0, "mean_raw": 0.0, "scores_per_judge": [ 100 ], "deductions_per_judge": { "deepseek_flash": [] }, "judge_disagreement": 0 }, { "idx": 62, "duration_s": 17, "mean_score": 100.0, "mean_raw": 0.0, "scores_per_judge": [ 100 ], "deductions_per_judge": { "deepseek_flash": [] }, "judge_disagreement": 0 }, { "idx": 63, "duration_s": 21, "mean_score": 97.0, "mean_raw": 3.0, "scores_per_judge": [ 97 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 3 } ] }, "judge_disagreement": 0 }, { "idx": 64, "duration_s": 23, "mean_score": 97.0, "mean_raw": 3.0, "scores_per_judge": [ 97 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 3 } ] }, "judge_disagreement": 0 }, { "idx": 65, "duration_s": 20, "mean_score": 99.0, "mean_raw": 1.0, "scores_per_judge": [ 99 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 66, "duration_s": 24, "mean_score": 91.0, "mean_raw": 9.0, "scores_per_judge": [ 91 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "culture", "points": 8 } ] }, "judge_disagreement": 0 }, { "idx": 67, "duration_s": 16, "mean_score": 92.0, "mean_raw": 8.0, "scores_per_judge": [ 92 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "mishear-minor", "points": 2 }, { "category": "mishear-severe", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 68, "duration_s": 18, "mean_score": 99.0, "mean_raw": 1.0, "scores_per_judge": [ 99 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 69, "duration_s": 18, "mean_score": 100.0, "mean_raw": 0.0, "scores_per_judge": [ 100 ], "deductions_per_judge": { "deepseek_flash": [] }, "judge_disagreement": 0 }, { "idx": 70, "duration_s": 17, "mean_score": 56.0, "mean_raw": 44.0, "scores_per_judge": [ 56 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "polarity", "points": 35 }, { "category": "tech", "points": 8 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 71, "duration_s": 51, "mean_score": 96.0, "mean_raw": 4.0, "scores_per_judge": [ 96 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 4 } ] }, "judge_disagreement": 0 }, { "idx": 72, "duration_s": 54, "mean_score": 98.0, "mean_raw": 2.0, "scores_per_judge": [ 98 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 73, "duration_s": 56, "mean_score": 99.0, "mean_raw": 1.0, "scores_per_judge": [ 99 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 74, "duration_s": 45, "mean_score": 96.0, "mean_raw": 4.0, "scores_per_judge": [ 96 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "paraphrase", "points": 3 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 75, "duration_s": 74, "mean_score": 92.0, "mean_raw": 8.0, "scores_per_judge": [ 92 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 3 } ] }, "judge_disagreement": 0 }, { "idx": 76, "duration_s": 58, "mean_score": 77.0, "mean_raw": 23.0, "scores_per_judge": [ 77 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "semantic-inversion", "points": 20 }, { "category": "filler", "points": 3 } ] }, "judge_disagreement": 0 }, { "idx": 77, "duration_s": 130, "mean_score": 60.0, "mean_raw": 40.0, "scores_per_judge": [ 60 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "semantic-inversion", "points": 20 }, { "category": "brand", "points": 10 }, { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 78, "duration_s": 61, "mean_score": 93.0, "mean_raw": 7.0, "scores_per_judge": [ 93 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 5 }, { "category": "mishear-minor", "points": 2 } ] }, "judge_disagreement": 0 }, { "idx": 79, "duration_s": 47, "mean_score": 96.0, "mean_raw": 4.0, "scores_per_judge": [ 96 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 80, "duration_s": 42, "mean_score": 97.0, "mean_raw": 3.0, "scores_per_judge": [ 97 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 81, "duration_s": 38, "mean_score": 94.0, "mean_raw": 6.0, "scores_per_judge": [ 94 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 3 }, { "category": "paraphrase", "points": 3 } ] }, "judge_disagreement": 0 }, { "idx": 82, "duration_s": 38, "mean_score": 90.0, "mean_raw": 10.0, "scores_per_judge": [ 90 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "tech", "points": 8 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 83, "duration_s": 37, "mean_score": 97.0, "mean_raw": 3.0, "scores_per_judge": [ 97 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 84, "duration_s": 36, "mean_score": 64.0, "mean_raw": 36.0, "scores_per_judge": [ 64 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "polarity", "points": 35 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 85, "duration_s": 36, "mean_score": 89.0, "mean_raw": 11.0, "scores_per_judge": [ 89 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 }, { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 86, "duration_s": 35, "mean_score": 91.0, "mean_raw": 9.0, "scores_per_judge": [ 91 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "tech", "points": 8 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 87, "duration_s": 34, "mean_score": 85.0, "mean_raw": 15.0, "scores_per_judge": [ 85 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "numeric", "points": 10 }, { "category": "filler", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 88, "duration_s": 34, "mean_score": 99.0, "mean_raw": 1.0, "scores_per_judge": [ 99 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 89, "duration_s": 34, "mean_score": 100.0, "mean_raw": 0.0, "scores_per_judge": [ 100 ], "deductions_per_judge": { "deepseek_flash": [] }, "judge_disagreement": 0 }, { "idx": 90, "duration_s": 33, "mean_score": 98.0, "mean_raw": 2.0, "scores_per_judge": [ 98 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 2 } ] }, "judge_disagreement": 0 }, { "idx": 91, "duration_s": 33, "mean_score": 99.0, "mean_raw": 1.0, "scores_per_judge": [ 99 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 92, "duration_s": 32, "mean_score": 98.0, "mean_raw": 2.0, "scores_per_judge": [ 98 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 2 } ] }, "judge_disagreement": 0 }, { "idx": 93, "duration_s": 32, "mean_score": 98.0, "mean_raw": 2.0, "scores_per_judge": [ 98 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 2 } ] }, "judge_disagreement": 0 }, { "idx": 94, "duration_s": 32, "mean_score": 93.0, "mean_raw": 7.0, "scores_per_judge": [ 93 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 2 } ] }, "judge_disagreement": 0 }, { "idx": 95, "duration_s": 32, "mean_score": 94.0, "mean_raw": 6.0, "scores_per_judge": [ 94 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "paraphrase", "points": 3 }, { "category": "filler", "points": 3 } ] }, "judge_disagreement": 0 }, { "idx": 96, "duration_s": 31, "mean_score": 100.0, "mean_raw": 0.0, "scores_per_judge": [ 100 ], "deductions_per_judge": { "deepseek_flash": [] }, "judge_disagreement": 0 }, { "idx": 97, "duration_s": 30, "mean_score": 97.0, "mean_raw": 3.0, "scores_per_judge": [ 97 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 98, "duration_s": 30, "mean_score": 99.0, "mean_raw": 1.0, "scores_per_judge": [ 99 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 99, "duration_s": 30, "mean_score": 100.0, "mean_raw": 0.0, "scores_per_judge": [ 100 ], "deductions_per_judge": { "deepseek_flash": [] }, "judge_disagreement": 0 }, { "idx": 100, "duration_s": 30, "mean_score": 98.0, "mean_raw": 2.0, "scores_per_judge": [ 98 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 2 } ] }, "judge_disagreement": 0 } ], "total_deductions": 708, "avg_deductions_per_sample": 7.08, "mean_clamped_score": 92.9, "n_judge_sample_pairs": 100, "gt_chars": 22625, "ded_per_100char": 3.13, "fidelity_score": 84.4, "overall_score": 92.9, "overall_raw_deductions": 7.08, "score_per_judge": { "deepseek_flash": 92.9 }, "per_group_deductions": { "meaning-flip": { "deepseek_flash": 210 }, "entity": { "deepseek_flash": 182 }, "cultural": { "deepseek_flash": 40 }, "fidelity": { "deepseek_flash": 276 } }, "per_bucket": { "general": { "n_samples": 40, "mean_raw": 7.2, "mean_score": 92.8, "total_raw": 287.0, "idxs": [ 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40 ], "worst_idx": 19 }, "idiom-test": { "n_samples": 5, "mean_raw": 5.4, "mean_score": 94.6, "total_raw": 27.0, "idxs": [ 41, 42, 43, 44, 45 ], "worst_idx": 44 }, "long-mixed": { "n_samples": 5, "mean_raw": 6.6, "mean_score": 93.4, "total_raw": 33.0, "idxs": [ 46, 47, 48, 49, 50 ], "worst_idx": 48 } }, "per_group_per_100char": { "meaning-flip": 0.928, "entity": 0.804, "cultural": 0.177, "fidelity": 1.22 }, "per_group_fidelity": { "meaning-flip": 77.7, "entity": 85.7, "cultural": 89.3, "fidelity": 85.7 } }, "r2t2": { "scores_by_judge": { "deepseek_flash": [ 97, 100, 100, 100, 93, 99, 99, 97, 98, 99, 82, 98, 59, 90, 99, 86, 87, 76, 62, 93, 99, 90, 100, 97, 100, 100, 100, 96, 90, 97, 97, 96, 79, 65, 96, 99, 98, 76, 79, 88, 100, 80, 87, 90, 91, 97, 88, 91, 87, 88, 94, 95, 99, 97, 89, 86, 95, 100, 96, 62, 100, 100, 97, 98, 100, 91, 98, 89, 99, 56, 95, 91, 91, 97, 96, 71, 81, 96, 97, 97, 93, 91, 95, 63, 97, 90, 87, 100, 100, 97, 98, 97, 90, 93, 96, 100, 93, 98, 99, 96 ] }, "per_category_deductions": { "polarity": { "deepseek_flash": 140 }, "semantic-inversion": { "deepseek_flash": 40 }, "hallucination": { "deepseek_flash": 0 }, "numeric": { "deepseek_flash": 10 }, "brand": { "deepseek_flash": 140 }, "tech": { "deepseek_flash": 96 }, "culture": { "deepseek_flash": 48 }, "proper-noun": { "deepseek_flash": 30 }, "dialect": { "deepseek_flash": 0 }, "mishear-severe": { "deepseek_flash": 50 }, "content-drop": { "deepseek_flash": 18 }, "paraphrase": { "deepseek_flash": 24 }, "mishear-minor": { "deepseek_flash": 34 }, "filler": { "deepseek_flash": 189 }, "format": { "deepseek_flash": 0 }, "unscoreable": { "deepseek_flash": 0 } }, "per_tier_deductions": { "tier1": { "deepseek_flash": 190 }, "tier2": { "deepseek_flash": 314 }, "tier3": { "deepseek_flash": 126 }, "tier4": { "deepseek_flash": 189 }, "unscoreable": { "deepseek_flash": 0 } }, "raw_deductions_by_judge": { "deepseek_flash": [ 3, 0, 0, 0, 7, 1, 1, 3, 2, 1, 18, 2, 41, 10, 1, 14, 13, 24, 38, 7, 1, 10, 0, 3, 0, 0, 0, 4, 10, 3, 3, 4, 21, 35, 4, 1, 2, 24, 21, 12, 0, 20, 13, 10, 9, 3, 12, 9, 13, 12, 6, 5, 1, 3, 11, 14, 5, 0, 4, 38, 0, 0, 3, 2, 0, 9, 2, 11, 1, 44, 5, 9, 9, 3, 4, 29, 19, 4, 3, 3, 7, 9, 5, 37, 3, 10, 13, 0, 0, 3, 2, 3, 10, 7, 4, 0, 7, 2, 1, 4 ] }, "sample_details": [ { "idx": 1, "duration_s": 5, "mean_score": 97.0, "mean_raw": 3.0, "scores_per_judge": [ 97 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "paraphrase", "points": 3 } ] }, "judge_disagreement": 0 }, { "idx": 2, "duration_s": 7, "mean_score": 100.0, "mean_raw": 0.0, "scores_per_judge": [ 100 ], "deductions_per_judge": { "deepseek_flash": [] }, "judge_disagreement": 0 }, { "idx": 3, "duration_s": 8, "mean_score": 100.0, "mean_raw": 0.0, "scores_per_judge": [ 100 ], "deductions_per_judge": { "deepseek_flash": [] }, "judge_disagreement": 0 }, { "idx": 4, "duration_s": 11, "mean_score": 100.0, "mean_raw": 0.0, "scores_per_judge": [ 100 ], "deductions_per_judge": { "deepseek_flash": [] }, "judge_disagreement": 0 }, { "idx": 5, "duration_s": 16, "mean_score": 93.0, "mean_raw": 7.0, "scores_per_judge": [ 93 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 4 }, { "category": "content-drop", "points": 3 } ] }, "judge_disagreement": 0 }, { "idx": 6, "duration_s": 17, "mean_score": 99.0, "mean_raw": 1.0, "scores_per_judge": [ 99 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 7, "duration_s": 20, "mean_score": 99.0, "mean_raw": 1.0, "scores_per_judge": [ 99 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 8, "duration_s": 30, "mean_score": 97.0, "mean_raw": 3.0, "scores_per_judge": [ 97 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 2 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 9, "duration_s": 32, "mean_score": 98.0, "mean_raw": 2.0, "scores_per_judge": [ 98 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 10, "duration_s": 35, "mean_score": 99.0, "mean_raw": 1.0, "scores_per_judge": [ 99 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 11, "duration_s": 40, "mean_score": 82.0, "mean_raw": 18.0, "scores_per_judge": [ 82 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "tech", "points": 8 }, { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 12, "duration_s": 46, "mean_score": 98.0, "mean_raw": 2.0, "scores_per_judge": [ 98 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 13, "duration_s": 60, "mean_score": 59.0, "mean_raw": 41.0, "scores_per_judge": [ 59 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "brand", "points": 10 }, { "category": "brand", "points": 10 }, { "category": "tech", "points": 8 }, { "category": "tech", "points": 8 }, { "category": "mishear-minor", "points": 2 }, { "category": "filler", "points": 3 } ] }, "judge_disagreement": 0 }, { "idx": 14, "duration_s": 64, "mean_score": 90.0, "mean_raw": 10.0, "scores_per_judge": [ 90 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 }, { "category": "content-drop", "points": 3 }, { "category": "filler", "points": 2 } ] }, "judge_disagreement": 0 }, { "idx": 15, "duration_s": 69, "mean_score": 99.0, "mean_raw": 1.0, "scores_per_judge": [ 99 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 16, "duration_s": 81, "mean_score": 86.0, "mean_raw": 14.0, "scores_per_judge": [ 86 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "proper-noun", "points": 6 }, { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 3 } ] }, "judge_disagreement": 0 }, { "idx": 17, "duration_s": 100, "mean_score": 87.0, "mean_raw": 13.0, "scores_per_judge": [ 87 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "brand", "points": 10 }, { "category": "filler", "points": 3 } ] }, "judge_disagreement": 0 }, { "idx": 18, "duration_s": 125, "mean_score": 76.0, "mean_raw": 24.0, "scores_per_judge": [ 76 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "brand", "points": 10 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-minor", "points": 2 }, { "category": "mishear-minor", "points": 2 }, { "category": "mishear-minor", "points": 2 }, { "category": "filler", "points": 3 } ] }, "judge_disagreement": 0 }, { "idx": 19, "duration_s": 133, "mean_score": 62.0, "mean_raw": 38.0, "scores_per_judge": [ 62 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "brand", "points": 10 }, { "category": "brand", "points": 10 }, { "category": "tech", "points": 8 }, { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 20, "duration_s": 164, "mean_score": 93.0, "mean_raw": 7.0, "scores_per_judge": [ 93 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 5 }, { "category": "mishear-minor", "points": 2 } ] }, "judge_disagreement": 0 }, { "idx": 21, "duration_s": 5, "mean_score": 99.0, "mean_raw": 1.0, "scores_per_judge": [ 99 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 22, "duration_s": 7, "mean_score": 90.0, "mean_raw": 10.0, "scores_per_judge": [ 90 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "brand", "points": 10 } ] }, "judge_disagreement": 0 }, { "idx": 23, "duration_s": 10, "mean_score": 100.0, "mean_raw": 0.0, "scores_per_judge": [ 100 ], "deductions_per_judge": { "deepseek_flash": [] }, "judge_disagreement": 0 }, { "idx": 24, "duration_s": 14, "mean_score": 97.0, "mean_raw": 3.0, "scores_per_judge": [ 97 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 25, "duration_s": 16, "mean_score": 100.0, "mean_raw": 0.0, "scores_per_judge": [ 100 ], "deductions_per_judge": { "deepseek_flash": [] }, "judge_disagreement": 0 }, { "idx": 26, "duration_s": 23, "mean_score": 100.0, "mean_raw": 0.0, "scores_per_judge": [ 100 ], "deductions_per_judge": { "deepseek_flash": [] }, "judge_disagreement": 0 }, { "idx": 27, "duration_s": 25, "mean_score": 100.0, "mean_raw": 0.0, "scores_per_judge": [ 100 ], "deductions_per_judge": { "deepseek_flash": [] }, "judge_disagreement": 0 }, { "idx": 28, "duration_s": 30, "mean_score": 96.0, "mean_raw": 4.0, "scores_per_judge": [ 96 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-minor", "points": 2 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 29, "duration_s": 34, "mean_score": 90.0, "mean_raw": 10.0, "scores_per_judge": [ 90 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "brand", "points": 10 } ] }, "judge_disagreement": 0 }, { "idx": 30, "duration_s": 38, "mean_score": 97.0, "mean_raw": 3.0, "scores_per_judge": [ 97 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 3 } ] }, "judge_disagreement": 0 }, { "idx": 31, "duration_s": 44, "mean_score": 97.0, "mean_raw": 3.0, "scores_per_judge": [ 97 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 3 } ] }, "judge_disagreement": 0 }, { "idx": 32, "duration_s": 50, "mean_score": 96.0, "mean_raw": 4.0, "scores_per_judge": [ 96 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 4 } ] }, "judge_disagreement": 0 }, { "idx": 33, "duration_s": 62, "mean_score": 79.0, "mean_raw": 21.0, "scores_per_judge": [ 79 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "brand", "points": 10 }, { "category": "proper-noun", "points": 6 }, { "category": "paraphrase", "points": 3 }, { "category": "filler", "points": 2 } ] }, "judge_disagreement": 0 }, { "idx": 34, "duration_s": 66, "mean_score": 65.0, "mean_raw": 35.0, "scores_per_judge": [ 65 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "polarity", "points": 35 } ] }, "judge_disagreement": 0 }, { "idx": 35, "duration_s": 77, "mean_score": 96.0, "mean_raw": 4.0, "scores_per_judge": [ 96 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 4 } ] }, "judge_disagreement": 0 }, { "idx": 36, "duration_s": 94, "mean_score": 99.0, "mean_raw": 1.0, "scores_per_judge": [ 99 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 37, "duration_s": 106, "mean_score": 98.0, "mean_raw": 2.0, "scores_per_judge": [ 98 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 2 } ] }, "judge_disagreement": 0 }, { "idx": 38, "duration_s": 126, "mean_score": 76.0, "mean_raw": 24.0, "scores_per_judge": [ 76 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "tech", "points": 8 }, { "category": "culture", "points": 8 }, { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 3 } ] }, "judge_disagreement": 0 }, { "idx": 39, "duration_s": 158, "mean_score": 79.0, "mean_raw": 21.0, "scores_per_judge": [ 79 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "proper-noun", "points": 6 }, { "category": "tech", "points": 8 }, { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 2 } ] }, "judge_disagreement": 0 }, { "idx": 40, "duration_s": 279, "mean_score": 88.0, "mean_raw": 12.0, "scores_per_judge": [ 88 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "culture", "points": 8 }, { "category": "filler", "points": 4 } ] }, "judge_disagreement": 0 }, { "idx": 41, "duration_s": 34, "mean_score": 100.0, "mean_raw": 0.0, "scores_per_judge": [ 100 ], "deductions_per_judge": { "deepseek_flash": [] }, "judge_disagreement": 0 }, { "idx": 42, "duration_s": 43, "mean_score": 80.0, "mean_raw": 20.0, "scores_per_judge": [ 80 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "semantic-inversion", "points": 20 } ] }, "judge_disagreement": 0 }, { "idx": 43, "duration_s": 29, "mean_score": 87.0, "mean_raw": 13.0, "scores_per_judge": [ 87 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "culture", "points": 8 }, { "category": "mishear-minor", "points": 2 }, { "category": "mishear-minor", "points": 2 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 44, "duration_s": 71, "mean_score": 90.0, "mean_raw": 10.0, "scores_per_judge": [ 90 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "culture", "points": 8 }, { "category": "filler", "points": 2 } ] }, "judge_disagreement": 0 }, { "idx": 45, "duration_s": 46, "mean_score": 91.0, "mean_raw": 9.0, "scores_per_judge": [ 91 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 }, { "category": "mishear-minor", "points": 2 }, { "category": "filler", "points": 2 } ] }, "judge_disagreement": 0 }, { "idx": 46, "duration_s": 62, "mean_score": 97.0, "mean_raw": 3.0, "scores_per_judge": [ 97 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 3 } ] }, "judge_disagreement": 0 }, { "idx": 47, "duration_s": 67, "mean_score": 88.0, "mean_raw": 12.0, "scores_per_judge": [ 88 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "brand", "points": 10 }, { "category": "mishear-minor", "points": 2 } ] }, "judge_disagreement": 0 }, { "idx": 48, "duration_s": 72, "mean_score": 91.0, "mean_raw": 9.0, "scores_per_judge": [ 91 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "tech", "points": 8 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 49, "duration_s": 87, "mean_score": 87.0, "mean_raw": 13.0, "scores_per_judge": [ 87 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "brand", "points": 10 }, { "category": "filler", "points": 3 } ] }, "judge_disagreement": 0 }, { "idx": 50, "duration_s": 106, "mean_score": 88.0, "mean_raw": 12.0, "scores_per_judge": [ 88 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "culture", "points": 8 }, { "category": "mishear-minor", "points": 2 }, { "category": "filler", "points": 2 } ] }, "judge_disagreement": 0 }, { "idx": 51, "duration_s": 33, "mean_score": 94.0, "mean_raw": 6.0, "scores_per_judge": [ 94 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "paraphrase", "points": 3 }, { "category": "paraphrase", "points": 3 } ] }, "judge_disagreement": 0 }, { "idx": 52, "duration_s": 25, "mean_score": 95.0, "mean_raw": 5.0, "scores_per_judge": [ 95 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 53, "duration_s": 35, "mean_score": 99.0, "mean_raw": 1.0, "scores_per_judge": [ 99 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 54, "duration_s": 36, "mean_score": 97.0, "mean_raw": 3.0, "scores_per_judge": [ 97 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 3 } ] }, "judge_disagreement": 0 }, { "idx": 55, "duration_s": 33, "mean_score": 89.0, "mean_raw": 11.0, "scores_per_judge": [ 89 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "proper-noun", "points": 6 }, { "category": "filler", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 56, "duration_s": 15, "mean_score": 86.0, "mean_raw": 14.0, "scores_per_judge": [ 86 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "brand", "points": 10 }, { "category": "content-drop", "points": 3 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 57, "duration_s": 38, "mean_score": 95.0, "mean_raw": 5.0, "scores_per_judge": [ 95 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-minor", "points": 2 }, { "category": "filler", "points": 3 } ] }, "judge_disagreement": 0 }, { "idx": 58, "duration_s": 15, "mean_score": 100.0, "mean_raw": 0.0, "scores_per_judge": [ 100 ], "deductions_per_judge": { "deepseek_flash": [] }, "judge_disagreement": 0 }, { "idx": 59, "duration_s": 28, "mean_score": 96.0, "mean_raw": 4.0, "scores_per_judge": [ 96 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-minor", "points": 2 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 60, "duration_s": 18, "mean_score": 62.0, "mean_raw": 38.0, "scores_per_judge": [ 62 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "polarity", "points": 35 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 61, "duration_s": 19, "mean_score": 100.0, "mean_raw": 0.0, "scores_per_judge": [ 100 ], "deductions_per_judge": { "deepseek_flash": [] }, "judge_disagreement": 0 }, { "idx": 62, "duration_s": 17, "mean_score": 100.0, "mean_raw": 0.0, "scores_per_judge": [ 100 ], "deductions_per_judge": { "deepseek_flash": [] }, "judge_disagreement": 0 }, { "idx": 63, "duration_s": 21, "mean_score": 97.0, "mean_raw": 3.0, "scores_per_judge": [ 97 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 3 } ] }, "judge_disagreement": 0 }, { "idx": 64, "duration_s": 23, "mean_score": 98.0, "mean_raw": 2.0, "scores_per_judge": [ 98 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 2 } ] }, "judge_disagreement": 0 }, { "idx": 65, "duration_s": 20, "mean_score": 100.0, "mean_raw": 0.0, "scores_per_judge": [ 100 ], "deductions_per_judge": { "deepseek_flash": [] }, "judge_disagreement": 0 }, { "idx": 66, "duration_s": 24, "mean_score": 91.0, "mean_raw": 9.0, "scores_per_judge": [ 91 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "culture", "points": 8 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 67, "duration_s": 16, "mean_score": 98.0, "mean_raw": 2.0, "scores_per_judge": [ 98 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-minor", "points": 2 } ] }, "judge_disagreement": 0 }, { "idx": 68, "duration_s": 18, "mean_score": 89.0, "mean_raw": 11.0, "scores_per_judge": [ 89 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "brand", "points": 10 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 69, "duration_s": 18, "mean_score": 99.0, "mean_raw": 1.0, "scores_per_judge": [ 99 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 70, "duration_s": 17, "mean_score": 56.0, "mean_raw": 44.0, "scores_per_judge": [ 56 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "polarity", "points": 35 }, { "category": "tech", "points": 8 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 71, "duration_s": 51, "mean_score": 95.0, "mean_raw": 5.0, "scores_per_judge": [ 95 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-minor", "points": 2 }, { "category": "filler", "points": 3 } ] }, "judge_disagreement": 0 }, { "idx": 72, "duration_s": 54, "mean_score": 91.0, "mean_raw": 9.0, "scores_per_judge": [ 91 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "tech", "points": 8 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 73, "duration_s": 56, "mean_score": 91.0, "mean_raw": 9.0, "scores_per_judge": [ 91 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "paraphrase", "points": 3 }, { "category": "content-drop", "points": 3 }, { "category": "filler", "points": 3 } ] }, "judge_disagreement": 0 }, { "idx": 74, "duration_s": 45, "mean_score": 97.0, "mean_raw": 3.0, "scores_per_judge": [ 97 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "paraphrase", "points": 3 } ] }, "judge_disagreement": 0 }, { "idx": 75, "duration_s": 74, "mean_score": 96.0, "mean_raw": 4.0, "scores_per_judge": [ 96 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 4 } ] }, "judge_disagreement": 0 }, { "idx": 76, "duration_s": 58, "mean_score": 71.0, "mean_raw": 29.0, "scores_per_judge": [ 71 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "semantic-inversion", "points": 20 }, { "category": "proper-noun", "points": 6 }, { "category": "filler", "points": 3 } ] }, "judge_disagreement": 0 }, { "idx": 77, "duration_s": 130, "mean_score": 81.0, "mean_raw": 19.0, "scores_per_judge": [ 81 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "brand", "points": 10 }, { "category": "filler", "points": 5 }, { "category": "mishear-minor", "points": 2 }, { "category": "mishear-minor", "points": 2 } ] }, "judge_disagreement": 0 }, { "idx": 78, "duration_s": 61, "mean_score": 96.0, "mean_raw": 4.0, "scores_per_judge": [ 96 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 4 } ] }, "judge_disagreement": 0 }, { "idx": 79, "duration_s": 47, "mean_score": 97.0, "mean_raw": 3.0, "scores_per_judge": [ 97 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 3 } ] }, "judge_disagreement": 0 }, { "idx": 80, "duration_s": 42, "mean_score": 97.0, "mean_raw": 3.0, "scores_per_judge": [ 97 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 3 } ] }, "judge_disagreement": 0 }, { "idx": 81, "duration_s": 38, "mean_score": 93.0, "mean_raw": 7.0, "scores_per_judge": [ 93 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "paraphrase", "points": 3 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 82, "duration_s": 38, "mean_score": 91.0, "mean_raw": 9.0, "scores_per_judge": [ 91 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "tech", "points": 8 } ] }, "judge_disagreement": 0 }, { "idx": 83, "duration_s": 37, "mean_score": 95.0, "mean_raw": 5.0, "scores_per_judge": [ 95 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 84, "duration_s": 36, "mean_score": 63.0, "mean_raw": 37.0, "scores_per_judge": [ 63 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "polarity", "points": 35 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 85, "duration_s": 36, "mean_score": 97.0, "mean_raw": 3.0, "scores_per_judge": [ 97 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 3 } ] }, "judge_disagreement": 0 }, { "idx": 86, "duration_s": 35, "mean_score": 90.0, "mean_raw": 10.0, "scores_per_judge": [ 90 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "tech", "points": 8 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 87, "duration_s": 34, "mean_score": 87.0, "mean_raw": 13.0, "scores_per_judge": [ 87 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "numeric", "points": 10 }, { "category": "filler", "points": 3 } ] }, "judge_disagreement": 0 }, { "idx": 88, "duration_s": 34, "mean_score": 100.0, "mean_raw": 0.0, "scores_per_judge": [ 100 ], "deductions_per_judge": { "deepseek_flash": [] }, "judge_disagreement": 0 }, { "idx": 89, "duration_s": 34, "mean_score": 100.0, "mean_raw": 0.0, "scores_per_judge": [ 100 ], "deductions_per_judge": { "deepseek_flash": [] }, "judge_disagreement": 0 }, { "idx": 90, "duration_s": 33, "mean_score": 97.0, "mean_raw": 3.0, "scores_per_judge": [ 97 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "content-drop", "points": 3 } ] }, "judge_disagreement": 0 }, { "idx": 91, "duration_s": 33, "mean_score": 98.0, "mean_raw": 2.0, "scores_per_judge": [ 98 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 92, "duration_s": 32, "mean_score": 97.0, "mean_raw": 3.0, "scores_per_judge": [ 97 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "content-drop", "points": 3 } ] }, "judge_disagreement": 0 }, { "idx": 93, "duration_s": 32, "mean_score": 90.0, "mean_raw": 10.0, "scores_per_judge": [ 90 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "tech", "points": 8 }, { "category": "filler", "points": 2 } ] }, "judge_disagreement": 0 }, { "idx": 94, "duration_s": 32, "mean_score": 93.0, "mean_raw": 7.0, "scores_per_judge": [ 93 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 95, "duration_s": 32, "mean_score": 96.0, "mean_raw": 4.0, "scores_per_judge": [ 96 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "paraphrase", "points": 3 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 96, "duration_s": 31, "mean_score": 100.0, "mean_raw": 0.0, "scores_per_judge": [ 100 ], "deductions_per_judge": { "deepseek_flash": [] }, "judge_disagreement": 0 }, { "idx": 97, "duration_s": 30, "mean_score": 93.0, "mean_raw": 7.0, "scores_per_judge": [ 93 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 98, "duration_s": 30, "mean_score": 98.0, "mean_raw": 2.0, "scores_per_judge": [ 98 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 2 } ] }, "judge_disagreement": 0 }, { "idx": 99, "duration_s": 30, "mean_score": 99.0, "mean_raw": 1.0, "scores_per_judge": [ 99 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 100, "duration_s": 30, "mean_score": 96.0, "mean_raw": 4.0, "scores_per_judge": [ 96 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 4 } ] }, "judge_disagreement": 0 } ], "total_deductions": 819, "avg_deductions_per_sample": 8.19, "mean_clamped_score": 91.8, "n_judge_sample_pairs": 100, "gt_chars": 22625, "ded_per_100char": 3.62, "fidelity_score": 81.9, "overall_score": 91.8, "overall_raw_deductions": 8.19, "score_per_judge": { "deepseek_flash": 91.8 }, "per_group_deductions": { "meaning-flip": { "deepseek_flash": 190 }, "entity": { "deepseek_flash": 266 }, "cultural": { "deepseek_flash": 48 }, "fidelity": { "deepseek_flash": 315 } }, "per_bucket": { "general": { "n_samples": 40, "mean_raw": 8.6, "mean_score": 91.4, "total_raw": 344.0, "idxs": [ 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40 ], "worst_idx": 13 }, "idiom-test": { "n_samples": 5, "mean_raw": 10.4, "mean_score": 89.6, "total_raw": 52.0, "idxs": [ 41, 42, 43, 44, 45 ], "worst_idx": 42 }, "long-mixed": { "n_samples": 5, "mean_raw": 9.8, "mean_score": 90.2, "total_raw": 49.0, "idxs": [ 46, 47, 48, 49, 50 ], "worst_idx": 49 } }, "per_group_per_100char": { "meaning-flip": 0.84, "entity": 1.176, "cultural": 0.212, "fidelity": 1.392 }, "per_group_fidelity": { "meaning-flip": 79.9, "entity": 79.2, "cultural": 87.2, "fidelity": 83.7 } }, "soniox": { "scores_by_judge": { "deepseek_flash": [ 100, 99, 99, 100, 97, 99, 88, 99, 98, 98, 79, 85, 70, 80, 96, 91, 73, 80, 55, 78, 99, 90, 100, 90, 100, 100, 97, 94, 99, 89, 100, 95, 91, 89, 90, 95, 98, 82, 79, 67, 67, 97, 79, 75, 68, 96, 89, 88, 95, 79, 84, 83, 97, 96, 95, 89, 97, 94, 98, 97, 95, 100, 99, 80, 99, 90, 98, 90, 97, 99, 90, 99, 81, 98, 89, 94, 89, 96, 99, 97, 92, 90, 96, 97, 93, 88, 71, 95, 100, 91, 98, 100, 92, 97, 100, 99, 98, 98, 99, 96 ] }, "per_category_deductions": { "polarity": { "deepseek_flash": 0 }, "semantic-inversion": { "deepseek_flash": 0 }, "hallucination": { "deepseek_flash": 30 }, "numeric": { "deepseek_flash": 10 }, "brand": { "deepseek_flash": 130 }, "tech": { "deepseek_flash": 104 }, "culture": { "deepseek_flash": 112 }, "proper-noun": { "deepseek_flash": 48 }, "dialect": { "deepseek_flash": 20 }, "mishear-severe": { "deepseek_flash": 155 }, "content-drop": { "deepseek_flash": 0 }, "paraphrase": { "deepseek_flash": 15 }, "mishear-minor": { "deepseek_flash": 34 }, "filler": { "deepseek_flash": 202 }, "format": { "deepseek_flash": 0 }, "unscoreable": { "deepseek_flash": 0 } }, "per_tier_deductions": { "tier1": { "deepseek_flash": 40 }, "tier2": { "deepseek_flash": 414 }, "tier3": { "deepseek_flash": 204 }, "tier4": { "deepseek_flash": 202 }, "unscoreable": { "deepseek_flash": 0 } }, "raw_deductions_by_judge": { "deepseek_flash": [ 0, 1, 1, 0, 3, 1, 12, 1, 2, 2, 21, 15, 30, 20, 4, 9, 27, 20, 45, 22, 1, 10, 0, 10, 0, 0, 3, 6, 1, 11, 0, 5, 9, 11, 10, 5, 2, 18, 21, 33, 33, 3, 21, 25, 32, 4, 11, 12, 5, 21, 16, 17, 3, 4, 5, 11, 3, 6, 2, 3, 5, 0, 1, 20, 1, 10, 2, 10, 3, 1, 10, 1, 19, 2, 11, 6, 11, 4, 1, 3, 8, 10, 4, 3, 7, 12, 29, 5, 0, 9, 2, 0, 8, 3, 0, 1, 2, 2, 1, 4 ] }, "sample_details": [ { "idx": 1, "duration_s": 5, "mean_score": 100.0, "mean_raw": 0.0, "scores_per_judge": [ 100 ], "deductions_per_judge": { "deepseek_flash": [] }, "judge_disagreement": 0 }, { "idx": 2, "duration_s": 7, "mean_score": 99.0, "mean_raw": 1.0, "scores_per_judge": [ 99 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 3, "duration_s": 8, "mean_score": 99.0, "mean_raw": 1.0, "scores_per_judge": [ 99 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 4, "duration_s": 11, "mean_score": 100.0, "mean_raw": 0.0, "scores_per_judge": [ 100 ], "deductions_per_judge": { "deepseek_flash": [] }, "judge_disagreement": 0 }, { "idx": 5, "duration_s": 16, "mean_score": 97.0, "mean_raw": 3.0, "scores_per_judge": [ 97 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 6, "duration_s": 17, "mean_score": 99.0, "mean_raw": 1.0, "scores_per_judge": [ 99 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 7, "duration_s": 20, "mean_score": 88.0, "mean_raw": 12.0, "scores_per_judge": [ 88 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "tech", "points": 8 }, { "category": "mishear-minor", "points": 2 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 8, "duration_s": 30, "mean_score": 99.0, "mean_raw": 1.0, "scores_per_judge": [ 99 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 9, "duration_s": 32, "mean_score": 98.0, "mean_raw": 2.0, "scores_per_judge": [ 98 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 10, "duration_s": 35, "mean_score": 98.0, "mean_raw": 2.0, "scores_per_judge": [ 98 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 11, "duration_s": 40, "mean_score": 79.0, "mean_raw": 21.0, "scores_per_judge": [ 79 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "tech", "points": 8 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 12, "duration_s": 46, "mean_score": 85.0, "mean_raw": 15.0, "scores_per_judge": [ 85 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 }, { "category": "culture", "points": 8 }, { "category": "filler", "points": 2 } ] }, "judge_disagreement": 0 }, { "idx": 13, "duration_s": 60, "mean_score": 70.0, "mean_raw": 30.0, "scores_per_judge": [ 70 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "brand", "points": 10 }, { "category": "brand", "points": 10 }, { "category": "tech", "points": 8 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 14, "duration_s": 64, "mean_score": 80.0, "mean_raw": 20.0, "scores_per_judge": [ 80 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 }, { "category": "culture", "points": 8 }, { "category": "mishear-minor", "points": 2 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 15, "duration_s": 69, "mean_score": 96.0, "mean_raw": 4.0, "scores_per_judge": [ 96 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 16, "duration_s": 81, "mean_score": 91.0, "mean_raw": 9.0, "scores_per_judge": [ 91 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "proper-noun", "points": 6 }, { "category": "filler", "points": 3 } ] }, "judge_disagreement": 0 }, { "idx": 17, "duration_s": 100, "mean_score": 73.0, "mean_raw": 27.0, "scores_per_judge": [ 73 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "brand", "points": 10 }, { "category": "tech", "points": 8 }, { "category": "tech", "points": 8 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 18, "duration_s": 125, "mean_score": 80.0, "mean_raw": 20.0, "scores_per_judge": [ 80 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "brand", "points": 10 }, { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 19, "duration_s": 133, "mean_score": 55.0, "mean_raw": 45.0, "scores_per_judge": [ 55 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "brand", "points": 10 }, { "category": "brand", "points": 10 }, { "category": "brand", "points": 10 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 20, "duration_s": 164, "mean_score": 78.0, "mean_raw": 22.0, "scores_per_judge": [ 78 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 }, { "category": "culture", "points": 8 }, { "category": "mishear-minor", "points": 2 }, { "category": "mishear-minor", "points": 2 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 21, "duration_s": 5, "mean_score": 99.0, "mean_raw": 1.0, "scores_per_judge": [ 99 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 22, "duration_s": 7, "mean_score": 90.0, "mean_raw": 10.0, "scores_per_judge": [ 90 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "brand", "points": 10 } ] }, "judge_disagreement": 0 }, { "idx": 23, "duration_s": 10, "mean_score": 100.0, "mean_raw": 0.0, "scores_per_judge": [ 100 ], "deductions_per_judge": { "deepseek_flash": [] }, "judge_disagreement": 0 }, { "idx": 24, "duration_s": 14, "mean_score": 90.0, "mean_raw": 10.0, "scores_per_judge": [ 90 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "tech", "points": 8 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 25, "duration_s": 16, "mean_score": 100.0, "mean_raw": 0.0, "scores_per_judge": [ 100 ], "deductions_per_judge": { "deepseek_flash": [] }, "judge_disagreement": 0 }, { "idx": 26, "duration_s": 23, "mean_score": 100.0, "mean_raw": 0.0, "scores_per_judge": [ 100 ], "deductions_per_judge": { "deepseek_flash": [] }, "judge_disagreement": 0 }, { "idx": 27, "duration_s": 25, "mean_score": 97.0, "mean_raw": 3.0, "scores_per_judge": [ 97 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "paraphrase", "points": 3 } ] }, "judge_disagreement": 0 }, { "idx": 28, "duration_s": 30, "mean_score": 94.0, "mean_raw": 6.0, "scores_per_judge": [ 94 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 29, "duration_s": 34, "mean_score": 99.0, "mean_raw": 1.0, "scores_per_judge": [ 99 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 30, "duration_s": 38, "mean_score": 89.0, "mean_raw": 11.0, "scores_per_judge": [ 89 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "culture", "points": 8 }, { "category": "filler", "points": 3 } ] }, "judge_disagreement": 0 }, { "idx": 31, "duration_s": 44, "mean_score": 100.0, "mean_raw": 0.0, "scores_per_judge": [ 100 ], "deductions_per_judge": { "deepseek_flash": [] }, "judge_disagreement": 0 }, { "idx": 32, "duration_s": 50, "mean_score": 95.0, "mean_raw": 5.0, "scores_per_judge": [ 95 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 33, "duration_s": 62, "mean_score": 91.0, "mean_raw": 9.0, "scores_per_judge": [ 91 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "proper-noun", "points": 6 }, { "category": "filler", "points": 3 } ] }, "judge_disagreement": 0 }, { "idx": 34, "duration_s": 66, "mean_score": 89.0, "mean_raw": 11.0, "scores_per_judge": [ 89 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "proper-noun", "points": 6 }, { "category": "mishear-minor", "points": 2 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 35, "duration_s": 77, "mean_score": 90.0, "mean_raw": 10.0, "scores_per_judge": [ 90 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 }, { "category": "mishear-minor", "points": 2 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 36, "duration_s": 94, "mean_score": 95.0, "mean_raw": 5.0, "scores_per_judge": [ 95 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 37, "duration_s": 106, "mean_score": 98.0, "mean_raw": 2.0, "scores_per_judge": [ 98 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 38, "duration_s": 126, "mean_score": 82.0, "mean_raw": 18.0, "scores_per_judge": [ 82 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "tech", "points": 8 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-severe", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 39, "duration_s": 158, "mean_score": 79.0, "mean_raw": 21.0, "scores_per_judge": [ 79 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "tech", "points": 8 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 3 } ] }, "judge_disagreement": 0 }, { "idx": 40, "duration_s": 279, "mean_score": 67.0, "mean_raw": 33.0, "scores_per_judge": [ 67 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "culture", "points": 8 }, { "category": "culture", "points": 8 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-minor", "points": 2 }, { "category": "mishear-minor", "points": 2 }, { "category": "paraphrase", "points": 3 }, { "category": "filler", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 41, "duration_s": 34, "mean_score": 67.0, "mean_raw": 33.0, "scores_per_judge": [ 67 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "culture", "points": 8 }, { "category": "culture", "points": 8 }, { "category": "culture", "points": 8 }, { "category": "culture", "points": 8 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 42, "duration_s": 43, "mean_score": 97.0, "mean_raw": 3.0, "scores_per_judge": [ 97 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 43, "duration_s": 29, "mean_score": 79.0, "mean_raw": 21.0, "scores_per_judge": [ 79 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "culture", "points": 8 }, { "category": "proper-noun", "points": 6 }, { "category": "proper-noun", "points": 6 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 44, "duration_s": 71, "mean_score": 75.0, "mean_raw": 25.0, "scores_per_judge": [ 75 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-severe", "points": 5 }, { "category": "culture", "points": 8 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 45, "duration_s": 46, "mean_score": 68.0, "mean_raw": 32.0, "scores_per_judge": [ 68 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "dialect", "points": 5 }, { "category": "mishear-severe", "points": 5 }, { "category": "dialect", "points": 5 }, { "category": "filler", "points": 1 }, { "category": "mishear-minor", "points": 2 }, { "category": "dialect", "points": 5 }, { "category": "filler", "points": 1 }, { "category": "mishear-minor", "points": 2 }, { "category": "dialect", "points": 5 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 46, "duration_s": 62, "mean_score": 96.0, "mean_raw": 4.0, "scores_per_judge": [ 96 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 4 } ] }, "judge_disagreement": 0 }, { "idx": 47, "duration_s": 67, "mean_score": 89.0, "mean_raw": 11.0, "scores_per_judge": [ 89 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "tech", "points": 8 }, { "category": "filler", "points": 3 } ] }, "judge_disagreement": 0 }, { "idx": 48, "duration_s": 72, "mean_score": 88.0, "mean_raw": 12.0, "scores_per_judge": [ 88 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "brand", "points": 10 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 49, "duration_s": 87, "mean_score": 95.0, "mean_raw": 5.0, "scores_per_judge": [ 95 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 50, "duration_s": 106, "mean_score": 79.0, "mean_raw": 21.0, "scores_per_judge": [ 79 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "culture", "points": 8 }, { "category": "tech", "points": 8 }, { "category": "mishear-minor", "points": 2 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 51, "duration_s": 33, "mean_score": 84.0, "mean_raw": 16.0, "scores_per_judge": [ 84 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-minor", "points": 2 }, { "category": "mishear-minor", "points": 2 }, { "category": "filler", "points": 1 }, { "category": "proper-noun", "points": 6 }, { "category": "mishear-severe", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 52, "duration_s": 25, "mean_score": 83.0, "mean_raw": 17.0, "scores_per_judge": [ 83 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "proper-noun", "points": 6 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 53, "duration_s": 35, "mean_score": 97.0, "mean_raw": 3.0, "scores_per_judge": [ 97 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 54, "duration_s": 36, "mean_score": 96.0, "mean_raw": 4.0, "scores_per_judge": [ 96 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 4 } ] }, "judge_disagreement": 0 }, { "idx": 55, "duration_s": 33, "mean_score": 95.0, "mean_raw": 5.0, "scores_per_judge": [ 95 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 56, "duration_s": 15, "mean_score": 89.0, "mean_raw": 11.0, "scores_per_judge": [ 89 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "brand", "points": 10 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 57, "duration_s": 38, "mean_score": 97.0, "mean_raw": 3.0, "scores_per_judge": [ 97 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 58, "duration_s": 15, "mean_score": 94.0, "mean_raw": 6.0, "scores_per_judge": [ 94 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 59, "duration_s": 28, "mean_score": 98.0, "mean_raw": 2.0, "scores_per_judge": [ 98 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 60, "duration_s": 18, "mean_score": 97.0, "mean_raw": 3.0, "scores_per_judge": [ 97 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 61, "duration_s": 19, "mean_score": 95.0, "mean_raw": 5.0, "scores_per_judge": [ 95 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 62, "duration_s": 17, "mean_score": 100.0, "mean_raw": 0.0, "scores_per_judge": [ 100 ], "deductions_per_judge": { "deepseek_flash": [] }, "judge_disagreement": 0 }, { "idx": 63, "duration_s": 21, "mean_score": 99.0, "mean_raw": 1.0, "scores_per_judge": [ 99 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 64, "duration_s": 23, "mean_score": 80.0, "mean_raw": 20.0, "scores_per_judge": [ 80 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "hallucination", "points": 15 }, { "category": "paraphrase", "points": 3 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 65, "duration_s": 20, "mean_score": 99.0, "mean_raw": 1.0, "scores_per_judge": [ 99 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 66, "duration_s": 24, "mean_score": 90.0, "mean_raw": 10.0, "scores_per_judge": [ 90 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "culture", "points": 8 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 67, "duration_s": 16, "mean_score": 98.0, "mean_raw": 2.0, "scores_per_judge": [ 98 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 2 } ] }, "judge_disagreement": 0 }, { "idx": 68, "duration_s": 18, "mean_score": 90.0, "mean_raw": 10.0, "scores_per_judge": [ 90 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "brand", "points": 10 } ] }, "judge_disagreement": 0 }, { "idx": 69, "duration_s": 18, "mean_score": 97.0, "mean_raw": 3.0, "scores_per_judge": [ 97 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 70, "duration_s": 17, "mean_score": 99.0, "mean_raw": 1.0, "scores_per_judge": [ 99 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 71, "duration_s": 51, "mean_score": 90.0, "mean_raw": 10.0, "scores_per_judge": [ 90 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 }, { "category": "mishear-minor", "points": 2 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 72, "duration_s": 54, "mean_score": 99.0, "mean_raw": 1.0, "scores_per_judge": [ 99 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 73, "duration_s": 56, "mean_score": 81.0, "mean_raw": 19.0, "scores_per_judge": [ 81 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "brand", "points": 10 }, { "category": "mishear-severe", "points": 5 }, { "category": "paraphrase", "points": 3 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 74, "duration_s": 45, "mean_score": 98.0, "mean_raw": 2.0, "scores_per_judge": [ 98 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 75, "duration_s": 74, "mean_score": 89.0, "mean_raw": 11.0, "scores_per_judge": [ 89 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "proper-noun", "points": 6 }, { "category": "mishear-minor", "points": 2 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 76, "duration_s": 58, "mean_score": 94.0, "mean_raw": 6.0, "scores_per_judge": [ 94 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "paraphrase", "points": 3 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 77, "duration_s": 130, "mean_score": 89.0, "mean_raw": 11.0, "scores_per_judge": [ 89 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "brand", "points": 10 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 78, "duration_s": 61, "mean_score": 96.0, "mean_raw": 4.0, "scores_per_judge": [ 96 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "mishear-minor", "points": 2 } ] }, "judge_disagreement": 0 }, { "idx": 79, "duration_s": 47, "mean_score": 99.0, "mean_raw": 1.0, "scores_per_judge": [ 99 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 80, "duration_s": 42, "mean_score": 97.0, "mean_raw": 3.0, "scores_per_judge": [ 97 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 3 } ] }, "judge_disagreement": 0 }, { "idx": 81, "duration_s": 38, "mean_score": 92.0, "mean_raw": 8.0, "scores_per_judge": [ 92 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 82, "duration_s": 38, "mean_score": 90.0, "mean_raw": 10.0, "scores_per_judge": [ 90 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "tech", "points": 8 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 83, "duration_s": 37, "mean_score": 96.0, "mean_raw": 4.0, "scores_per_judge": [ 96 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 84, "duration_s": 36, "mean_score": 97.0, "mean_raw": 3.0, "scores_per_judge": [ 97 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 3 } ] }, "judge_disagreement": 0 }, { "idx": 85, "duration_s": 36, "mean_score": 93.0, "mean_raw": 7.0, "scores_per_judge": [ 93 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 86, "duration_s": 35, "mean_score": 88.0, "mean_raw": 12.0, "scores_per_judge": [ 88 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "tech", "points": 8 }, { "category": "filler", "points": 4 } ] }, "judge_disagreement": 0 }, { "idx": 87, "duration_s": 34, "mean_score": 71.0, "mean_raw": 29.0, "scores_per_judge": [ 71 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "hallucination", "points": 15 }, { "category": "numeric", "points": 10 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 88, "duration_s": 34, "mean_score": 95.0, "mean_raw": 5.0, "scores_per_judge": [ 95 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 89, "duration_s": 34, "mean_score": 100.0, "mean_raw": 0.0, "scores_per_judge": [ 100 ], "deductions_per_judge": { "deepseek_flash": [] }, "judge_disagreement": 0 }, { "idx": 90, "duration_s": 33, "mean_score": 91.0, "mean_raw": 9.0, "scores_per_judge": [ 91 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "tech", "points": 8 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 91, "duration_s": 33, "mean_score": 98.0, "mean_raw": 2.0, "scores_per_judge": [ 98 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 2 } ] }, "judge_disagreement": 0 }, { "idx": 92, "duration_s": 32, "mean_score": 100.0, "mean_raw": 0.0, "scores_per_judge": [ 100 ], "deductions_per_judge": { "deepseek_flash": [] }, "judge_disagreement": 0 }, { "idx": 93, "duration_s": 32, "mean_score": 92.0, "mean_raw": 8.0, "scores_per_judge": [ 92 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 94, "duration_s": 32, "mean_score": 97.0, "mean_raw": 3.0, "scores_per_judge": [ 97 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 95, "duration_s": 32, "mean_score": 100.0, "mean_raw": 0.0, "scores_per_judge": [ 100 ], "deductions_per_judge": { "deepseek_flash": [] }, "judge_disagreement": 0 }, { "idx": 96, "duration_s": 31, "mean_score": 99.0, "mean_raw": 1.0, "scores_per_judge": [ 99 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 97, "duration_s": 30, "mean_score": 98.0, "mean_raw": 2.0, "scores_per_judge": [ 98 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 2 } ] }, "judge_disagreement": 0 }, { "idx": 98, "duration_s": 30, "mean_score": 98.0, "mean_raw": 2.0, "scores_per_judge": [ 98 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 99, "duration_s": 30, "mean_score": 99.0, "mean_raw": 1.0, "scores_per_judge": [ 99 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 100, "duration_s": 30, "mean_score": 96.0, "mean_raw": 4.0, "scores_per_judge": [ 96 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-minor", "points": 2 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 } ], "total_deductions": 860, "avg_deductions_per_sample": 8.6, "mean_clamped_score": 91.4, "n_judge_sample_pairs": 100, "gt_chars": 22625, "ded_per_100char": 3.8, "fidelity_score": 81.0, "overall_score": 91.4, "overall_raw_deductions": 8.6, "score_per_judge": { "deepseek_flash": 91.4 }, "per_group_deductions": { "meaning-flip": { "deepseek_flash": 40 }, "entity": { "deepseek_flash": 282 }, "cultural": { "deepseek_flash": 132 }, "fidelity": { "deepseek_flash": 406 } }, "per_bucket": { "general": { "n_samples": 40, "mean_raw": 9.8, "mean_score": 90.2, "total_raw": 392.0, "idxs": [ 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40 ], "worst_idx": 19 }, "idiom-test": { "n_samples": 5, "mean_raw": 22.8, "mean_score": 77.2, "total_raw": 114.0, "idxs": [ 41, 42, 43, 44, 45 ], "worst_idx": 41 }, "long-mixed": { "n_samples": 5, "mean_raw": 10.6, "mean_score": 89.4, "total_raw": 53.0, "idxs": [ 46, 47, 48, 49, 50 ], "worst_idx": 50 } }, "per_group_per_100char": { "meaning-flip": 0.177, "entity": 1.246, "cultural": 0.583, "fidelity": 1.794 }, "per_group_fidelity": { "meaning-flip": 95.8, "entity": 77.9, "cultural": 64.7, "fidelity": 79.0 } }, "stepfun": { "scores_by_judge": { "deepseek_flash": [ 100, 98, 100, 98, 98, 89, 88, 92, 95, 92, 77, 96, 68, 67, 98, 73, 80, 81, 44, 95, 100, 94, 100, 90, 84, 100, 98, 93, 81, 54, 52, 97, 85, 60, 88, 98, 97, 93, 59, 66, 100, 98, 88, 80, 94, 93, 84, 90, 88, 75, 93, 94, 97, 96, 77, 90, 80, 98, 92, 79, 95, 100, 98, 95, 100, 91, 99, 90, 100, 22, 72, 93, 84, 95, 95, 76, 59, 94, 95, 97, 87, 93, 96, 96, 92, 82, 82, 99, 100, 90, 63, 100, 68, 96, 77, 85, 82, 79, 85, 97 ] }, "per_category_deductions": { "polarity": { "deepseek_flash": 210 }, "semantic-inversion": { "deepseek_flash": 60 }, "hallucination": { "deepseek_flash": 90 }, "numeric": { "deepseek_flash": 20 }, "brand": { "deepseek_flash": 170 }, "tech": { "deepseek_flash": 192 }, "culture": { "deepseek_flash": 48 }, "proper-noun": { "deepseek_flash": 48 }, "dialect": { "deepseek_flash": 0 }, "mishear-severe": { "deepseek_flash": 180 }, "content-drop": { "deepseek_flash": 18 }, "paraphrase": { "deepseek_flash": 15 }, "mishear-minor": { "deepseek_flash": 44 }, "filler": { "deepseek_flash": 200 }, "format": { "deepseek_flash": 2 }, "unscoreable": { "deepseek_flash": 0 } }, "per_tier_deductions": { "tier1": { "deepseek_flash": 380 }, "tier2": { "deepseek_flash": 458 }, "tier3": { "deepseek_flash": 257 }, "tier4": { "deepseek_flash": 202 }, "unscoreable": { "deepseek_flash": 0 } }, "raw_deductions_by_judge": { "deepseek_flash": [ 0, 2, 0, 2, 2, 11, 12, 8, 5, 8, 23, 4, 32, 33, 2, 27, 20, 19, 56, 5, 0, 6, 0, 10, 16, 0, 2, 7, 19, 46, 48, 3, 15, 40, 12, 2, 3, 7, 41, 34, 0, 2, 12, 20, 6, 7, 16, 10, 12, 25, 7, 6, 3, 4, 23, 10, 20, 2, 8, 21, 5, 0, 2, 5, 0, 9, 1, 10, 0, 78, 28, 7, 16, 5, 5, 24, 41, 6, 5, 3, 13, 7, 4, 4, 8, 18, 18, 1, 0, 10, 37, 0, 32, 4, 23, 15, 18, 21, 15, 3 ] }, "sample_details": [ { "idx": 1, "duration_s": 5, "mean_score": 100.0, "mean_raw": 0.0, "scores_per_judge": [ 100 ], "deductions_per_judge": { "deepseek_flash": [] }, "judge_disagreement": 0 }, { "idx": 2, "duration_s": 7, "mean_score": 98.0, "mean_raw": 2.0, "scores_per_judge": [ 98 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 3, "duration_s": 8, "mean_score": 100.0, "mean_raw": 0.0, "scores_per_judge": [ 100 ], "deductions_per_judge": { "deepseek_flash": [] }, "judge_disagreement": 0 }, { "idx": 4, "duration_s": 11, "mean_score": 98.0, "mean_raw": 2.0, "scores_per_judge": [ 98 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 5, "duration_s": 16, "mean_score": 98.0, "mean_raw": 2.0, "scores_per_judge": [ 98 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 6, "duration_s": 17, "mean_score": 89.0, "mean_raw": 11.0, "scores_per_judge": [ 89 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "tech", "points": 8 }, { "category": "paraphrase", "points": 3 } ] }, "judge_disagreement": 0 }, { "idx": 7, "duration_s": 20, "mean_score": 88.0, "mean_raw": 12.0, "scores_per_judge": [ 88 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "content-drop", "points": 3 }, { "category": "tech", "points": 8 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 8, "duration_s": 30, "mean_score": 92.0, "mean_raw": 8.0, "scores_per_judge": [ 92 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "tech", "points": 8 } ] }, "judge_disagreement": 0 }, { "idx": 9, "duration_s": 32, "mean_score": 95.0, "mean_raw": 5.0, "scores_per_judge": [ 95 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 10, "duration_s": 35, "mean_score": 92.0, "mean_raw": 8.0, "scores_per_judge": [ 92 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 11, "duration_s": 40, "mean_score": 77.0, "mean_raw": 23.0, "scores_per_judge": [ 77 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "tech", "points": 8 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 12, "duration_s": 46, "mean_score": 96.0, "mean_raw": 4.0, "scores_per_judge": [ 96 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 3 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 13, "duration_s": 60, "mean_score": 68.0, "mean_raw": 32.0, "scores_per_judge": [ 68 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "tech", "points": 8 }, { "category": "brand", "points": 10 }, { "category": "brand", "points": 10 }, { "category": "mishear-minor", "points": 2 } ] }, "judge_disagreement": 0 }, { "idx": 14, "duration_s": 64, "mean_score": 67.0, "mean_raw": 33.0, "scores_per_judge": [ 67 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-severe", "points": 5 }, { "category": "proper-noun", "points": 6 }, { "category": "proper-noun", "points": 6 }, { "category": "mishear-minor", "points": 2 }, { "category": "mishear-minor", "points": 2 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 15, "duration_s": 69, "mean_score": 98.0, "mean_raw": 2.0, "scores_per_judge": [ 98 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 16, "duration_s": 81, "mean_score": 73.0, "mean_raw": 27.0, "scores_per_judge": [ 73 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "proper-noun", "points": 6 }, { "category": "mishear-severe", "points": 5 }, { "category": "numeric", "points": 10 }, { "category": "mishear-minor", "points": 2 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 17, "duration_s": 100, "mean_score": 80.0, "mean_raw": 20.0, "scores_per_judge": [ 80 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "brand", "points": 10 }, { "category": "tech", "points": 8 }, { "category": "filler", "points": 2 } ] }, "judge_disagreement": 0 }, { "idx": 18, "duration_s": 125, "mean_score": 81.0, "mean_raw": 19.0, "scores_per_judge": [ 81 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "brand", "points": 10 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-minor", "points": 2 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 19, "duration_s": 133, "mean_score": 44.0, "mean_raw": 56.0, "scores_per_judge": [ 44 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "brand", "points": 10 }, { "category": "brand", "points": 10 }, { "category": "tech", "points": 8 }, { "category": "tech", "points": 8 }, { "category": "content-drop", "points": 3 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-minor", "points": 2 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 20, "duration_s": 164, "mean_score": 95.0, "mean_raw": 5.0, "scores_per_judge": [ 95 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 21, "duration_s": 5, "mean_score": 100.0, "mean_raw": 0.0, "scores_per_judge": [ 100 ], "deductions_per_judge": { "deepseek_flash": [] }, "judge_disagreement": 0 }, { "idx": 22, "duration_s": 7, "mean_score": 94.0, "mean_raw": 6.0, "scores_per_judge": [ 94 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "proper-noun", "points": 6 } ] }, "judge_disagreement": 0 }, { "idx": 23, "duration_s": 10, "mean_score": 100.0, "mean_raw": 0.0, "scores_per_judge": [ 100 ], "deductions_per_judge": { "deepseek_flash": [] }, "judge_disagreement": 0 }, { "idx": 24, "duration_s": 14, "mean_score": 90.0, "mean_raw": 10.0, "scores_per_judge": [ 90 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "tech", "points": 8 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 25, "duration_s": 16, "mean_score": 84.0, "mean_raw": 16.0, "scores_per_judge": [ 84 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "tech", "points": 8 }, { "category": "tech", "points": 8 } ] }, "judge_disagreement": 0 }, { "idx": 26, "duration_s": 23, "mean_score": 100.0, "mean_raw": 0.0, "scores_per_judge": [ 100 ], "deductions_per_judge": { "deepseek_flash": [] }, "judge_disagreement": 0 }, { "idx": 27, "duration_s": 25, "mean_score": 98.0, "mean_raw": 2.0, "scores_per_judge": [ 98 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 28, "duration_s": 30, "mean_score": 93.0, "mean_raw": 7.0, "scores_per_judge": [ 93 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 29, "duration_s": 34, "mean_score": 81.0, "mean_raw": 19.0, "scores_per_judge": [ 81 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "brand", "points": 10 }, { "category": "tech", "points": 8 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 30, "duration_s": 38, "mean_score": 54.0, "mean_raw": 46.0, "scores_per_judge": [ 54 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "polarity", "points": 35 }, { "category": "tech", "points": 8 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 31, "duration_s": 44, "mean_score": 52.0, "mean_raw": 48.0, "scores_per_judge": [ 52 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "polarity", "points": 35 }, { "category": "brand", "points": 10 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 32, "duration_s": 50, "mean_score": 97.0, "mean_raw": 3.0, "scores_per_judge": [ 97 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 33, "duration_s": 62, "mean_score": 85.0, "mean_raw": 15.0, "scores_per_judge": [ 85 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "brand", "points": 10 }, { "category": "content-drop", "points": 3 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 34, "duration_s": 66, "mean_score": 60.0, "mean_raw": 40.0, "scores_per_judge": [ 60 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "polarity", "points": 35 }, { "category": "filler", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 35, "duration_s": 77, "mean_score": 88.0, "mean_raw": 12.0, "scores_per_judge": [ 88 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "tech", "points": 8 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 36, "duration_s": 94, "mean_score": 98.0, "mean_raw": 2.0, "scores_per_judge": [ 98 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 37, "duration_s": 106, "mean_score": 97.0, "mean_raw": 3.0, "scores_per_judge": [ 97 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 38, "duration_s": 126, "mean_score": 93.0, "mean_raw": 7.0, "scores_per_judge": [ 93 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 }, { "category": "mishear-minor", "points": 2 } ] }, "judge_disagreement": 0 }, { "idx": 39, "duration_s": 158, "mean_score": 59.0, "mean_raw": 41.0, "scores_per_judge": [ 59 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "brand", "points": 10 }, { "category": "brand", "points": 10 }, { "category": "proper-noun", "points": 6 }, { "category": "proper-noun", "points": 6 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-minor", "points": 2 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 40, "duration_s": 279, "mean_score": 66.0, "mean_raw": 34.0, "scores_per_judge": [ 66 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 }, { "category": "mishear-severe", "points": 5 }, { "category": "culture", "points": 8 }, { "category": "culture", "points": 8 }, { "category": "paraphrase", "points": 3 }, { "category": "filler", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 41, "duration_s": 34, "mean_score": 100.0, "mean_raw": 0.0, "scores_per_judge": [ 100 ], "deductions_per_judge": { "deepseek_flash": [] }, "judge_disagreement": 0 }, { "idx": 42, "duration_s": 43, "mean_score": 98.0, "mean_raw": 2.0, "scores_per_judge": [ 98 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-minor", "points": 2 } ] }, "judge_disagreement": 0 }, { "idx": 43, "duration_s": 29, "mean_score": 88.0, "mean_raw": 12.0, "scores_per_judge": [ 88 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "culture", "points": 8 }, { "category": "filler", "points": 4 } ] }, "judge_disagreement": 0 }, { "idx": 44, "duration_s": 71, "mean_score": 80.0, "mean_raw": 20.0, "scores_per_judge": [ 80 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "proper-noun", "points": 6 }, { "category": "mishear-severe", "points": 5 }, { "category": "culture", "points": 8 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 45, "duration_s": 46, "mean_score": 94.0, "mean_raw": 6.0, "scores_per_judge": [ 94 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-minor", "points": 2 }, { "category": "filler", "points": 4 } ] }, "judge_disagreement": 0 }, { "idx": 46, "duration_s": 62, "mean_score": 93.0, "mean_raw": 7.0, "scores_per_judge": [ 93 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 47, "duration_s": 67, "mean_score": 84.0, "mean_raw": 16.0, "scores_per_judge": [ 84 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "brand", "points": 10 }, { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 48, "duration_s": 72, "mean_score": 90.0, "mean_raw": 10.0, "scores_per_judge": [ 90 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "tech", "points": 8 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 49, "duration_s": 87, "mean_score": 88.0, "mean_raw": 12.0, "scores_per_judge": [ 88 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 }, { "category": "paraphrase", "points": 3 }, { "category": "filler", "points": 4 } ] }, "judge_disagreement": 0 }, { "idx": 50, "duration_s": 106, "mean_score": 75.0, "mean_raw": 25.0, "scores_per_judge": [ 75 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "culture", "points": 8 }, { "category": "tech", "points": 8 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-minor", "points": 2 }, { "category": "filler", "points": 2 } ] }, "judge_disagreement": 0 }, { "idx": 51, "duration_s": 33, "mean_score": 93.0, "mean_raw": 7.0, "scores_per_judge": [ 93 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 }, { "category": "mishear-minor", "points": 2 } ] }, "judge_disagreement": 0 }, { "idx": 52, "duration_s": 25, "mean_score": 94.0, "mean_raw": 6.0, "scores_per_judge": [ 94 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-minor", "points": 2 }, { "category": "filler", "points": 4 } ] }, "judge_disagreement": 0 }, { "idx": 53, "duration_s": 35, "mean_score": 97.0, "mean_raw": 3.0, "scores_per_judge": [ 97 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 54, "duration_s": 36, "mean_score": 96.0, "mean_raw": 4.0, "scores_per_judge": [ 96 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 55, "duration_s": 33, "mean_score": 77.0, "mean_raw": 23.0, "scores_per_judge": [ 77 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "proper-noun", "points": 6 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-minor", "points": 2 }, { "category": "mishear-minor", "points": 2 }, { "category": "mishear-minor", "points": 2 }, { "category": "mishear-minor", "points": 2 }, { "category": "filler", "points": 4 } ] }, "judge_disagreement": 0 }, { "idx": 56, "duration_s": 15, "mean_score": 90.0, "mean_raw": 10.0, "scores_per_judge": [ 90 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "brand", "points": 10 } ] }, "judge_disagreement": 0 }, { "idx": 57, "duration_s": 38, "mean_score": 80.0, "mean_raw": 20.0, "scores_per_judge": [ 80 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "tech", "points": 8 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 58, "duration_s": 15, "mean_score": 98.0, "mean_raw": 2.0, "scores_per_judge": [ 98 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 59, "duration_s": 28, "mean_score": 92.0, "mean_raw": 8.0, "scores_per_judge": [ 92 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "tech", "points": 8 } ] }, "judge_disagreement": 0 }, { "idx": 60, "duration_s": 18, "mean_score": 79.0, "mean_raw": 21.0, "scores_per_judge": [ 79 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "semantic-inversion", "points": 20 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 61, "duration_s": 19, "mean_score": 95.0, "mean_raw": 5.0, "scores_per_judge": [ 95 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 62, "duration_s": 17, "mean_score": 100.0, "mean_raw": 0.0, "scores_per_judge": [ 100 ], "deductions_per_judge": { "deepseek_flash": [] }, "judge_disagreement": 0 }, { "idx": 63, "duration_s": 21, "mean_score": 98.0, "mean_raw": 2.0, "scores_per_judge": [ 98 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 64, "duration_s": 23, "mean_score": 95.0, "mean_raw": 5.0, "scores_per_judge": [ 95 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 65, "duration_s": 20, "mean_score": 100.0, "mean_raw": 0.0, "scores_per_judge": [ 100 ], "deductions_per_judge": { "deepseek_flash": [] }, "judge_disagreement": 0 }, { "idx": 66, "duration_s": 24, "mean_score": 91.0, "mean_raw": 9.0, "scores_per_judge": [ 91 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "culture", "points": 8 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 67, "duration_s": 16, "mean_score": 99.0, "mean_raw": 1.0, "scores_per_judge": [ 99 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 68, "duration_s": 18, "mean_score": 90.0, "mean_raw": 10.0, "scores_per_judge": [ 90 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "brand", "points": 10 } ] }, "judge_disagreement": 0 }, { "idx": 69, "duration_s": 18, "mean_score": 100.0, "mean_raw": 0.0, "scores_per_judge": [ 100 ], "deductions_per_judge": { "deepseek_flash": [] }, "judge_disagreement": 0 }, { "idx": 70, "duration_s": 17, "mean_score": 22.0, "mean_raw": 78.0, "scores_per_judge": [ 22 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "polarity", "points": 35 }, { "category": "polarity", "points": 35 }, { "category": "tech", "points": 8 } ] }, "judge_disagreement": 0 }, { "idx": 71, "duration_s": 51, "mean_score": 72.0, "mean_raw": 28.0, "scores_per_judge": [ 72 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "tech", "points": 8 }, { "category": "brand", "points": 10 }, { "category": "content-drop", "points": 3 }, { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 72, "duration_s": 54, "mean_score": 93.0, "mean_raw": 7.0, "scores_per_judge": [ 93 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 }, { "category": "mishear-minor", "points": 2 } ] }, "judge_disagreement": 0 }, { "idx": 73, "duration_s": 56, "mean_score": 84.0, "mean_raw": 16.0, "scores_per_judge": [ 84 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "brand", "points": 10 }, { "category": "paraphrase", "points": 3 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 74, "duration_s": 45, "mean_score": 95.0, "mean_raw": 5.0, "scores_per_judge": [ 95 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 75, "duration_s": 74, "mean_score": 95.0, "mean_raw": 5.0, "scores_per_judge": [ 95 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 76, "duration_s": 58, "mean_score": 76.0, "mean_raw": 24.0, "scores_per_judge": [ 76 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "semantic-inversion", "points": 20 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 77, "duration_s": 130, "mean_score": 59.0, "mean_raw": 41.0, "scores_per_judge": [ 59 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "semantic-inversion", "points": 20 }, { "category": "brand", "points": 10 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-minor", "points": 2 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 78, "duration_s": 61, "mean_score": 94.0, "mean_raw": 6.0, "scores_per_judge": [ 94 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 4 }, { "category": "mishear-minor", "points": 2 } ] }, "judge_disagreement": 0 }, { "idx": 79, "duration_s": 47, "mean_score": 95.0, "mean_raw": 5.0, "scores_per_judge": [ 95 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "paraphrase", "points": 3 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 80, "duration_s": 42, "mean_score": 97.0, "mean_raw": 3.0, "scores_per_judge": [ 97 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 81, "duration_s": 38, "mean_score": 87.0, "mean_raw": 13.0, "scores_per_judge": [ 87 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 }, { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 82, "duration_s": 38, "mean_score": 93.0, "mean_raw": 7.0, "scores_per_judge": [ 93 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "content-drop", "points": 3 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "format", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 83, "duration_s": 37, "mean_score": 96.0, "mean_raw": 4.0, "scores_per_judge": [ 96 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 84, "duration_s": 36, "mean_score": 96.0, "mean_raw": 4.0, "scores_per_judge": [ 96 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-minor", "points": 2 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 85, "duration_s": 36, "mean_score": 92.0, "mean_raw": 8.0, "scores_per_judge": [ 92 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 }, { "category": "mishear-minor", "points": 2 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 86, "duration_s": 35, "mean_score": 82.0, "mean_raw": 18.0, "scores_per_judge": [ 82 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "tech", "points": 8 }, { "category": "tech", "points": 8 }, { "category": "filler", "points": 1 }, { "category": "format", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 87, "duration_s": 34, "mean_score": 82.0, "mean_raw": 18.0, "scores_per_judge": [ 82 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "numeric", "points": 10 }, { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 88, "duration_s": 34, "mean_score": 99.0, "mean_raw": 1.0, "scores_per_judge": [ 99 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 89, "duration_s": 34, "mean_score": 100.0, "mean_raw": 0.0, "scores_per_judge": [ 100 ], "deductions_per_judge": { "deepseek_flash": [] }, "judge_disagreement": 0 }, { "idx": 90, "duration_s": 33, "mean_score": 90.0, "mean_raw": 10.0, "scores_per_judge": [ 90 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "tech", "points": 8 }, { "category": "filler", "points": 2 } ] }, "judge_disagreement": 0 }, { "idx": 91, "duration_s": 33, "mean_score": 63.0, "mean_raw": 37.0, "scores_per_judge": [ 63 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "polarity", "points": 35 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 92, "duration_s": 32, "mean_score": 100.0, "mean_raw": 0.0, "scores_per_judge": [ 100 ], "deductions_per_judge": { "deepseek_flash": [] }, "judge_disagreement": 0 }, { "idx": 93, "duration_s": 32, "mean_score": 68.0, "mean_raw": 32.0, "scores_per_judge": [ 68 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "tech", "points": 8 }, { "category": "content-drop", "points": 3 }, { "category": "mishear-severe", "points": 5 }, { "category": "hallucination", "points": 15 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 94, "duration_s": 32, "mean_score": 96.0, "mean_raw": 4.0, "scores_per_judge": [ 96 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 95, "duration_s": 32, "mean_score": 77.0, "mean_raw": 23.0, "scores_per_judge": [ 77 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "hallucination", "points": 15 }, { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 96, "duration_s": 31, "mean_score": 85.0, "mean_raw": 15.0, "scores_per_judge": [ 85 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "hallucination", "points": 15 } ] }, "judge_disagreement": 0 }, { "idx": 97, "duration_s": 30, "mean_score": 82.0, "mean_raw": 18.0, "scores_per_judge": [ 82 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "hallucination", "points": 15 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 98, "duration_s": 30, "mean_score": 79.0, "mean_raw": 21.0, "scores_per_judge": [ 79 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "hallucination", "points": 15 }, { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 99, "duration_s": 30, "mean_score": 85.0, "mean_raw": 15.0, "scores_per_judge": [ 85 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "hallucination", "points": 15 } ] }, "judge_disagreement": 0 }, { "idx": 100, "duration_s": 30, "mean_score": 97.0, "mean_raw": 3.0, "scores_per_judge": [ 97 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 } ], "total_deductions": 1297, "avg_deductions_per_sample": 12.97, "mean_clamped_score": 87.0, "n_judge_sample_pairs": 100, "gt_chars": 22625, "ded_per_100char": 5.73, "fidelity_score": 71.3, "overall_score": 87.0, "overall_raw_deductions": 12.97, "score_per_judge": { "deepseek_flash": 87.0 }, "per_group_deductions": { "meaning-flip": { "deepseek_flash": 380 }, "entity": { "deepseek_flash": 410 }, "cultural": { "deepseek_flash": 48 }, "fidelity": { "deepseek_flash": 459 } }, "per_bucket": { "general": { "n_samples": 40, "mean_raw": 14.6, "mean_score": 85.5, "total_raw": 582.0, "idxs": [ 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40 ], "worst_idx": 19 }, "idiom-test": { "n_samples": 5, "mean_raw": 8.0, "mean_score": 92.0, "total_raw": 40.0, "idxs": [ 41, 42, 43, 44, 45 ], "worst_idx": 44 }, "long-mixed": { "n_samples": 5, "mean_raw": 14.0, "mean_score": 86.0, "total_raw": 70.0, "idxs": [ 46, 47, 48, 49, 50 ], "worst_idx": 50 } }, "per_group_per_100char": { "meaning-flip": 1.68, "entity": 1.812, "cultural": 0.212, "fidelity": 2.029 }, "per_group_fidelity": { "meaning-flip": 59.7, "entity": 67.9, "cultural": 87.2, "fidelity": 76.2 } }, "vibevoice": { "scores_by_judge": { "deepseek_flash": [ 100, 100, 100, 100, 90, 99, 98, 99, 98, 99, 83, 98, 65, 78, 98, 91, 88, 86, 61, 98, 100, 95, 100, 90, 92, 99, 100, 98, 90, 99, 65, 98, 80, 59, 85, 96, 100, 85, 89, 79, 100, 92, 91, 92, 96, 98, 86, 90, 88, 70, 91, 90, 98, 97, 93, 75, 96, 98, 80, 63, 95, 100, 90, 95, 100, 75, 99, 100, 84, 98, 96, 92, 100, 96, 96, 77, 60, 94, 97, 98, 85, 86, 95, 62, 93, 79, 89, 97, 100, 91, 98, 99, 99, 95, 98, 100, 99, 99, 99, 96 ] }, "per_category_deductions": { "polarity": { "deepseek_flash": 140 }, "semantic-inversion": { "deepseek_flash": 60 }, "hallucination": { "deepseek_flash": 30 }, "numeric": { "deepseek_flash": 10 }, "brand": { "deepseek_flash": 130 }, "tech": { "deepseek_flash": 104 }, "culture": { "deepseek_flash": 56 }, "proper-noun": { "deepseek_flash": 24 }, "dialect": { "deepseek_flash": 0 }, "mishear-severe": { "deepseek_flash": 115 }, "content-drop": { "deepseek_flash": 3 }, "paraphrase": { "deepseek_flash": 9 }, "mishear-minor": { "deepseek_flash": 26 }, "filler": { "deepseek_flash": 180 }, "format": { "deepseek_flash": 0 }, "unscoreable": { "deepseek_flash": 0 } }, "per_tier_deductions": { "tier1": { "deepseek_flash": 240 }, "tier2": { "deepseek_flash": 314 }, "tier3": { "deepseek_flash": 153 }, "tier4": { "deepseek_flash": 180 }, "unscoreable": { "deepseek_flash": 0 } }, "raw_deductions_by_judge": { "deepseek_flash": [ 0, 0, 0, 0, 10, 1, 2, 1, 2, 1, 17, 2, 35, 22, 2, 9, 12, 14, 39, 2, 0, 5, 0, 10, 8, 1, 0, 2, 10, 1, 35, 2, 20, 41, 15, 4, 0, 15, 11, 21, 0, 8, 9, 8, 4, 2, 14, 10, 12, 30, 9, 10, 2, 3, 7, 25, 4, 2, 20, 37, 5, 0, 10, 5, 0, 25, 1, 0, 16, 2, 4, 8, 0, 4, 4, 23, 40, 6, 3, 2, 15, 14, 5, 38, 7, 21, 11, 3, 0, 9, 2, 1, 1, 5, 2, 0, 1, 1, 1, 4 ] }, "sample_details": [ { "idx": 1, "duration_s": 5, "mean_score": 100.0, "mean_raw": 0.0, "scores_per_judge": [ 100 ], "deductions_per_judge": { "deepseek_flash": [] }, "judge_disagreement": 0 }, { "idx": 2, "duration_s": 7, "mean_score": 100.0, "mean_raw": 0.0, "scores_per_judge": [ 100 ], "deductions_per_judge": { "deepseek_flash": [] }, "judge_disagreement": 0 }, { "idx": 3, "duration_s": 8, "mean_score": 100.0, "mean_raw": 0.0, "scores_per_judge": [ 100 ], "deductions_per_judge": { "deepseek_flash": [] }, "judge_disagreement": 0 }, { "idx": 4, "duration_s": 11, "mean_score": 100.0, "mean_raw": 0.0, "scores_per_judge": [ 100 ], "deductions_per_judge": { "deepseek_flash": [] }, "judge_disagreement": 0 }, { "idx": 5, "duration_s": 16, "mean_score": 90.0, "mean_raw": 10.0, "scores_per_judge": [ 90 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "culture", "points": 8 }, { "category": "filler", "points": 2 } ] }, "judge_disagreement": 0 }, { "idx": 6, "duration_s": 17, "mean_score": 99.0, "mean_raw": 1.0, "scores_per_judge": [ 99 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 7, "duration_s": 20, "mean_score": 98.0, "mean_raw": 2.0, "scores_per_judge": [ 98 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 8, "duration_s": 30, "mean_score": 99.0, "mean_raw": 1.0, "scores_per_judge": [ 99 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 9, "duration_s": 32, "mean_score": 98.0, "mean_raw": 2.0, "scores_per_judge": [ 98 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 10, "duration_s": 35, "mean_score": 99.0, "mean_raw": 1.0, "scores_per_judge": [ 99 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 11, "duration_s": 40, "mean_score": 83.0, "mean_raw": 17.0, "scores_per_judge": [ 83 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "tech", "points": 8 }, { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 4 } ] }, "judge_disagreement": 0 }, { "idx": 12, "duration_s": 46, "mean_score": 98.0, "mean_raw": 2.0, "scores_per_judge": [ 98 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 13, "duration_s": 60, "mean_score": 65.0, "mean_raw": 35.0, "scores_per_judge": [ 65 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "brand", "points": 10 }, { "category": "brand", "points": 10 }, { "category": "tech", "points": 8 }, { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 14, "duration_s": 64, "mean_score": 78.0, "mean_raw": 22.0, "scores_per_judge": [ 78 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "proper-noun", "points": 6 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-minor", "points": 2 }, { "category": "filler", "points": 4 } ] }, "judge_disagreement": 0 }, { "idx": 15, "duration_s": 69, "mean_score": 98.0, "mean_raw": 2.0, "scores_per_judge": [ 98 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 16, "duration_s": 81, "mean_score": 91.0, "mean_raw": 9.0, "scores_per_judge": [ 91 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "proper-noun", "points": 6 }, { "category": "content-drop", "points": 3 } ] }, "judge_disagreement": 0 }, { "idx": 17, "duration_s": 100, "mean_score": 88.0, "mean_raw": 12.0, "scores_per_judge": [ 88 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "tech", "points": 8 }, { "category": "filler", "points": 4 } ] }, "judge_disagreement": 0 }, { "idx": 18, "duration_s": 125, "mean_score": 86.0, "mean_raw": 14.0, "scores_per_judge": [ 86 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "brand", "points": 10 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 19, "duration_s": 133, "mean_score": 61.0, "mean_raw": 39.0, "scores_per_judge": [ 61 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "brand", "points": 10 }, { "category": "brand", "points": 10 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 4 } ] }, "judge_disagreement": 0 }, { "idx": 20, "duration_s": 164, "mean_score": 98.0, "mean_raw": 2.0, "scores_per_judge": [ 98 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-minor", "points": 2 } ] }, "judge_disagreement": 0 }, { "idx": 21, "duration_s": 5, "mean_score": 100.0, "mean_raw": 0.0, "scores_per_judge": [ 100 ], "deductions_per_judge": { "deepseek_flash": [] }, "judge_disagreement": 0 }, { "idx": 22, "duration_s": 7, "mean_score": 95.0, "mean_raw": 5.0, "scores_per_judge": [ 95 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 23, "duration_s": 10, "mean_score": 100.0, "mean_raw": 0.0, "scores_per_judge": [ 100 ], "deductions_per_judge": { "deepseek_flash": [] }, "judge_disagreement": 0 }, { "idx": 24, "duration_s": 14, "mean_score": 90.0, "mean_raw": 10.0, "scores_per_judge": [ 90 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "tech", "points": 8 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 25, "duration_s": 16, "mean_score": 92.0, "mean_raw": 8.0, "scores_per_judge": [ 92 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "tech", "points": 8 } ] }, "judge_disagreement": 0 }, { "idx": 26, "duration_s": 23, "mean_score": 99.0, "mean_raw": 1.0, "scores_per_judge": [ 99 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 27, "duration_s": 25, "mean_score": 100.0, "mean_raw": 0.0, "scores_per_judge": [ 100 ], "deductions_per_judge": { "deepseek_flash": [] }, "judge_disagreement": 0 }, { "idx": 28, "duration_s": 30, "mean_score": 98.0, "mean_raw": 2.0, "scores_per_judge": [ 98 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 29, "duration_s": 34, "mean_score": 90.0, "mean_raw": 10.0, "scores_per_judge": [ 90 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "brand", "points": 10 } ] }, "judge_disagreement": 0 }, { "idx": 30, "duration_s": 38, "mean_score": 99.0, "mean_raw": 1.0, "scores_per_judge": [ 99 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 31, "duration_s": 44, "mean_score": 65.0, "mean_raw": 35.0, "scores_per_judge": [ 65 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "polarity", "points": 35 } ] }, "judge_disagreement": 0 }, { "idx": 32, "duration_s": 50, "mean_score": 98.0, "mean_raw": 2.0, "scores_per_judge": [ 98 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 33, "duration_s": 62, "mean_score": 80.0, "mean_raw": 20.0, "scores_per_judge": [ 80 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "brand", "points": 10 }, { "category": "proper-noun", "points": 6 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 34, "duration_s": 66, "mean_score": 59.0, "mean_raw": 41.0, "scores_per_judge": [ 59 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "polarity", "points": 35 }, { "category": "proper-noun", "points": 6 } ] }, "judge_disagreement": 0 }, { "idx": 35, "duration_s": 77, "mean_score": 85.0, "mean_raw": 15.0, "scores_per_judge": [ 85 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "tech", "points": 8 }, { "category": "mishear-minor", "points": 2 }, { "category": "filler", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 36, "duration_s": 94, "mean_score": 96.0, "mean_raw": 4.0, "scores_per_judge": [ 96 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-minor", "points": 2 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 37, "duration_s": 106, "mean_score": 100.0, "mean_raw": 0.0, "scores_per_judge": [ 100 ], "deductions_per_judge": { "deepseek_flash": [] }, "judge_disagreement": 0 }, { "idx": 38, "duration_s": 126, "mean_score": 85.0, "mean_raw": 15.0, "scores_per_judge": [ 85 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "tech", "points": 8 }, { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 39, "duration_s": 158, "mean_score": 89.0, "mean_raw": 11.0, "scores_per_judge": [ 89 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "brand", "points": 10 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 40, "duration_s": 279, "mean_score": 79.0, "mean_raw": 21.0, "scores_per_judge": [ 79 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "culture", "points": 8 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-severe", "points": 5 }, { "category": "paraphrase", "points": 3 } ] }, "judge_disagreement": 0 }, { "idx": 41, "duration_s": 34, "mean_score": 100.0, "mean_raw": 0.0, "scores_per_judge": [ 100 ], "deductions_per_judge": { "deepseek_flash": [] }, "judge_disagreement": 0 }, { "idx": 42, "duration_s": 43, "mean_score": 92.0, "mean_raw": 8.0, "scores_per_judge": [ 92 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "culture", "points": 8 } ] }, "judge_disagreement": 0 }, { "idx": 43, "duration_s": 29, "mean_score": 91.0, "mean_raw": 9.0, "scores_per_judge": [ 91 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "culture", "points": 8 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 44, "duration_s": 71, "mean_score": 92.0, "mean_raw": 8.0, "scores_per_judge": [ 92 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 45, "duration_s": 46, "mean_score": 96.0, "mean_raw": 4.0, "scores_per_judge": [ 96 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "paraphrase", "points": 3 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 46, "duration_s": 62, "mean_score": 98.0, "mean_raw": 2.0, "scores_per_judge": [ 98 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 47, "duration_s": 67, "mean_score": 86.0, "mean_raw": 14.0, "scores_per_judge": [ 86 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "brand", "points": 10 }, { "category": "mishear-minor", "points": 2 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 48, "duration_s": 72, "mean_score": 90.0, "mean_raw": 10.0, "scores_per_judge": [ 90 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "tech", "points": 8 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 49, "duration_s": 87, "mean_score": 88.0, "mean_raw": 12.0, "scores_per_judge": [ 88 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "brand", "points": 10 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 50, "duration_s": 106, "mean_score": 70.0, "mean_raw": 30.0, "scores_per_judge": [ 70 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "semantic-inversion", "points": 20 }, { "category": "culture", "points": 8 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 51, "duration_s": 33, "mean_score": 91.0, "mean_raw": 9.0, "scores_per_judge": [ 91 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "culture", "points": 8 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 52, "duration_s": 25, "mean_score": 90.0, "mean_raw": 10.0, "scores_per_judge": [ 90 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 53, "duration_s": 35, "mean_score": 98.0, "mean_raw": 2.0, "scores_per_judge": [ 98 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 54, "duration_s": 36, "mean_score": 97.0, "mean_raw": 3.0, "scores_per_judge": [ 97 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 55, "duration_s": 33, "mean_score": 93.0, "mean_raw": 7.0, "scores_per_judge": [ 93 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "paraphrase", "points": 3 }, { "category": "filler", "points": 4 } ] }, "judge_disagreement": 0 }, { "idx": 56, "duration_s": 15, "mean_score": 75.0, "mean_raw": 25.0, "scores_per_judge": [ 75 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "brand", "points": 10 }, { "category": "brand", "points": 10 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 57, "duration_s": 38, "mean_score": 96.0, "mean_raw": 4.0, "scores_per_judge": [ 96 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 58, "duration_s": 15, "mean_score": 98.0, "mean_raw": 2.0, "scores_per_judge": [ 98 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 59, "duration_s": 28, "mean_score": 80.0, "mean_raw": 20.0, "scores_per_judge": [ 80 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "hallucination", "points": 15 }, { "category": "mishear-minor", "points": 2 }, { "category": "filler", "points": 3 } ] }, "judge_disagreement": 0 }, { "idx": 60, "duration_s": 18, "mean_score": 63.0, "mean_raw": 37.0, "scores_per_judge": [ 63 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "polarity", "points": 35 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 61, "duration_s": 19, "mean_score": 95.0, "mean_raw": 5.0, "scores_per_judge": [ 95 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 62, "duration_s": 17, "mean_score": 100.0, "mean_raw": 0.0, "scores_per_judge": [ 100 ], "deductions_per_judge": { "deepseek_flash": [] }, "judge_disagreement": 0 }, { "idx": 63, "duration_s": 21, "mean_score": 90.0, "mean_raw": 10.0, "scores_per_judge": [ 90 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "tech", "points": 8 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 64, "duration_s": 23, "mean_score": 95.0, "mean_raw": 5.0, "scores_per_judge": [ 95 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 65, "duration_s": 20, "mean_score": 100.0, "mean_raw": 0.0, "scores_per_judge": [ 100 ], "deductions_per_judge": { "deepseek_flash": [] }, "judge_disagreement": 0 }, { "idx": 66, "duration_s": 24, "mean_score": 75.0, "mean_raw": 25.0, "scores_per_judge": [ 75 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "tech", "points": 8 }, { "category": "culture", "points": 8 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-minor", "points": 2 }, { "category": "filler", "points": 2 } ] }, "judge_disagreement": 0 }, { "idx": 67, "duration_s": 16, "mean_score": 99.0, "mean_raw": 1.0, "scores_per_judge": [ 99 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 68, "duration_s": 18, "mean_score": 100.0, "mean_raw": 0.0, "scores_per_judge": [ 100 ], "deductions_per_judge": { "deepseek_flash": [] }, "judge_disagreement": 0 }, { "idx": 69, "duration_s": 18, "mean_score": 84.0, "mean_raw": 16.0, "scores_per_judge": [ 84 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "hallucination", "points": 15 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 70, "duration_s": 17, "mean_score": 98.0, "mean_raw": 2.0, "scores_per_judge": [ 98 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 71, "duration_s": 51, "mean_score": 96.0, "mean_raw": 4.0, "scores_per_judge": [ 96 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 72, "duration_s": 54, "mean_score": 92.0, "mean_raw": 8.0, "scores_per_judge": [ 92 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 }, { "category": "mishear-minor", "points": 2 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 73, "duration_s": 56, "mean_score": 100.0, "mean_raw": 0.0, "scores_per_judge": [ 100 ], "deductions_per_judge": { "deepseek_flash": [] }, "judge_disagreement": 0 }, { "idx": 74, "duration_s": 45, "mean_score": 96.0, "mean_raw": 4.0, "scores_per_judge": [ 96 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-minor", "points": 2 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 75, "duration_s": 74, "mean_score": 96.0, "mean_raw": 4.0, "scores_per_judge": [ 96 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 76, "duration_s": 58, "mean_score": 77.0, "mean_raw": 23.0, "scores_per_judge": [ 77 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "semantic-inversion", "points": 20 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 77, "duration_s": 130, "mean_score": 60.0, "mean_raw": 40.0, "scores_per_judge": [ 60 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "semantic-inversion", "points": 20 }, { "category": "brand", "points": 10 }, { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 78, "duration_s": 61, "mean_score": 94.0, "mean_raw": 6.0, "scores_per_judge": [ 94 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "mishear-minor", "points": 2 } ] }, "judge_disagreement": 0 }, { "idx": 79, "duration_s": 47, "mean_score": 97.0, "mean_raw": 3.0, "scores_per_judge": [ 97 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 80, "duration_s": 42, "mean_score": 98.0, "mean_raw": 2.0, "scores_per_judge": [ 98 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 81, "duration_s": 38, "mean_score": 85.0, "mean_raw": 15.0, "scores_per_judge": [ 85 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-minor", "points": 2 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 82, "duration_s": 38, "mean_score": 86.0, "mean_raw": 14.0, "scores_per_judge": [ 86 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "tech", "points": 8 }, { "category": "mishear-severe", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 83, "duration_s": 37, "mean_score": 95.0, "mean_raw": 5.0, "scores_per_judge": [ 95 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "mishear-minor", "points": 2 } ] }, "judge_disagreement": 0 }, { "idx": 84, "duration_s": 36, "mean_score": 62.0, "mean_raw": 38.0, "scores_per_judge": [ 62 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "polarity", "points": 35 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 85, "duration_s": 36, "mean_score": 93.0, "mean_raw": 7.0, "scores_per_judge": [ 93 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 2 } ] }, "judge_disagreement": 0 }, { "idx": 86, "duration_s": 35, "mean_score": 79.0, "mean_raw": 21.0, "scores_per_judge": [ 79 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "numeric", "points": 10 }, { "category": "tech", "points": 8 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 87, "duration_s": 34, "mean_score": 89.0, "mean_raw": 11.0, "scores_per_judge": [ 89 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 }, { "category": "mishear-minor", "points": 2 }, { "category": "filler", "points": 4 } ] }, "judge_disagreement": 0 }, { "idx": 88, "duration_s": 34, "mean_score": 97.0, "mean_raw": 3.0, "scores_per_judge": [ 97 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 3 } ] }, "judge_disagreement": 0 }, { "idx": 89, "duration_s": 34, "mean_score": 100.0, "mean_raw": 0.0, "scores_per_judge": [ 100 ], "deductions_per_judge": { "deepseek_flash": [] }, "judge_disagreement": 0 }, { "idx": 90, "duration_s": 33, "mean_score": 91.0, "mean_raw": 9.0, "scores_per_judge": [ 91 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "tech", "points": 8 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 91, "duration_s": 33, "mean_score": 98.0, "mean_raw": 2.0, "scores_per_judge": [ 98 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 92, "duration_s": 32, "mean_score": 99.0, "mean_raw": 1.0, "scores_per_judge": [ 99 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 93, "duration_s": 32, "mean_score": 99.0, "mean_raw": 1.0, "scores_per_judge": [ 99 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 94, "duration_s": 32, "mean_score": 95.0, "mean_raw": 5.0, "scores_per_judge": [ 95 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 95, "duration_s": 32, "mean_score": 98.0, "mean_raw": 2.0, "scores_per_judge": [ 98 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 96, "duration_s": 31, "mean_score": 100.0, "mean_raw": 0.0, "scores_per_judge": [ 100 ], "deductions_per_judge": { "deepseek_flash": [] }, "judge_disagreement": 0 }, { "idx": 97, "duration_s": 30, "mean_score": 99.0, "mean_raw": 1.0, "scores_per_judge": [ 99 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 98, "duration_s": 30, "mean_score": 99.0, "mean_raw": 1.0, "scores_per_judge": [ 99 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 99, "duration_s": 30, "mean_score": 99.0, "mean_raw": 1.0, "scores_per_judge": [ 99 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 100, "duration_s": 30, "mean_score": 96.0, "mean_raw": 4.0, "scores_per_judge": [ 96 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 } ], "total_deductions": 887, "avg_deductions_per_sample": 8.87, "mean_clamped_score": 91.1, "n_judge_sample_pairs": 100, "gt_chars": 22625, "ded_per_100char": 3.92, "fidelity_score": 80.4, "overall_score": 91.1, "overall_raw_deductions": 8.87, "score_per_judge": { "deepseek_flash": 91.1 }, "per_group_deductions": { "meaning-flip": { "deepseek_flash": 240 }, "entity": { "deepseek_flash": 258 }, "cultural": { "deepseek_flash": 56 }, "fidelity": { "deepseek_flash": 333 } }, "per_bucket": { "general": { "n_samples": 40, "mean_raw": 9.3, "mean_score": 90.7, "total_raw": 372.0, "idxs": [ 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40 ], "worst_idx": 34 }, "idiom-test": { "n_samples": 5, "mean_raw": 5.8, "mean_score": 94.2, "total_raw": 29.0, "idxs": [ 41, 42, 43, 44, 45 ], "worst_idx": 43 }, "long-mixed": { "n_samples": 5, "mean_raw": 13.6, "mean_score": 86.4, "total_raw": 68.0, "idxs": [ 46, 47, 48, 49, 50 ], "worst_idx": 50 } }, "per_group_per_100char": { "meaning-flip": 1.061, "entity": 1.14, "cultural": 0.248, "fidelity": 1.472 }, "per_group_fidelity": { "meaning-flip": 74.6, "entity": 79.8, "cultural": 85.0, "fidelity": 82.8 } }, "voxtral": { "scores_by_judge": { "deepseek_flash": [ 94, 100, 100, 99, 97, 95, 89, 92, 90, 98, 77, 68, 60, 36, 88, 69, 77, 60, 67, 64, 100, 94, 100, 75, 87, 92, 95, 78, 79, 49, 62, 62, 77, 44, 79, 61, 95, 26, 57, 60, 55, 61, 72, 65, 36, 85, 84, 79, 67, 66, 37, 75, 90, 90, 95, 77, 38, 90, 64, 58, 82, 70, 97, 95, 98, 94, 97, 72, 81, 89, 64, 89, 100, 95, 82, 47, 58, 95, 92, 96, 76, 88, 89, 96, 83, 96, 95, 99, 96, 95, 89, 99, 78, 88, 86, 99, 95, 97, 98, 86 ] }, "per_category_deductions": { "polarity": { "deepseek_flash": 280 }, "semantic-inversion": { "deepseek_flash": 80 }, "hallucination": { "deepseek_flash": 135 }, "numeric": { "deepseek_flash": 10 }, "brand": { "deepseek_flash": 180 }, "tech": { "deepseek_flash": 152 }, "culture": { "deepseek_flash": 208 }, "proper-noun": { "deepseek_flash": 96 }, "dialect": { "deepseek_flash": 35 }, "mishear-severe": { "deepseek_flash": 375 }, "content-drop": { "deepseek_flash": 75 }, "paraphrase": { "deepseek_flash": 15 }, "mishear-minor": { "deepseek_flash": 48 }, "filler": { "deepseek_flash": 313 }, "format": { "deepseek_flash": 1 }, "unscoreable": { "deepseek_flash": 0 } }, "per_tier_deductions": { "tier1": { "deepseek_flash": 505 }, "tier2": { "deepseek_flash": 671 }, "tier3": { "deepseek_flash": 513 }, "tier4": { "deepseek_flash": 314 }, "unscoreable": { "deepseek_flash": 0 } }, "raw_deductions_by_judge": { "deepseek_flash": [ 6, 0, 0, 1, 3, 5, 11, 8, 10, 2, 23, 32, 40, 64, 12, 31, 23, 40, 33, 36, 0, 6, 0, 25, 13, 8, 5, 22, 21, 51, 38, 38, 23, 56, 21, 39, 5, 74, 43, 40, 45, 39, 28, 35, 64, 15, 16, 21, 33, 34, 63, 25, 10, 10, 5, 23, 62, 10, 36, 42, 18, 30, 3, 5, 2, 6, 3, 28, 19, 11, 36, 11, 0, 5, 18, 53, 42, 5, 8, 4, 24, 12, 11, 4, 17, 4, 5, 1, 4, 5, 11, 1, 22, 12, 14, 1, 5, 3, 2, 14 ] }, "sample_details": [ { "idx": 1, "duration_s": 5, "mean_score": 94.0, "mean_raw": 6.0, "scores_per_judge": [ 94 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 2, "duration_s": 7, "mean_score": 100.0, "mean_raw": 0.0, "scores_per_judge": [ 100 ], "deductions_per_judge": { "deepseek_flash": [] }, "judge_disagreement": 0 }, { "idx": 3, "duration_s": 8, "mean_score": 100.0, "mean_raw": 0.0, "scores_per_judge": [ 100 ], "deductions_per_judge": { "deepseek_flash": [] }, "judge_disagreement": 0 }, { "idx": 4, "duration_s": 11, "mean_score": 99.0, "mean_raw": 1.0, "scores_per_judge": [ 99 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 5, "duration_s": 16, "mean_score": 97.0, "mean_raw": 3.0, "scores_per_judge": [ 97 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 6, "duration_s": 17, "mean_score": 95.0, "mean_raw": 5.0, "scores_per_judge": [ 95 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "paraphrase", "points": 3 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 7, "duration_s": 20, "mean_score": 89.0, "mean_raw": 11.0, "scores_per_judge": [ 89 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 }, { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 8, "duration_s": 30, "mean_score": 92.0, "mean_raw": 8.0, "scores_per_judge": [ 92 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "content-drop", "points": 3 }, { "category": "filler", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 9, "duration_s": 32, "mean_score": 90.0, "mean_raw": 10.0, "scores_per_judge": [ 90 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 10, "duration_s": 35, "mean_score": 98.0, "mean_raw": 2.0, "scores_per_judge": [ 98 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 2 } ] }, "judge_disagreement": 0 }, { "idx": 11, "duration_s": 40, "mean_score": 77.0, "mean_raw": 23.0, "scores_per_judge": [ 77 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-severe", "points": 5 }, { "category": "paraphrase", "points": 3 }, { "category": "filler", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 12, "duration_s": 46, "mean_score": 68.0, "mean_raw": 32.0, "scores_per_judge": [ 68 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "semantic-inversion", "points": 20 }, { "category": "culture", "points": 8 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 13, "duration_s": 60, "mean_score": 60.0, "mean_raw": 40.0, "scores_per_judge": [ 60 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "brand", "points": 10 }, { "category": "brand", "points": 10 }, { "category": "tech", "points": 8 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-minor", "points": 2 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 14, "duration_s": 64, "mean_score": 36.0, "mean_raw": 64.0, "scores_per_judge": [ 36 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "hallucination", "points": 15 }, { "category": "proper-noun", "points": 6 }, { "category": "content-drop", "points": 3 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-severe", "points": 5 }, { "category": "proper-noun", "points": 6 }, { "category": "proper-noun", "points": 6 }, { "category": "mishear-severe", "points": 5 }, { "category": "content-drop", "points": 3 }, { "category": "content-drop", "points": 3 }, { "category": "mishear-minor", "points": 2 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 2 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 15, "duration_s": 69, "mean_score": 88.0, "mean_raw": 12.0, "scores_per_judge": [ 88 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 }, { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 2 } ] }, "judge_disagreement": 0 }, { "idx": 16, "duration_s": 81, "mean_score": 69.0, "mean_raw": 31.0, "scores_per_judge": [ 69 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 5 }, { "category": "mishear-minor", "points": 2 }, { "category": "mishear-severe", "points": 5 }, { "category": "proper-noun", "points": 6 }, { "category": "mishear-severe", "points": 5 }, { "category": "tech", "points": 8 } ] }, "judge_disagreement": 0 }, { "idx": 17, "duration_s": 100, "mean_score": 77.0, "mean_raw": 23.0, "scores_per_judge": [ 77 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "brand", "points": 10 }, { "category": "tech", "points": 8 }, { "category": "filler", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 18, "duration_s": 125, "mean_score": 60.0, "mean_raw": 40.0, "scores_per_judge": [ 60 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "hallucination", "points": 15 }, { "category": "brand", "points": 10 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 19, "duration_s": 133, "mean_score": 67.0, "mean_raw": 33.0, "scores_per_judge": [ 67 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "brand", "points": 10 }, { "category": "brand", "points": 10 }, { "category": "tech", "points": 8 }, { "category": "filler", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 20, "duration_s": 164, "mean_score": 64.0, "mean_raw": 36.0, "scores_per_judge": [ 64 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 }, { "category": "mishear-severe", "points": 5 }, { "category": "culture", "points": 8 }, { "category": "mishear-severe", "points": 5 }, { "category": "culture", "points": 8 }, { "category": "filler", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 21, "duration_s": 5, "mean_score": 100.0, "mean_raw": 0.0, "scores_per_judge": [ 100 ], "deductions_per_judge": { "deepseek_flash": [] }, "judge_disagreement": 0 }, { "idx": 22, "duration_s": 7, "mean_score": 94.0, "mean_raw": 6.0, "scores_per_judge": [ 94 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "proper-noun", "points": 6 } ] }, "judge_disagreement": 0 }, { "idx": 23, "duration_s": 10, "mean_score": 100.0, "mean_raw": 0.0, "scores_per_judge": [ 100 ], "deductions_per_judge": { "deepseek_flash": [] }, "judge_disagreement": 0 }, { "idx": 24, "duration_s": 14, "mean_score": 75.0, "mean_raw": 25.0, "scores_per_judge": [ 75 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "hallucination", "points": 15 }, { "category": "tech", "points": 8 }, { "category": "filler", "points": 2 } ] }, "judge_disagreement": 0 }, { "idx": 25, "duration_s": 16, "mean_score": 87.0, "mean_raw": 13.0, "scores_per_judge": [ 87 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "tech", "points": 8 }, { "category": "content-drop", "points": 3 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 26, "duration_s": 23, "mean_score": 92.0, "mean_raw": 8.0, "scores_per_judge": [ 92 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 }, { "category": "mishear-minor", "points": 2 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 27, "duration_s": 25, "mean_score": 95.0, "mean_raw": 5.0, "scores_per_judge": [ 95 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 3 }, { "category": "mishear-minor", "points": 2 } ] }, "judge_disagreement": 0 }, { "idx": 28, "duration_s": 30, "mean_score": 78.0, "mean_raw": 22.0, "scores_per_judge": [ 78 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 29, "duration_s": 34, "mean_score": 79.0, "mean_raw": 21.0, "scores_per_judge": [ 79 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "brand", "points": 10 }, { "category": "tech", "points": 8 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 30, "duration_s": 38, "mean_score": 49.0, "mean_raw": 51.0, "scores_per_judge": [ 49 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "polarity", "points": 35 }, { "category": "content-drop", "points": 3 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 31, "duration_s": 44, "mean_score": 62.0, "mean_raw": 38.0, "scores_per_judge": [ 62 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "polarity", "points": 35 }, { "category": "filler", "points": 3 } ] }, "judge_disagreement": 0 }, { "idx": 32, "duration_s": 50, "mean_score": 62.0, "mean_raw": 38.0, "scores_per_judge": [ 62 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "brand", "points": 10 }, { "category": "brand", "points": 10 }, { "category": "tech", "points": 8 }, { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 33, "duration_s": 62, "mean_score": 77.0, "mean_raw": 23.0, "scores_per_judge": [ 77 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "brand", "points": 10 }, { "category": "proper-noun", "points": 6 }, { "category": "mishear-minor", "points": 2 }, { "category": "filler", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 34, "duration_s": 66, "mean_score": 44.0, "mean_raw": 56.0, "scores_per_judge": [ 44 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "polarity", "points": 35 }, { "category": "proper-noun", "points": 6 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 35, "duration_s": 77, "mean_score": 79.0, "mean_raw": 21.0, "scores_per_judge": [ 79 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "tech", "points": 8 }, { "category": "tech", "points": 8 }, { "category": "filler", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 36, "duration_s": 94, "mean_score": 61.0, "mean_raw": 39.0, "scores_per_judge": [ 61 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "polarity", "points": 35 }, { "category": "filler", "points": 4 } ] }, "judge_disagreement": 0 }, { "idx": 37, "duration_s": 106, "mean_score": 95.0, "mean_raw": 5.0, "scores_per_judge": [ 95 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 38, "duration_s": 126, "mean_score": 26.0, "mean_raw": 74.0, "scores_per_judge": [ 26 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "polarity", "points": 35 }, { "category": "tech", "points": 8 }, { "category": "culture", "points": 8 }, { "category": "numeric", "points": 10 }, { "category": "content-drop", "points": 3 }, { "category": "mishear-minor", "points": 2 }, { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 39, "duration_s": 158, "mean_score": 57.0, "mean_raw": 43.0, "scores_per_judge": [ 57 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "brand", "points": 10 }, { "category": "tech", "points": 8 }, { "category": "tech", "points": 8 }, { "category": "proper-noun", "points": 6 }, { "category": "proper-noun", "points": 6 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 40, "duration_s": 279, "mean_score": 60.0, "mean_raw": 40.0, "scores_per_judge": [ 60 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "content-drop", "points": 3 }, { "category": "culture", "points": 8 }, { "category": "mishear-minor", "points": 2 }, { "category": "mishear-minor", "points": 2 }, { "category": "mishear-severe", "points": 5 }, { "category": "culture", "points": 8 }, { "category": "mishear-minor", "points": 2 }, { "category": "dialect", "points": 5 }, { "category": "filler", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 41, "duration_s": 34, "mean_score": 55.0, "mean_raw": 45.0, "scores_per_judge": [ 55 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "culture", "points": 8 }, { "category": "culture", "points": 8 }, { "category": "culture", "points": 8 }, { "category": "culture", "points": 8 }, { "category": "culture", "points": 8 }, { "category": "filler", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 42, "duration_s": 43, "mean_score": 61.0, "mean_raw": 39.0, "scores_per_judge": [ 61 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 }, { "category": "dialect", "points": 5 }, { "category": "culture", "points": 8 }, { "category": "culture", "points": 8 }, { "category": "culture", "points": 8 }, { "category": "filler", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 43, "duration_s": 29, "mean_score": 72.0, "mean_raw": 28.0, "scores_per_judge": [ 72 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "hallucination", "points": 15 }, { "category": "content-drop", "points": 3 }, { "category": "culture", "points": 8 }, { "category": "mishear-minor", "points": 2 } ] }, "judge_disagreement": 0 }, { "idx": 44, "duration_s": 71, "mean_score": 65.0, "mean_raw": 35.0, "scores_per_judge": [ 65 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 }, { "category": "mishear-severe", "points": 5 }, { "category": "proper-noun", "points": 6 }, { "category": "proper-noun", "points": 6 }, { "category": "culture", "points": 8 }, { "category": "filler", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 45, "duration_s": 46, "mean_score": 36.0, "mean_raw": 64.0, "scores_per_judge": [ 36 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "dialect", "points": 5 }, { "category": "dialect", "points": 5 }, { "category": "dialect", "points": 5 }, { "category": "dialect", "points": 5 }, { "category": "dialect", "points": 5 }, { "category": "culture", "points": 8 }, { "category": "culture", "points": 8 }, { "category": "culture", "points": 8 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-minor", "points": 2 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 46, "duration_s": 62, "mean_score": 85.0, "mean_raw": 15.0, "scores_per_judge": [ 85 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "culture", "points": 8 }, { "category": "mishear-minor", "points": 2 }, { "category": "filler", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 47, "duration_s": 67, "mean_score": 84.0, "mean_raw": 16.0, "scores_per_judge": [ 84 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "brand", "points": 10 }, { "category": "mishear-minor", "points": 2 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 48, "duration_s": 72, "mean_score": 79.0, "mean_raw": 21.0, "scores_per_judge": [ 79 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "tech", "points": 8 }, { "category": "tech", "points": 8 }, { "category": "filler", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 49, "duration_s": 87, "mean_score": 67.0, "mean_raw": 33.0, "scores_per_judge": [ 67 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "brand", "points": 10 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-severe", "points": 5 }, { "category": "paraphrase", "points": 3 }, { "category": "filler", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 50, "duration_s": 106, "mean_score": 66.0, "mean_raw": 34.0, "scores_per_judge": [ 66 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 }, { "category": "culture", "points": 8 }, { "category": "culture", "points": 8 }, { "category": "tech", "points": 8 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 51, "duration_s": 33, "mean_score": 37.0, "mean_raw": 63.0, "scores_per_judge": [ 37 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "polarity", "points": 35 }, { "category": "culture", "points": 8 }, { "category": "proper-noun", "points": 6 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 52, "duration_s": 25, "mean_score": 75.0, "mean_raw": 25.0, "scores_per_judge": [ 75 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "proper-noun", "points": 6 }, { "category": "proper-noun", "points": 6 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 53, "duration_s": 35, "mean_score": 90.0, "mean_raw": 10.0, "scores_per_judge": [ 90 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 54, "duration_s": 36, "mean_score": 90.0, "mean_raw": 10.0, "scores_per_judge": [ 90 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 55, "duration_s": 33, "mean_score": 95.0, "mean_raw": 5.0, "scores_per_judge": [ 95 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 56, "duration_s": 15, "mean_score": 77.0, "mean_raw": 23.0, "scores_per_judge": [ 77 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "brand", "points": 10 }, { "category": "brand", "points": 10 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 57, "duration_s": 38, "mean_score": 38.0, "mean_raw": 62.0, "scores_per_judge": [ 38 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "polarity", "points": 35 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-severe", "points": 5 }, { "category": "content-drop", "points": 3 }, { "category": "content-drop", "points": 3 }, { "category": "content-drop", "points": 3 }, { "category": "content-drop", "points": 3 } ] }, "judge_disagreement": 0 }, { "idx": 58, "duration_s": 15, "mean_score": 90.0, "mean_raw": 10.0, "scores_per_judge": [ 90 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 }, { "category": "mishear-severe", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 59, "duration_s": 28, "mean_score": 64.0, "mean_raw": 36.0, "scores_per_judge": [ 64 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "hallucination", "points": 15 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-severe", "points": 5 }, { "category": "content-drop", "points": 3 }, { "category": "content-drop", "points": 3 } ] }, "judge_disagreement": 0 }, { "idx": 60, "duration_s": 18, "mean_score": 58.0, "mean_raw": 42.0, "scores_per_judge": [ 58 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "polarity", "points": 35 }, { "category": "content-drop", "points": 3 }, { "category": "filler", "points": 3 }, { "category": "format", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 61, "duration_s": 19, "mean_score": 82.0, "mean_raw": 18.0, "scores_per_judge": [ 82 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 2 }, { "category": "mishear-severe", "points": 5 }, { "category": "content-drop", "points": 3 }, { "category": "mishear-severe", "points": 5 }, { "category": "content-drop", "points": 3 } ] }, "judge_disagreement": 0 }, { "idx": 62, "duration_s": 17, "mean_score": 70.0, "mean_raw": 30.0, "scores_per_judge": [ 70 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "hallucination", "points": 15 }, { "category": "hallucination", "points": 15 } ] }, "judge_disagreement": 0 }, { "idx": 63, "duration_s": 21, "mean_score": 97.0, "mean_raw": 3.0, "scores_per_judge": [ 97 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 64, "duration_s": 23, "mean_score": 95.0, "mean_raw": 5.0, "scores_per_judge": [ 95 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 65, "duration_s": 20, "mean_score": 98.0, "mean_raw": 2.0, "scores_per_judge": [ 98 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 66, "duration_s": 24, "mean_score": 94.0, "mean_raw": 6.0, "scores_per_judge": [ 94 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "paraphrase", "points": 3 }, { "category": "content-drop", "points": 3 } ] }, "judge_disagreement": 0 }, { "idx": 67, "duration_s": 16, "mean_score": 97.0, "mean_raw": 3.0, "scores_per_judge": [ 97 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-minor", "points": 2 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 68, "duration_s": 18, "mean_score": 72.0, "mean_raw": 28.0, "scores_per_judge": [ 72 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "hallucination", "points": 15 }, { "category": "brand", "points": 10 }, { "category": "mishear-minor", "points": 2 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 69, "duration_s": 18, "mean_score": 81.0, "mean_raw": 19.0, "scores_per_judge": [ 81 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "hallucination", "points": 15 }, { "category": "content-drop", "points": 3 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 70, "duration_s": 17, "mean_score": 89.0, "mean_raw": 11.0, "scores_per_judge": [ 89 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 1 }, { "category": "content-drop", "points": 3 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 71, "duration_s": 51, "mean_score": 64.0, "mean_raw": 36.0, "scores_per_judge": [ 64 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 }, { "category": "mishear-minor", "points": 2 }, { "category": "mishear-severe", "points": 5 }, { "category": "brand", "points": 10 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 4 } ] }, "judge_disagreement": 0 }, { "idx": 72, "duration_s": 54, "mean_score": 89.0, "mean_raw": 11.0, "scores_per_judge": [ 89 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 }, { "category": "mishear-minor", "points": 2 }, { "category": "filler", "points": 4 } ] }, "judge_disagreement": 0 }, { "idx": 73, "duration_s": 56, "mean_score": 100.0, "mean_raw": 0.0, "scores_per_judge": [ 100 ], "deductions_per_judge": { "deepseek_flash": [] }, "judge_disagreement": 0 }, { "idx": 74, "duration_s": 45, "mean_score": 95.0, "mean_raw": 5.0, "scores_per_judge": [ 95 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 75, "duration_s": 74, "mean_score": 82.0, "mean_raw": 18.0, "scores_per_judge": [ 82 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "tech", "points": 8 }, { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 76, "duration_s": 58, "mean_score": 47.0, "mean_raw": 53.0, "scores_per_judge": [ 47 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "semantic-inversion", "points": 20 }, { "category": "culture", "points": 8 }, { "category": "culture", "points": 8 }, { "category": "proper-noun", "points": 6 }, { "category": "proper-noun", "points": 6 }, { "category": "filler", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 77, "duration_s": 130, "mean_score": 58.0, "mean_raw": 42.0, "scores_per_judge": [ 58 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "semantic-inversion", "points": 20 }, { "category": "brand", "points": 10 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-minor", "points": 2 }, { "category": "filler", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 78, "duration_s": 61, "mean_score": 95.0, "mean_raw": 5.0, "scores_per_judge": [ 95 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 79, "duration_s": 47, "mean_score": 92.0, "mean_raw": 8.0, "scores_per_judge": [ 92 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "content-drop", "points": 3 }, { "category": "filler", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 80, "duration_s": 42, "mean_score": 96.0, "mean_raw": 4.0, "scores_per_judge": [ 96 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 81, "duration_s": 38, "mean_score": 76.0, "mean_raw": 24.0, "scores_per_judge": [ 76 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "semantic-inversion", "points": 20 }, { "category": "paraphrase", "points": 3 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 82, "duration_s": 38, "mean_score": 88.0, "mean_raw": 12.0, "scores_per_judge": [ 88 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "tech", "points": 8 }, { "category": "mishear-minor", "points": 2 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 83, "duration_s": 37, "mean_score": 89.0, "mean_raw": 11.0, "scores_per_judge": [ 89 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-minor", "points": 2 } ] }, "judge_disagreement": 0 }, { "idx": 84, "duration_s": 36, "mean_score": 96.0, "mean_raw": 4.0, "scores_per_judge": [ 96 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 85, "duration_s": 36, "mean_score": 83.0, "mean_raw": 17.0, "scores_per_judge": [ 83 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 }, { "category": "mishear-minor", "points": 2 }, { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 3 } ] }, "judge_disagreement": 0 }, { "idx": 86, "duration_s": 35, "mean_score": 96.0, "mean_raw": 4.0, "scores_per_judge": [ 96 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 87, "duration_s": 34, "mean_score": 95.0, "mean_raw": 5.0, "scores_per_judge": [ 95 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 88, "duration_s": 34, "mean_score": 99.0, "mean_raw": 1.0, "scores_per_judge": [ 99 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 89, "duration_s": 34, "mean_score": 96.0, "mean_raw": 4.0, "scores_per_judge": [ 96 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "mishear-minor", "points": 2 } ] }, "judge_disagreement": 0 }, { "idx": 90, "duration_s": 33, "mean_score": 95.0, "mean_raw": 5.0, "scores_per_judge": [ 95 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 91, "duration_s": 33, "mean_score": 89.0, "mean_raw": 11.0, "scores_per_judge": [ 89 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "culture", "points": 8 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 92, "duration_s": 32, "mean_score": 99.0, "mean_raw": 1.0, "scores_per_judge": [ 99 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 93, "duration_s": 32, "mean_score": 78.0, "mean_raw": 22.0, "scores_per_judge": [ 78 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "tech", "points": 8 }, { "category": "content-drop", "points": 3 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 94, "duration_s": 32, "mean_score": 88.0, "mean_raw": 12.0, "scores_per_judge": [ 88 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 }, { "category": "mishear-minor", "points": 2 }, { "category": "filler", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 95, "duration_s": 32, "mean_score": 86.0, "mean_raw": 14.0, "scores_per_judge": [ 86 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 }, { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 4 } ] }, "judge_disagreement": 0 }, { "idx": 96, "duration_s": 31, "mean_score": 99.0, "mean_raw": 1.0, "scores_per_judge": [ 99 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 97, "duration_s": 30, "mean_score": 95.0, "mean_raw": 5.0, "scores_per_judge": [ 95 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 98, "duration_s": 30, "mean_score": 97.0, "mean_raw": 3.0, "scores_per_judge": [ 97 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 99, "duration_s": 30, "mean_score": 98.0, "mean_raw": 2.0, "scores_per_judge": [ 98 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 100, "duration_s": 30, "mean_score": 86.0, "mean_raw": 14.0, "scores_per_judge": [ 86 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "content-drop", "points": 3 }, { "category": "content-drop", "points": 3 }, { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 } ], "total_deductions": 2003, "avg_deductions_per_sample": 20.03, "mean_clamped_score": 80.0, "n_judge_sample_pairs": 100, "gt_chars": 22625, "ded_per_100char": 8.85, "fidelity_score": 55.7, "overall_score": 80.0, "overall_raw_deductions": 20.03, "score_per_judge": { "deepseek_flash": 80.0 }, "per_group_deductions": { "meaning-flip": { "deepseek_flash": 505 }, "entity": { "deepseek_flash": 428 }, "cultural": { "deepseek_flash": 243 }, "fidelity": { "deepseek_flash": 827 } }, "per_bucket": { "general": { "n_samples": 40, "mean_raw": 22.7, "mean_score": 77.3, "total_raw": 908.0, "idxs": [ 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40 ], "worst_idx": 38 }, "idiom-test": { "n_samples": 5, "mean_raw": 42.2, "mean_score": 57.8, "total_raw": 211.0, "idxs": [ 41, 42, 43, 44, 45 ], "worst_idx": 45 }, "long-mixed": { "n_samples": 5, "mean_raw": 23.8, "mean_score": 76.2, "total_raw": 119.0, "idxs": [ 46, 47, 48, 49, 50 ], "worst_idx": 50 } }, "per_group_per_100char": { "meaning-flip": 2.232, "entity": 1.892, "cultural": 1.074, "fidelity": 3.655 }, "per_group_fidelity": { "meaning-flip": 46.5, "entity": 66.5, "cultural": 35.1, "fidelity": 57.2 } }, "whisper": { "scores_by_judge": { "deepseek_flash": [ 100, 100, 95, 98, 99, 99, 94, 99, 98, 98, 87, 96, 65, 82, 97, 84, 85, 75, 62, 83, 99, 98, 100, 89, 97, 96, 98, 90, 86, 62, 61, 85, 79, 85, 99, 95, 93, 77, 81, 62, 57, 55, 83, 71, 51, 91, 86, 85, 82, 79, 96, 86, 95, 96, 95, 88, 98, 94, 94, 93, 94, 100, 99, 98, 100, 91, 98, 99, 100, 90, 75, 94, 85, 96, 95, 69, 64, 95, 91, 98, 96, 85, 97, 95, 90, 79, 83, 99, 100, 99, 96, 99, 98, 98, 97, 100, 97, 97, 100, 100 ] }, "per_category_deductions": { "polarity": { "deepseek_flash": 70 }, "semantic-inversion": { "deepseek_flash": 80 }, "hallucination": { "deepseek_flash": 0 }, "numeric": { "deepseek_flash": 20 }, "brand": { "deepseek_flash": 160 }, "tech": { "deepseek_flash": 88 }, "culture": { "deepseek_flash": 120 }, "proper-noun": { "deepseek_flash": 48 }, "dialect": { "deepseek_flash": 30 }, "mishear-severe": { "deepseek_flash": 100 }, "content-drop": { "deepseek_flash": 3 }, "paraphrase": { "deepseek_flash": 12 }, "mishear-minor": { "deepseek_flash": 36 }, "filler": { "deepseek_flash": 294 }, "format": { "deepseek_flash": 0 }, "unscoreable": { "deepseek_flash": 0 } }, "per_tier_deductions": { "tier1": { "deepseek_flash": 170 }, "tier2": { "deepseek_flash": 446 }, "tier3": { "deepseek_flash": 151 }, "tier4": { "deepseek_flash": 294 }, "unscoreable": { "deepseek_flash": 0 } }, "raw_deductions_by_judge": { "deepseek_flash": [ 0, 0, 5, 2, 1, 1, 6, 1, 2, 2, 13, 4, 35, 18, 3, 16, 15, 25, 38, 17, 1, 2, 0, 11, 3, 4, 2, 10, 14, 38, 39, 15, 21, 15, 1, 5, 7, 23, 19, 38, 43, 45, 17, 29, 49, 9, 14, 15, 18, 21, 4, 14, 5, 4, 5, 12, 2, 6, 6, 7, 6, 0, 1, 2, 0, 9, 2, 1, 0, 10, 25, 6, 15, 4, 5, 31, 36, 5, 9, 2, 4, 15, 3, 5, 10, 21, 17, 1, 0, 1, 4, 1, 2, 2, 3, 0, 3, 3, 0, 0 ] }, "sample_details": [ { "idx": 1, "duration_s": 5, "mean_score": 100.0, "mean_raw": 0.0, "scores_per_judge": [ 100 ], "deductions_per_judge": { "deepseek_flash": [] }, "judge_disagreement": 0 }, { "idx": 2, "duration_s": 7, "mean_score": 100.0, "mean_raw": 0.0, "scores_per_judge": [ 100 ], "deductions_per_judge": { "deepseek_flash": [] }, "judge_disagreement": 0 }, { "idx": 3, "duration_s": 8, "mean_score": 95.0, "mean_raw": 5.0, "scores_per_judge": [ 95 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 4, "duration_s": 11, "mean_score": 98.0, "mean_raw": 2.0, "scores_per_judge": [ 98 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 5, "duration_s": 16, "mean_score": 99.0, "mean_raw": 1.0, "scores_per_judge": [ 99 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 6, "duration_s": 17, "mean_score": 99.0, "mean_raw": 1.0, "scores_per_judge": [ 99 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 7, "duration_s": 20, "mean_score": 94.0, "mean_raw": 6.0, "scores_per_judge": [ 94 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 8, "duration_s": 30, "mean_score": 99.0, "mean_raw": 1.0, "scores_per_judge": [ 99 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 9, "duration_s": 32, "mean_score": 98.0, "mean_raw": 2.0, "scores_per_judge": [ 98 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 10, "duration_s": 35, "mean_score": 98.0, "mean_raw": 2.0, "scores_per_judge": [ 98 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 11, "duration_s": 40, "mean_score": 87.0, "mean_raw": 13.0, "scores_per_judge": [ 87 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "tech", "points": 8 }, { "category": "filler", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 12, "duration_s": 46, "mean_score": 96.0, "mean_raw": 4.0, "scores_per_judge": [ 96 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 13, "duration_s": 60, "mean_score": 65.0, "mean_raw": 35.0, "scores_per_judge": [ 65 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "brand", "points": 10 }, { "category": "brand", "points": 10 }, { "category": "tech", "points": 8 }, { "category": "mishear-minor", "points": 2 }, { "category": "filler", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 14, "duration_s": 64, "mean_score": 82.0, "mean_raw": 18.0, "scores_per_judge": [ 82 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 }, { "category": "mishear-severe", "points": 5 }, { "category": "paraphrase", "points": 3 }, { "category": "filler", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 15, "duration_s": 69, "mean_score": 97.0, "mean_raw": 3.0, "scores_per_judge": [ 97 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 16, "duration_s": 81, "mean_score": 84.0, "mean_raw": 16.0, "scores_per_judge": [ 84 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 }, { "category": "proper-noun", "points": 6 }, { "category": "filler", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 17, "duration_s": 100, "mean_score": 85.0, "mean_raw": 15.0, "scores_per_judge": [ 85 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "brand", "points": 10 }, { "category": "filler", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 18, "duration_s": 125, "mean_score": 75.0, "mean_raw": 25.0, "scores_per_judge": [ 75 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "brand", "points": 10 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 19, "duration_s": 133, "mean_score": 62.0, "mean_raw": 38.0, "scores_per_judge": [ 62 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "brand", "points": 10 }, { "category": "mishear-severe", "points": 5 }, { "category": "brand", "points": 10 }, { "category": "tech", "points": 8 } ] }, "judge_disagreement": 0 }, { "idx": 20, "duration_s": 164, "mean_score": 83.0, "mean_raw": 17.0, "scores_per_judge": [ 83 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "culture", "points": 8 }, { "category": "mishear-minor", "points": 2 }, { "category": "mishear-minor", "points": 2 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 21, "duration_s": 5, "mean_score": 99.0, "mean_raw": 1.0, "scores_per_judge": [ 99 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 22, "duration_s": 7, "mean_score": 98.0, "mean_raw": 2.0, "scores_per_judge": [ 98 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-minor", "points": 2 } ] }, "judge_disagreement": 0 }, { "idx": 23, "duration_s": 10, "mean_score": 100.0, "mean_raw": 0.0, "scores_per_judge": [ 100 ], "deductions_per_judge": { "deepseek_flash": [] }, "judge_disagreement": 0 }, { "idx": 24, "duration_s": 14, "mean_score": 89.0, "mean_raw": 11.0, "scores_per_judge": [ 89 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "tech", "points": 8 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 25, "duration_s": 16, "mean_score": 97.0, "mean_raw": 3.0, "scores_per_judge": [ 97 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 26, "duration_s": 23, "mean_score": 96.0, "mean_raw": 4.0, "scores_per_judge": [ 96 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 27, "duration_s": 25, "mean_score": 98.0, "mean_raw": 2.0, "scores_per_judge": [ 98 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-minor", "points": 2 } ] }, "judge_disagreement": 0 }, { "idx": 28, "duration_s": 30, "mean_score": 90.0, "mean_raw": 10.0, "scores_per_judge": [ 90 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 5 }, { "category": "mishear-severe", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 29, "duration_s": 34, "mean_score": 86.0, "mean_raw": 14.0, "scores_per_judge": [ 86 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "brand", "points": 10 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 30, "duration_s": 38, "mean_score": 62.0, "mean_raw": 38.0, "scores_per_judge": [ 62 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "polarity", "points": 35 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 31, "duration_s": 44, "mean_score": 61.0, "mean_raw": 39.0, "scores_per_judge": [ 61 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "polarity", "points": 35 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 32, "duration_s": 50, "mean_score": 85.0, "mean_raw": 15.0, "scores_per_judge": [ 85 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "brand", "points": 10 }, { "category": "filler", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 33, "duration_s": 62, "mean_score": 79.0, "mean_raw": 21.0, "scores_per_judge": [ 79 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "brand", "points": 10 }, { "category": "proper-noun", "points": 6 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 34, "duration_s": 66, "mean_score": 85.0, "mean_raw": 15.0, "scores_per_judge": [ 85 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-minor", "points": 2 }, { "category": "culture", "points": 8 }, { "category": "filler", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 35, "duration_s": 77, "mean_score": 99.0, "mean_raw": 1.0, "scores_per_judge": [ 99 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 36, "duration_s": 94, "mean_score": 95.0, "mean_raw": 5.0, "scores_per_judge": [ 95 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 37, "duration_s": 106, "mean_score": 93.0, "mean_raw": 7.0, "scores_per_judge": [ 93 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-minor", "points": 2 }, { "category": "filler", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 38, "duration_s": 126, "mean_score": 77.0, "mean_raw": 23.0, "scores_per_judge": [ 77 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "tech", "points": 8 }, { "category": "mishear-severe", "points": 5 }, { "category": "content-drop", "points": 3 }, { "category": "mishear-minor", "points": 2 }, { "category": "filler", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 39, "duration_s": 158, "mean_score": 81.0, "mean_raw": 19.0, "scores_per_judge": [ 81 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "proper-noun", "points": 6 }, { "category": "tech", "points": 8 }, { "category": "filler", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 40, "duration_s": 279, "mean_score": 62.0, "mean_raw": 38.0, "scores_per_judge": [ 62 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 }, { "category": "mishear-severe", "points": 5 }, { "category": "numeric", "points": 10 }, { "category": "culture", "points": 8 }, { "category": "dialect", "points": 5 }, { "category": "filler", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 41, "duration_s": 34, "mean_score": 57.0, "mean_raw": 43.0, "scores_per_judge": [ 57 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "culture", "points": 8 }, { "category": "culture", "points": 8 }, { "category": "culture", "points": 8 }, { "category": "culture", "points": 8 }, { "category": "culture", "points": 8 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 42, "duration_s": 43, "mean_score": 55.0, "mean_raw": 45.0, "scores_per_judge": [ 55 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "semantic-inversion", "points": 20 }, { "category": "dialect", "points": 5 }, { "category": "culture", "points": 8 }, { "category": "culture", "points": 8 }, { "category": "filler", "points": 4 } ] }, "judge_disagreement": 0 }, { "idx": 43, "duration_s": 29, "mean_score": 83.0, "mean_raw": 17.0, "scores_per_judge": [ 83 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "culture", "points": 8 }, { "category": "proper-noun", "points": 6 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 44, "duration_s": 71, "mean_score": 71.0, "mean_raw": 29.0, "scores_per_judge": [ 71 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 }, { "category": "proper-noun", "points": 6 }, { "category": "mishear-severe", "points": 5 }, { "category": "culture", "points": 8 }, { "category": "filler", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 45, "duration_s": 46, "mean_score": 51.0, "mean_raw": 49.0, "scores_per_judge": [ 51 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "dialect", "points": 5 }, { "category": "dialect", "points": 5 }, { "category": "proper-noun", "points": 6 }, { "category": "dialect", "points": 5 }, { "category": "semantic-inversion", "points": 20 }, { "category": "dialect", "points": 5 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 46, "duration_s": 62, "mean_score": 91.0, "mean_raw": 9.0, "scores_per_judge": [ 91 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-minor", "points": 2 }, { "category": "mishear-minor", "points": 2 }, { "category": "filler", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 47, "duration_s": 67, "mean_score": 86.0, "mean_raw": 14.0, "scores_per_judge": [ 86 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "brand", "points": 10 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 48, "duration_s": 72, "mean_score": 85.0, "mean_raw": 15.0, "scores_per_judge": [ 85 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "brand", "points": 10 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 49, "duration_s": 87, "mean_score": 82.0, "mean_raw": 18.0, "scores_per_judge": [ 82 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "brand", "points": 10 }, { "category": "paraphrase", "points": 3 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 50, "duration_s": 106, "mean_score": 79.0, "mean_raw": 21.0, "scores_per_judge": [ 79 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "culture", "points": 8 }, { "category": "culture", "points": 8 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 51, "duration_s": 33, "mean_score": 96.0, "mean_raw": 4.0, "scores_per_judge": [ 96 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 52, "duration_s": 25, "mean_score": 86.0, "mean_raw": 14.0, "scores_per_judge": [ 86 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "proper-noun", "points": 6 }, { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 3 } ] }, "judge_disagreement": 0 }, { "idx": 53, "duration_s": 35, "mean_score": 95.0, "mean_raw": 5.0, "scores_per_judge": [ 95 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 54, "duration_s": 36, "mean_score": 96.0, "mean_raw": 4.0, "scores_per_judge": [ 96 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 55, "duration_s": 33, "mean_score": 95.0, "mean_raw": 5.0, "scores_per_judge": [ 95 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 56, "duration_s": 15, "mean_score": 88.0, "mean_raw": 12.0, "scores_per_judge": [ 88 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "brand", "points": 10 } ] }, "judge_disagreement": 0 }, { "idx": 57, "duration_s": 38, "mean_score": 98.0, "mean_raw": 2.0, "scores_per_judge": [ 98 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 58, "duration_s": 15, "mean_score": 94.0, "mean_raw": 6.0, "scores_per_judge": [ 94 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 59, "duration_s": 28, "mean_score": 94.0, "mean_raw": 6.0, "scores_per_judge": [ 94 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "mishear-minor", "points": 2 } ] }, "judge_disagreement": 0 }, { "idx": 60, "duration_s": 18, "mean_score": 93.0, "mean_raw": 7.0, "scores_per_judge": [ 93 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 61, "duration_s": 19, "mean_score": 94.0, "mean_raw": 6.0, "scores_per_judge": [ 94 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 62, "duration_s": 17, "mean_score": 100.0, "mean_raw": 0.0, "scores_per_judge": [ 100 ], "deductions_per_judge": { "deepseek_flash": [] }, "judge_disagreement": 0 }, { "idx": 63, "duration_s": 21, "mean_score": 99.0, "mean_raw": 1.0, "scores_per_judge": [ 99 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 64, "duration_s": 23, "mean_score": 98.0, "mean_raw": 2.0, "scores_per_judge": [ 98 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 2 } ] }, "judge_disagreement": 0 }, { "idx": 65, "duration_s": 20, "mean_score": 100.0, "mean_raw": 0.0, "scores_per_judge": [ 100 ], "deductions_per_judge": { "deepseek_flash": [] }, "judge_disagreement": 0 }, { "idx": 66, "duration_s": 24, "mean_score": 91.0, "mean_raw": 9.0, "scores_per_judge": [ 91 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "culture", "points": 8 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 67, "duration_s": 16, "mean_score": 98.0, "mean_raw": 2.0, "scores_per_judge": [ 98 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 68, "duration_s": 18, "mean_score": 99.0, "mean_raw": 1.0, "scores_per_judge": [ 99 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 69, "duration_s": 18, "mean_score": 100.0, "mean_raw": 0.0, "scores_per_judge": [ 100 ], "deductions_per_judge": { "deepseek_flash": [] }, "judge_disagreement": 0 }, { "idx": 70, "duration_s": 17, "mean_score": 90.0, "mean_raw": 10.0, "scores_per_judge": [ 90 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "tech", "points": 8 } ] }, "judge_disagreement": 0 }, { "idx": 71, "duration_s": 51, "mean_score": 75.0, "mean_raw": 25.0, "scores_per_judge": [ 75 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "brand", "points": 10 }, { "category": "tech", "points": 8 }, { "category": "mishear-minor", "points": 2 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 72, "duration_s": 54, "mean_score": 94.0, "mean_raw": 6.0, "scores_per_judge": [ 94 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-minor", "points": 2 }, { "category": "filler", "points": 4 } ] }, "judge_disagreement": 0 }, { "idx": 73, "duration_s": 56, "mean_score": 85.0, "mean_raw": 15.0, "scores_per_judge": [ 85 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "brand", "points": 10 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 74, "duration_s": 45, "mean_score": 96.0, "mean_raw": 4.0, "scores_per_judge": [ 96 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 75, "duration_s": 74, "mean_score": 95.0, "mean_raw": 5.0, "scores_per_judge": [ 95 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 76, "duration_s": 58, "mean_score": 69.0, "mean_raw": 31.0, "scores_per_judge": [ 69 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "semantic-inversion", "points": 20 }, { "category": "proper-noun", "points": 6 }, { "category": "filler", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 77, "duration_s": 130, "mean_score": 64.0, "mean_raw": 36.0, "scores_per_judge": [ 64 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "semantic-inversion", "points": 20 }, { "category": "brand", "points": 10 }, { "category": "mishear-minor", "points": 2 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 78, "duration_s": 61, "mean_score": 95.0, "mean_raw": 5.0, "scores_per_judge": [ 95 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 79, "duration_s": 47, "mean_score": 91.0, "mean_raw": 9.0, "scores_per_judge": [ 91 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "mishear-minor", "points": 2 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "paraphrase", "points": 3 } ] }, "judge_disagreement": 0 }, { "idx": 80, "duration_s": 42, "mean_score": 98.0, "mean_raw": 2.0, "scores_per_judge": [ 98 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 81, "duration_s": 38, "mean_score": 96.0, "mean_raw": 4.0, "scores_per_judge": [ 96 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 82, "duration_s": 38, "mean_score": 85.0, "mean_raw": 15.0, "scores_per_judge": [ 85 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 }, { "category": "tech", "points": 8 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 83, "duration_s": 37, "mean_score": 97.0, "mean_raw": 3.0, "scores_per_judge": [ 97 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 84, "duration_s": 36, "mean_score": 95.0, "mean_raw": 5.0, "scores_per_judge": [ 95 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 85, "duration_s": 36, "mean_score": 90.0, "mean_raw": 10.0, "scores_per_judge": [ 90 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 }, { "category": "mishear-minor", "points": 2 }, { "category": "filler", "points": 3 } ] }, "judge_disagreement": 0 }, { "idx": 86, "duration_s": 35, "mean_score": 79.0, "mean_raw": 21.0, "scores_per_judge": [ 79 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "tech", "points": 8 }, { "category": "tech", "points": 8 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 87, "duration_s": 34, "mean_score": 83.0, "mean_raw": 17.0, "scores_per_judge": [ 83 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "numeric", "points": 10 }, { "category": "mishear-minor", "points": 2 }, { "category": "filler", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 88, "duration_s": 34, "mean_score": 99.0, "mean_raw": 1.0, "scores_per_judge": [ 99 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 89, "duration_s": 34, "mean_score": 100.0, "mean_raw": 0.0, "scores_per_judge": [ 100 ], "deductions_per_judge": { "deepseek_flash": [] }, "judge_disagreement": 0 }, { "idx": 90, "duration_s": 33, "mean_score": 99.0, "mean_raw": 1.0, "scores_per_judge": [ 99 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 91, "duration_s": 33, "mean_score": 96.0, "mean_raw": 4.0, "scores_per_judge": [ 96 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 4 } ] }, "judge_disagreement": 0 }, { "idx": 92, "duration_s": 32, "mean_score": 99.0, "mean_raw": 1.0, "scores_per_judge": [ 99 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 93, "duration_s": 32, "mean_score": 98.0, "mean_raw": 2.0, "scores_per_judge": [ 98 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-minor", "points": 2 } ] }, "judge_disagreement": 0 }, { "idx": 94, "duration_s": 32, "mean_score": 98.0, "mean_raw": 2.0, "scores_per_judge": [ 98 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 95, "duration_s": 32, "mean_score": 97.0, "mean_raw": 3.0, "scores_per_judge": [ 97 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "paraphrase", "points": 3 } ] }, "judge_disagreement": 0 }, { "idx": 96, "duration_s": 31, "mean_score": 100.0, "mean_raw": 0.0, "scores_per_judge": [ 100 ], "deductions_per_judge": { "deepseek_flash": [] }, "judge_disagreement": 0 }, { "idx": 97, "duration_s": 30, "mean_score": 97.0, "mean_raw": 3.0, "scores_per_judge": [ 97 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 98, "duration_s": 30, "mean_score": 97.0, "mean_raw": 3.0, "scores_per_judge": [ 97 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 99, "duration_s": 30, "mean_score": 100.0, "mean_raw": 0.0, "scores_per_judge": [ 100 ], "deductions_per_judge": { "deepseek_flash": [] }, "judge_disagreement": 0 }, { "idx": 100, "duration_s": 30, "mean_score": 100.0, "mean_raw": 0.0, "scores_per_judge": [ 100 ], "deductions_per_judge": { "deepseek_flash": [] }, "judge_disagreement": 0 } ], "total_deductions": 1061, "avg_deductions_per_sample": 10.61, "mean_clamped_score": 89.4, "n_judge_sample_pairs": 100, "gt_chars": 22625, "ded_per_100char": 4.69, "fidelity_score": 76.6, "overall_score": 89.4, "overall_raw_deductions": 10.61, "score_per_judge": { "deepseek_flash": 89.4 }, "per_group_deductions": { "meaning-flip": { "deepseek_flash": 170 }, "entity": { "deepseek_flash": 296 }, "cultural": { "deepseek_flash": 150 }, "fidelity": { "deepseek_flash": 445 } }, "per_bucket": { "general": { "n_samples": 40, "mean_raw": 11.8, "mean_score": 88.2, "total_raw": 472.0, "idxs": [ 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40 ], "worst_idx": 31 }, "idiom-test": { "n_samples": 5, "mean_raw": 36.6, "mean_score": 63.4, "total_raw": 183.0, "idxs": [ 41, 42, 43, 44, 45 ], "worst_idx": 45 }, "long-mixed": { "n_samples": 5, "mean_raw": 15.4, "mean_score": 84.6, "total_raw": 77.0, "idxs": [ 46, 47, 48, 49, 50 ], "worst_idx": 50 } }, "per_group_per_100char": { "meaning-flip": 0.751, "entity": 1.308, "cultural": 0.663, "fidelity": 1.967 }, "per_group_fidelity": { "meaning-flip": 82.0, "entity": 76.8, "cultural": 59.9, "fidelity": 77.0 } }, "whisper_v3_turbo": { "scores_by_judge": { "deepseek_flash": [ 100, 98, 99, 98, 97, 97, 85, 95, 95, 97, 88, 95, 67, 77, 70, 72, 70, 25, 34, 84, 100, 93, 100, 89, 84, 98, 99, 63, 52, 50, 93, 95, 79, 34, 61, 52, 90, 80, 62, 28, 55, 66, 71, 41, 43, 74, 87, 82, 79, 67, 68, 85, 95, 85, 74, 66, 43, 98, 91, 85, 89, 100, 88, 96, 99, 97, 97, 90, 91, 93, 51, 90, 85, 95, null, 47, 65, 95, 95, 85, 75, 83, 95, 95, 80, 81, 85, 99, 97, 95, 96, 99, 77, 77, 95, 100, 93, 95, 98, 96 ] }, "per_category_deductions": { "polarity": { "deepseek_flash": 140 }, "semantic-inversion": { "deepseek_flash": 80 }, "hallucination": { "deepseek_flash": 135 }, "numeric": { "deepseek_flash": 10 }, "brand": { "deepseek_flash": 180 }, "tech": { "deepseek_flash": 224 }, "culture": { "deepseek_flash": 224 }, "proper-noun": { "deepseek_flash": 102 }, "dialect": { "deepseek_flash": 25 }, "mishear-severe": { "deepseek_flash": 280 }, "content-drop": { "deepseek_flash": 30 }, "paraphrase": { "deepseek_flash": 9 }, "mishear-minor": { "deepseek_flash": 38 }, "filler": { "deepseek_flash": 364 }, "format": { "deepseek_flash": 0 }, "unscoreable": { "deepseek_flash": 0 } }, "per_tier_deductions": { "tier1": { "deepseek_flash": 365 }, "tier2": { "deepseek_flash": 755 }, "tier3": { "deepseek_flash": 357 }, "tier4": { "deepseek_flash": 364 }, "unscoreable": { "deepseek_flash": 0 } }, "raw_deductions_by_judge": { "deepseek_flash": [ 0, 2, 1, 2, 3, 3, 15, 5, 5, 3, 12, 5, 33, 23, 30, 28, 30, 75, 66, 16, 0, 7, 0, 11, 16, 2, 1, 37, 48, 50, 7, 5, 21, 66, 39, 48, 10, 20, 38, 72, 45, 34, 29, 59, 57, 26, 13, 18, 21, 33, 32, 15, 5, 15, 26, 34, 57, 2, 9, 15, 11, 0, 12, 4, 1, 3, 3, 10, 9, 7, 49, 10, 15, 5, null, 53, 35, 5, 5, 15, 25, 17, 5, 5, 20, 19, 15, 1, 3, 5, 4, 1, 23, 23, 5, 0, 7, 5, 2, 4 ] }, "sample_details": [ { "idx": 1, "duration_s": 5, "mean_score": 100.0, "mean_raw": 0.0, "scores_per_judge": [ 100 ], "deductions_per_judge": { "deepseek_flash": [] }, "judge_disagreement": 0 }, { "idx": 2, "duration_s": 7, "mean_score": 98.0, "mean_raw": 2.0, "scores_per_judge": [ 98 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 3, "duration_s": 8, "mean_score": 99.0, "mean_raw": 1.0, "scores_per_judge": [ 99 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 4, "duration_s": 11, "mean_score": 98.0, "mean_raw": 2.0, "scores_per_judge": [ 98 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 5, "duration_s": 16, "mean_score": 97.0, "mean_raw": 3.0, "scores_per_judge": [ 97 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 6, "duration_s": 17, "mean_score": 97.0, "mean_raw": 3.0, "scores_per_judge": [ 97 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 7, "duration_s": 20, "mean_score": 85.0, "mean_raw": 15.0, "scores_per_judge": [ 85 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "brand", "points": 10 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 8, "duration_s": 30, "mean_score": 95.0, "mean_raw": 5.0, "scores_per_judge": [ 95 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 9, "duration_s": 32, "mean_score": 95.0, "mean_raw": 5.0, "scores_per_judge": [ 95 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 10, "duration_s": 35, "mean_score": 97.0, "mean_raw": 3.0, "scores_per_judge": [ 97 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 11, "duration_s": 40, "mean_score": 88.0, "mean_raw": 12.0, "scores_per_judge": [ 88 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 }, { "category": "mishear-minor", "points": 2 }, { "category": "filler", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 12, "duration_s": 46, "mean_score": 95.0, "mean_raw": 5.0, "scores_per_judge": [ 95 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 13, "duration_s": 60, "mean_score": 67.0, "mean_raw": 33.0, "scores_per_judge": [ 67 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "brand", "points": 10 }, { "category": "brand", "points": 10 }, { "category": "tech", "points": 8 }, { "category": "filler", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 14, "duration_s": 64, "mean_score": 77.0, "mean_raw": 23.0, "scores_per_judge": [ 77 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 }, { "category": "mishear-severe", "points": 5 }, { "category": "content-drop", "points": 3 }, { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 15, "duration_s": 69, "mean_score": 70.0, "mean_raw": 30.0, "scores_per_judge": [ 70 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "hallucination", "points": 15 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 16, "duration_s": 81, "mean_score": 72.0, "mean_raw": 28.0, "scores_per_judge": [ 72 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "proper-noun", "points": 6 }, { "category": "proper-noun", "points": 6 }, { "category": "tech", "points": 8 }, { "category": "content-drop", "points": 3 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 17, "duration_s": 100, "mean_score": 70.0, "mean_raw": 30.0, "scores_per_judge": [ 70 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "brand", "points": 10 }, { "category": "hallucination", "points": 15 }, { "category": "filler", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 18, "duration_s": 125, "mean_score": 25.0, "mean_raw": 75.0, "scores_per_judge": [ 25 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "hallucination", "points": 15 }, { "category": "hallucination", "points": 15 }, { "category": "brand", "points": 10 }, { "category": "tech", "points": 8 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-minor", "points": 2 }, { "category": "filler", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 19, "duration_s": 133, "mean_score": 34.0, "mean_raw": 66.0, "scores_per_judge": [ 34 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "hallucination", "points": 15 }, { "category": "brand", "points": 10 }, { "category": "brand", "points": 10 }, { "category": "tech", "points": 8 }, { "category": "tech", "points": 8 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 20, "duration_s": 164, "mean_score": 84.0, "mean_raw": 16.0, "scores_per_judge": [ 84 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 }, { "category": "culture", "points": 8 }, { "category": "filler", "points": 3 } ] }, "judge_disagreement": 0 }, { "idx": 21, "duration_s": 5, "mean_score": 100.0, "mean_raw": 0.0, "scores_per_judge": [ 100 ], "deductions_per_judge": { "deepseek_flash": [] }, "judge_disagreement": 0 }, { "idx": 22, "duration_s": 7, "mean_score": 93.0, "mean_raw": 7.0, "scores_per_judge": [ 93 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "proper-noun", "points": 6 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 23, "duration_s": 10, "mean_score": 100.0, "mean_raw": 0.0, "scores_per_judge": [ 100 ], "deductions_per_judge": { "deepseek_flash": [] }, "judge_disagreement": 0 }, { "idx": 24, "duration_s": 14, "mean_score": 89.0, "mean_raw": 11.0, "scores_per_judge": [ 89 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "tech", "points": 8 }, { "category": "filler", "points": 3 } ] }, "judge_disagreement": 0 }, { "idx": 25, "duration_s": 16, "mean_score": 84.0, "mean_raw": 16.0, "scores_per_judge": [ 84 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "tech", "points": 8 }, { "category": "content-drop", "points": 3 }, { "category": "filler", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 26, "duration_s": 23, "mean_score": 98.0, "mean_raw": 2.0, "scores_per_judge": [ 98 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 27, "duration_s": 25, "mean_score": 99.0, "mean_raw": 1.0, "scores_per_judge": [ 99 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 28, "duration_s": 30, "mean_score": 63.0, "mean_raw": 37.0, "scores_per_judge": [ 63 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "semantic-inversion", "points": 20 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 2 } ] }, "judge_disagreement": 0 }, { "idx": 29, "duration_s": 34, "mean_score": 52.0, "mean_raw": 48.0, "scores_per_judge": [ 52 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "hallucination", "points": 15 }, { "category": "brand", "points": 10 }, { "category": "brand", "points": 10 }, { "category": "tech", "points": 8 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 30, "duration_s": 38, "mean_score": 50.0, "mean_raw": 50.0, "scores_per_judge": [ 50 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "polarity", "points": 35 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 31, "duration_s": 44, "mean_score": 93.0, "mean_raw": 7.0, "scores_per_judge": [ 93 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-minor", "points": 2 }, { "category": "filler", "points": 2 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 32, "duration_s": 50, "mean_score": 95.0, "mean_raw": 5.0, "scores_per_judge": [ 95 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 33, "duration_s": 62, "mean_score": 79.0, "mean_raw": 21.0, "scores_per_judge": [ 79 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "brand", "points": 10 }, { "category": "proper-noun", "points": 6 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 34, "duration_s": 66, "mean_score": 34.0, "mean_raw": 66.0, "scores_per_judge": [ 34 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "polarity", "points": 35 }, { "category": "proper-noun", "points": 6 }, { "category": "mishear-severe", "points": 5 }, { "category": "hallucination", "points": 15 }, { "category": "filler", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 35, "duration_s": 77, "mean_score": 61.0, "mean_raw": 39.0, "scores_per_judge": [ 61 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "tech", "points": 8 }, { "category": "tech", "points": 8 }, { "category": "tech", "points": 8 }, { "category": "mishear-severe", "points": 5 }, { "category": "content-drop", "points": 3 }, { "category": "mishear-minor", "points": 2 }, { "category": "filler", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 36, "duration_s": 94, "mean_score": 52.0, "mean_raw": 48.0, "scores_per_judge": [ 52 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "polarity", "points": 35 }, { "category": "mishear-severe", "points": 5 }, { "category": "content-drop", "points": 3 }, { "category": "filler", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 37, "duration_s": 106, "mean_score": 90.0, "mean_raw": 10.0, "scores_per_judge": [ 90 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 38, "duration_s": 126, "mean_score": 80.0, "mean_raw": 20.0, "scores_per_judge": [ 80 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "tech", "points": 8 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-minor", "points": 2 }, { "category": "filler", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 39, "duration_s": 158, "mean_score": 62.0, "mean_raw": 38.0, "scores_per_judge": [ 62 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "tech", "points": 8 }, { "category": "tech", "points": 8 }, { "category": "proper-noun", "points": 6 }, { "category": "proper-noun", "points": 6 }, { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 40, "duration_s": 279, "mean_score": 28.0, "mean_raw": 72.0, "scores_per_judge": [ 28 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 }, { "category": "culture", "points": 8 }, { "category": "culture", "points": 8 }, { "category": "tech", "points": 8 }, { "category": "culture", "points": 8 }, { "category": "mishear-severe", "points": 5 }, { "category": "dialect", "points": 5 }, { "category": "mishear-severe", "points": 5 }, { "category": "hallucination", "points": 15 }, { "category": "filler", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 41, "duration_s": 34, "mean_score": 55.0, "mean_raw": 45.0, "scores_per_judge": [ 55 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "culture", "points": 8 }, { "category": "culture", "points": 8 }, { "category": "culture", "points": 8 }, { "category": "culture", "points": 8 }, { "category": "culture", "points": 8 }, { "category": "filler", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 42, "duration_s": 43, "mean_score": 66.0, "mean_raw": 34.0, "scores_per_judge": [ 66 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "dialect", "points": 5 }, { "category": "culture", "points": 8 }, { "category": "culture", "points": 8 }, { "category": "culture", "points": 8 }, { "category": "filler", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 43, "duration_s": 29, "mean_score": 71.0, "mean_raw": 29.0, "scores_per_judge": [ 71 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "culture", "points": 8 }, { "category": "culture", "points": 8 }, { "category": "culture", "points": 8 }, { "category": "filler", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 44, "duration_s": 71, "mean_score": 41.0, "mean_raw": 59.0, "scores_per_judge": [ 41 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "proper-noun", "points": 6 }, { "category": "proper-noun", "points": 6 }, { "category": "proper-noun", "points": 6 }, { "category": "culture", "points": 8 }, { "category": "culture", "points": 8 }, { "category": "culture", "points": 8 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-minor", "points": 2 }, { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 45, "duration_s": 46, "mean_score": 43.0, "mean_raw": 57.0, "scores_per_judge": [ 43 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "dialect", "points": 5 }, { "category": "dialect", "points": 5 }, { "category": "dialect", "points": 5 }, { "category": "culture", "points": 8 }, { "category": "culture", "points": 8 }, { "category": "culture", "points": 8 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-severe", "points": 5 }, { "category": "content-drop", "points": 3 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 46, "duration_s": 62, "mean_score": 74.0, "mean_raw": 26.0, "scores_per_judge": [ 74 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "paraphrase", "points": 3 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-severe", "points": 5 }, { "category": "culture", "points": 8 } ] }, "judge_disagreement": 0 }, { "idx": 47, "duration_s": 67, "mean_score": 87.0, "mean_raw": 13.0, "scores_per_judge": [ 87 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "brand", "points": 10 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 48, "duration_s": 72, "mean_score": 82.0, "mean_raw": 18.0, "scores_per_judge": [ 82 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "tech", "points": 8 }, { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 4 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 49, "duration_s": 87, "mean_score": 79.0, "mean_raw": 21.0, "scores_per_judge": [ 79 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "brand", "points": 10 }, { "category": "proper-noun", "points": 6 }, { "category": "filler", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 50, "duration_s": 106, "mean_score": 67.0, "mean_raw": 33.0, "scores_per_judge": [ 67 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "culture", "points": 8 }, { "category": "culture", "points": 8 }, { "category": "tech", "points": 8 }, { "category": "mishear-minor", "points": 2 }, { "category": "mishear-minor", "points": 2 }, { "category": "mishear-minor", "points": 2 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 51, "duration_s": 33, "mean_score": 68.0, "mean_raw": 32.0, "scores_per_judge": [ 68 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "culture", "points": 8 }, { "category": "proper-noun", "points": 6 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-minor", "points": 2 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 52, "duration_s": 25, "mean_score": 85.0, "mean_raw": 15.0, "scores_per_judge": [ 85 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "proper-noun", "points": 6 }, { "category": "proper-noun", "points": 6 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 53, "duration_s": 35, "mean_score": 95.0, "mean_raw": 5.0, "scores_per_judge": [ 95 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 54, "duration_s": 36, "mean_score": 85.0, "mean_raw": 15.0, "scores_per_judge": [ 85 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 5 }, { "category": "mishear-minor", "points": 2 }, { "category": "tech", "points": 8 } ] }, "judge_disagreement": 0 }, { "idx": 55, "duration_s": 33, "mean_score": 74.0, "mean_raw": 26.0, "scores_per_judge": [ 74 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "proper-noun", "points": 6 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 56, "duration_s": 15, "mean_score": 66.0, "mean_raw": 34.0, "scores_per_judge": [ 66 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "brand", "points": 10 }, { "category": "brand", "points": 10 }, { "category": "brand", "points": 10 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 57, "duration_s": 38, "mean_score": 43.0, "mean_raw": 57.0, "scores_per_judge": [ 43 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "polarity", "points": 35 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-severe", "points": 5 }, { "category": "mishear-minor", "points": 2 }, { "category": "filler", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 58, "duration_s": 15, "mean_score": 98.0, "mean_raw": 2.0, "scores_per_judge": [ 98 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 59, "duration_s": 28, "mean_score": 91.0, "mean_raw": 9.0, "scores_per_judge": [ 91 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "paraphrase", "points": 3 }, { "category": "mishear-minor", "points": 2 } ] }, "judge_disagreement": 0 }, { "idx": 60, "duration_s": 18, "mean_score": 85.0, "mean_raw": 15.0, "scores_per_judge": [ 85 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "culture", "points": 8 }, { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 61, "duration_s": 19, "mean_score": 89.0, "mean_raw": 11.0, "scores_per_judge": [ 89 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 }, { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 62, "duration_s": 17, "mean_score": 100.0, "mean_raw": 0.0, "scores_per_judge": [ 100 ], "deductions_per_judge": { "deepseek_flash": [] }, "judge_disagreement": 0 }, { "idx": 63, "duration_s": 21, "mean_score": 88.0, "mean_raw": 12.0, "scores_per_judge": [ 88 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "tech", "points": 8 }, { "category": "content-drop", "points": 3 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 64, "duration_s": 23, "mean_score": 96.0, "mean_raw": 4.0, "scores_per_judge": [ 96 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 65, "duration_s": 20, "mean_score": 99.0, "mean_raw": 1.0, "scores_per_judge": [ 99 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 66, "duration_s": 24, "mean_score": 97.0, "mean_raw": 3.0, "scores_per_judge": [ 97 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "content-drop", "points": 3 } ] }, "judge_disagreement": 0 }, { "idx": 67, "duration_s": 16, "mean_score": 97.0, "mean_raw": 3.0, "scores_per_judge": [ 97 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 68, "duration_s": 18, "mean_score": 90.0, "mean_raw": 10.0, "scores_per_judge": [ 90 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "brand", "points": 10 } ] }, "judge_disagreement": 0 }, { "idx": 69, "duration_s": 18, "mean_score": 91.0, "mean_raw": 9.0, "scores_per_judge": [ 91 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 4 } ] }, "judge_disagreement": 0 }, { "idx": 70, "duration_s": 17, "mean_score": 93.0, "mean_raw": 7.0, "scores_per_judge": [ 93 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-minor", "points": 2 }, { "category": "filler", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 71, "duration_s": 51, "mean_score": 51.0, "mean_raw": 49.0, "scores_per_judge": [ 51 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "brand", "points": 10 }, { "category": "tech", "points": 8 }, { "category": "tech", "points": 8 }, { "category": "tech", "points": 8 }, { "category": "tech", "points": 8 }, { "category": "mishear-minor", "points": 2 }, { "category": "filler", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 72, "duration_s": 54, "mean_score": 90.0, "mean_raw": 10.0, "scores_per_judge": [ 90 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 73, "duration_s": 56, "mean_score": 85.0, "mean_raw": 15.0, "scores_per_judge": [ 85 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 }, { "category": "mishear-minor", "points": 2 }, { "category": "paraphrase", "points": 3 }, { "category": "filler", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 74, "duration_s": 45, "mean_score": 95.0, "mean_raw": 5.0, "scores_per_judge": [ 95 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 76, "duration_s": 58, "mean_score": 47.0, "mean_raw": 53.0, "scores_per_judge": [ 47 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "semantic-inversion", "points": 20 }, { "category": "culture", "points": 8 }, { "category": "culture", "points": 8 }, { "category": "proper-noun", "points": 6 }, { "category": "proper-noun", "points": 6 }, { "category": "filler", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 77, "duration_s": 130, "mean_score": 65.0, "mean_raw": 35.0, "scores_per_judge": [ 65 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "semantic-inversion", "points": 20 }, { "category": "brand", "points": 10 }, { "category": "filler", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 78, "duration_s": 61, "mean_score": 95.0, "mean_raw": 5.0, "scores_per_judge": [ 95 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 79, "duration_s": 47, "mean_score": 95.0, "mean_raw": 5.0, "scores_per_judge": [ 95 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 80, "duration_s": 42, "mean_score": 85.0, "mean_raw": 15.0, "scores_per_judge": [ 85 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 }, { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 81, "duration_s": 38, "mean_score": 75.0, "mean_raw": 25.0, "scores_per_judge": [ 75 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "semantic-inversion", "points": 20 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 82, "duration_s": 38, "mean_score": 83.0, "mean_raw": 17.0, "scores_per_judge": [ 83 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "tech", "points": 8 }, { "category": "mishear-severe", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 83, "duration_s": 37, "mean_score": 95.0, "mean_raw": 5.0, "scores_per_judge": [ 95 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 84, "duration_s": 36, "mean_score": 95.0, "mean_raw": 5.0, "scores_per_judge": [ 95 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 85, "duration_s": 36, "mean_score": 80.0, "mean_raw": 20.0, "scores_per_judge": [ 80 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 }, { "category": "mishear-minor", "points": 2 }, { "category": "mishear-severe", "points": 5 }, { "category": "content-drop", "points": 3 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 86, "duration_s": 35, "mean_score": 81.0, "mean_raw": 19.0, "scores_per_judge": [ 81 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "tech", "points": 8 }, { "category": "tech", "points": 8 }, { "category": "filler", "points": 3 } ] }, "judge_disagreement": 0 }, { "idx": 87, "duration_s": 34, "mean_score": 85.0, "mean_raw": 15.0, "scores_per_judge": [ 85 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "numeric", "points": 10 }, { "category": "filler", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 88, "duration_s": 34, "mean_score": 99.0, "mean_raw": 1.0, "scores_per_judge": [ 99 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 89, "duration_s": 34, "mean_score": 97.0, "mean_raw": 3.0, "scores_per_judge": [ 97 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 90, "duration_s": 33, "mean_score": 95.0, "mean_raw": 5.0, "scores_per_judge": [ 95 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-severe", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 91, "duration_s": 33, "mean_score": 96.0, "mean_raw": 4.0, "scores_per_judge": [ 96 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 92, "duration_s": 32, "mean_score": 99.0, "mean_raw": 1.0, "scores_per_judge": [ 99 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 93, "duration_s": 32, "mean_score": 77.0, "mean_raw": 23.0, "scores_per_judge": [ 77 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "tech", "points": 8 }, { "category": "tech", "points": 8 }, { "category": "content-drop", "points": 3 }, { "category": "mishear-minor", "points": 2 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 94, "duration_s": 32, "mean_score": 77.0, "mean_raw": 23.0, "scores_per_judge": [ 77 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "mishear-severe", "points": 5 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "hallucination", "points": 15 } ] }, "judge_disagreement": 0 }, { "idx": 95, "duration_s": 32, "mean_score": 95.0, "mean_raw": 5.0, "scores_per_judge": [ 95 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 96, "duration_s": 31, "mean_score": 100.0, "mean_raw": 0.0, "scores_per_judge": [ 100 ], "deductions_per_judge": { "deepseek_flash": [] }, "judge_disagreement": 0 }, { "idx": 97, "duration_s": 30, "mean_score": 93.0, "mean_raw": 7.0, "scores_per_judge": [ 93 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "mishear-minor", "points": 2 }, { "category": "filler", "points": 5 } ] }, "judge_disagreement": 0 }, { "idx": 98, "duration_s": 30, "mean_score": 95.0, "mean_raw": 5.0, "scores_per_judge": [ 95 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 99, "duration_s": 30, "mean_score": 98.0, "mean_raw": 2.0, "scores_per_judge": [ 98 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 }, { "idx": 100, "duration_s": 30, "mean_score": 96.0, "mean_raw": 4.0, "scores_per_judge": [ 96 ], "deductions_per_judge": { "deepseek_flash": [ { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 }, { "category": "filler", "points": 1 } ] }, "judge_disagreement": 0 } ], "total_deductions": 1841, "avg_deductions_per_sample": 18.6, "mean_clamped_score": 81.4, "n_judge_sample_pairs": 99, "gt_chars": 22330, "ded_per_100char": 8.24, "fidelity_score": 58.8, "overall_score": 81.4, "overall_raw_deductions": 18.6, "score_per_judge": { "deepseek_flash": 81.4 }, "per_group_deductions": { "meaning-flip": { "deepseek_flash": 365 }, "entity": { "deepseek_flash": 506 }, "cultural": { "deepseek_flash": 249 }, "fidelity": { "deepseek_flash": 721 } }, "per_bucket": { "general": { "n_samples": 40, "mean_raw": 21.4, "mean_score": 78.6, "total_raw": 855.0, "idxs": [ 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40 ], "worst_idx": 18 }, "idiom-test": { "n_samples": 5, "mean_raw": 44.8, "mean_score": 55.2, "total_raw": 224.0, "idxs": [ 41, 42, 43, 44, 45 ], "worst_idx": 44 }, "long-mixed": { "n_samples": 5, "mean_raw": 22.2, "mean_score": 77.8, "total_raw": 111.0, "idxs": [ 46, 47, 48, 49, 50 ], "worst_idx": 50 } }, "per_group_per_100char": { "meaning-flip": 1.635, "entity": 2.266, "cultural": 1.115, "fidelity": 3.229 }, "per_group_fidelity": { "meaning-flip": 60.8, "entity": 59.8, "cultural": 32.6, "fidelity": 62.2 } } }, "per_sample": { "1": { "duration_s": 5, "model_scores": { "apple_speechanalyzer": { "mean_score": 100.0, "mean_raw": 0.0 }, "doubao": { "mean_score": 100.0, "mean_raw": 0.0 }, "elevenlabs": { "mean_score": 100.0, "mean_raw": 0.0 }, "firered": { "mean_score": 100.0, "mean_raw": 0.0 }, "firered_aed": { "mean_score": 100.0, "mean_raw": 0.0 }, "flash": { "mean_score": 100.0, "mean_raw": 0.0 }, "funasr_nano": { "mean_score": 100.0, "mean_raw": 0.0 }, "gemini_3_5_flash_lite_stt": { "mean_score": 85.0, "mean_raw": 15.0 }, "gemini_3_5_flash_stt": { "mean_score": 79.0, "mean_raw": 21.0 }, "gemini_3_5_transcribe": { "mean_score": 100.0, "mean_raw": 0.0 }, "gemini_3_6_flash_stt": { "mean_score": 79.0, "mean_raw": 21.0 }, "gemini_3_7_flash_stt": { "mean_score": 99.0, "mean_raw": 1.0 }, "gemini_3_8_flash_stt": { "mean_score": 100.0, "mean_raw": 0.0 }, "gemini_flash_stt": { "mean_score": 79.0, "mean_raw": 21.0 }, "gemini_pro_stt": { "mean_score": 100.0, "mean_raw": 0.0 }, "gemini_stt": { "mean_score": 100.0, "mean_raw": 0.0 }, "glm_asr_nano": { "mean_score": 100.0, "mean_raw": 0.0 }, "gpt_transcribe": { "mean_score": 100.0, "mean_raw": 0.0 }, "mai_transcribe_2": { "mean_score": 100.0, "mean_raw": 0.0 }, "mimo": { "mean_score": 100.0, "mean_raw": 0.0 }, "mimo_stt": { "mean_score": 99.0, "mean_raw": 1.0 }, "moss_audio": { "mean_score": 100.0, "mean_raw": 0.0 }, "qwen3_oss": { "mean_score": 100.0, "mean_raw": 0.0 }, "qwen_audio_3_1_asr": { "mean_score": 100.0, "mean_raw": 0.0 }, "r2t2": { "mean_score": 97.0, "mean_raw": 3.0 }, "soniox": { "mean_score": 100.0, "mean_raw": 0.0 }, "stepfun": { "mean_score": 100.0, "mean_raw": 0.0 }, "vibevoice": { "mean_score": 100.0, "mean_raw": 0.0 }, "voxtral": { "mean_score": 94.0, "mean_raw": 6.0 }, "whisper": { "mean_score": 100.0, "mean_raw": 0.0 }, "whisper_v3_turbo": { "mean_score": 100.0, "mean_raw": 0.0 } } }, "2": { "duration_s": 7, "model_scores": { "apple_speechanalyzer": { "mean_score": 92.0, "mean_raw": 8.0 }, "doubao": { "mean_score": 100.0, "mean_raw": 0.0 }, "elevenlabs": { "mean_score": 100.0, "mean_raw": 0.0 }, "firered": { "mean_score": 95.0, "mean_raw": 5.0 }, "firered_aed": { "mean_score": 100.0, "mean_raw": 0.0 }, "flash": { "mean_score": 100.0, "mean_raw": 0.0 }, "funasr_nano": { "mean_score": 100.0, "mean_raw": 0.0 }, "gemini_3_5_flash_lite_stt": { "mean_score": 99.0, "mean_raw": 1.0 }, "gemini_3_5_flash_stt": { "mean_score": 100.0, "mean_raw": 0.0 }, "gemini_3_5_transcribe": { "mean_score": 100.0, "mean_raw": 0.0 }, "gemini_3_6_flash_stt": { "mean_score": 100.0, "mean_raw": 0.0 }, "gemini_3_7_flash_stt": { "mean_score": 99.0, "mean_raw": 1.0 }, "gemini_3_8_flash_stt": { "mean_score": 99.0, "mean_raw": 1.0 }, "gemini_flash_stt": { "mean_score": 100.0, "mean_raw": 0.0 }, "gemini_pro_stt": { "mean_score": 100.0, "mean_raw": 0.0 }, "gemini_stt": { "mean_score": 100.0, "mean_raw": 0.0 }, "glm_asr_nano": { "mean_score": 100.0, "mean_raw": 0.0 }, "gpt_transcribe": { "mean_score": 100.0, "mean_raw": 0.0 }, "mai_transcribe_2": { "mean_score": 99.0, "mean_raw": 1.0 }, "mimo": { "mean_score": 100.0, "mean_raw": 0.0 }, "mimo_stt": { "mean_score": 99.0, "mean_raw": 1.0 }, "moss_audio": { "mean_score": 100.0, "mean_raw": 0.0 }, "qwen3_oss": { "mean_score": 99.0, "mean_raw": 1.0 }, "qwen_audio_3_1_asr": { "mean_score": 100.0, "mean_raw": 0.0 }, "r2t2": { "mean_score": 100.0, "mean_raw": 0.0 }, "soniox": { "mean_score": 99.0, "mean_raw": 1.0 }, "stepfun": { "mean_score": 98.0, "mean_raw": 2.0 }, "vibevoice": { "mean_score": 100.0, "mean_raw": 0.0 }, "voxtral": { "mean_score": 100.0, "mean_raw": 0.0 }, "whisper": { "mean_score": 100.0, "mean_raw": 0.0 }, "whisper_v3_turbo": { "mean_score": 98.0, "mean_raw": 2.0 } } }, "3": { "duration_s": 8, "model_scores": { "apple_speechanalyzer": { "mean_score": 100.0, "mean_raw": 0.0 }, "doubao": { "mean_score": 100.0, "mean_raw": 0.0 }, "elevenlabs": { "mean_score": 100.0, "mean_raw": 0.0 }, "firered": { "mean_score": 95.0, "mean_raw": 5.0 }, "firered_aed": { "mean_score": 100.0, "mean_raw": 0.0 }, "flash": { "mean_score": 100.0, "mean_raw": 0.0 }, "funasr_nano": { "mean_score": 100.0, "mean_raw": 0.0 }, "gemini_3_5_flash_lite_stt": { "mean_score": 95.0, "mean_raw": 5.0 }, "gemini_3_5_flash_stt": { "mean_score": 100.0, "mean_raw": 0.0 }, "gemini_3_5_transcribe": { "mean_score": 99.0, "mean_raw": 1.0 }, "gemini_3_6_flash_stt": { "mean_score": 99.0, "mean_raw": 1.0 }, "gemini_flash_stt": { "mean_score": 99.0, "mean_raw": 1.0 }, "gemini_pro_stt": { "mean_score": 99.0, "mean_raw": 1.0 }, "gemini_stt": { "mean_score": 99.0, "mean_raw": 1.0 }, "glm_asr_nano": { "mean_score": 100.0, "mean_raw": 0.0 }, "gpt_transcribe": { "mean_score": 99.0, "mean_raw": 1.0 }, "mai_transcribe_2": { "mean_score": 100.0, "mean_raw": 0.0 }, "mimo": { "mean_score": 100.0, "mean_raw": 0.0 }, "mimo_stt": { "mean_score": 84.0, "mean_raw": 16.0 }, "moss_audio": { "mean_score": 100.0, "mean_raw": 0.0 }, "qwen3_oss": { "mean_score": 100.0, "mean_raw": 0.0 }, "qwen_audio_3_1_asr": { "mean_score": 100.0, "mean_raw": 0.0 }, "r2t2": { "mean_score": 100.0, "mean_raw": 0.0 }, "soniox": { "mean_score": 99.0, "mean_raw": 1.0 }, "stepfun": { "mean_score": 100.0, "mean_raw": 0.0 }, "vibevoice": { "mean_score": 100.0, "mean_raw": 0.0 }, "voxtral": { "mean_score": 100.0, "mean_raw": 0.0 }, "whisper": { "mean_score": 95.0, "mean_raw": 5.0 }, "whisper_v3_turbo": { "mean_score": 99.0, "mean_raw": 1.0 } } }, "4": { "duration_s": 11, "model_scores": { "apple_speechanalyzer": { "mean_score": 100.0, "mean_raw": 0.0 }, "doubao": { "mean_score": 100.0, "mean_raw": 0.0 }, "elevenlabs": { "mean_score": 92.0, "mean_raw": 8.0 }, "firered": { "mean_score": 92.0, "mean_raw": 8.0 }, "firered_aed": { "mean_score": 100.0, "mean_raw": 0.0 }, "flash": { "mean_score": 100.0, "mean_raw": 0.0 }, "funasr_nano": { "mean_score": 100.0, "mean_raw": 0.0 }, "gemini_3_5_flash_lite_stt": { "mean_score": 91.0, "mean_raw": 9.0 }, "gemini_3_5_flash_stt": { "mean_score": 92.0, "mean_raw": 8.0 }, "gemini_3_5_transcribe": { "mean_score": 98.0, "mean_raw": 2.0 }, "gemini_3_6_flash_stt": { "mean_score": 98.0, "mean_raw": 2.0 }, "gemini_3_7_flash_stt": { "mean_score": 100.0, "mean_raw": 0.0 }, "gemini_3_8_flash_stt": { "mean_score": 98.0, "mean_raw": 2.0 }, "gemini_flash_stt": { "mean_score": 99.0, "mean_raw": 1.0 }, "gemini_pro_stt": { "mean_score": 100.0, "mean_raw": 0.0 }, "gemini_stt": { "mean_score": 95.0, "mean_raw": 5.0 }, "glm_asr_nano": { "mean_score": 100.0, "mean_raw": 0.0 }, "gpt_transcribe": { "mean_score": 98.0, "mean_raw": 2.0 }, "mai_transcribe_2": { "mean_score": 100.0, "mean_raw": 0.0 }, "mimo": { "mean_score": 100.0, "mean_raw": 0.0 }, "mimo_stt": { "mean_score": 91.0, "mean_raw": 9.0 }, "moss_audio": { "mean_score": 100.0, "mean_raw": 0.0 }, "qwen3_oss": { "mean_score": 100.0, "mean_raw": 0.0 }, "qwen_audio_3_1_asr": { "mean_score": 100.0, "mean_raw": 0.0 }, "r2t2": { "mean_score": 100.0, "mean_raw": 0.0 }, "soniox": { "mean_score": 100.0, "mean_raw": 0.0 }, "stepfun": { "mean_score": 98.0, "mean_raw": 2.0 }, "vibevoice": { "mean_score": 100.0, "mean_raw": 0.0 }, "voxtral": { "mean_score": 99.0, "mean_raw": 1.0 }, "whisper": { "mean_score": 98.0, "mean_raw": 2.0 }, "whisper_v3_turbo": { "mean_score": 98.0, "mean_raw": 2.0 } } }, "5": { "duration_s": 16, "model_scores": { "apple_speechanalyzer": { "mean_score": 89.0, "mean_raw": 11.0 }, "doubao": { "mean_score": 97.0, "mean_raw": 3.0 }, "elevenlabs": { "mean_score": 98.0, "mean_raw": 2.0 }, "firered": { "mean_score": 97.0, "mean_raw": 3.0 }, "firered_aed": { "mean_score": 98.0, "mean_raw": 2.0 }, "flash": { "mean_score": 98.0, "mean_raw": 2.0 }, "funasr_nano": { "mean_score": 97.0, "mean_raw": 3.0 }, "gemini_3_5_flash_lite_stt": { "mean_score": 97.0, "mean_raw": 3.0 }, "gemini_3_5_flash_stt": { "mean_score": 87.0, "mean_raw": 13.0 }, "gemini_3_5_transcribe": { "mean_score": 99.0, "mean_raw": 1.0 }, "gemini_3_6_flash_stt": { "mean_score": 61.0, "mean_raw": 39.0 }, "gemini_3_7_flash_stt": { "mean_score": 98.0, "mean_raw": 2.0 }, "gemini_3_8_flash_stt": { "mean_score": 97.0, "mean_raw": 3.0 }, "gemini_flash_stt": { "mean_score": 94.0, "mean_raw": 6.0 }, "gemini_pro_stt": { "mean_score": 92.0, "mean_raw": 8.0 }, "gemini_stt": { "mean_score": 92.0, "mean_raw": 8.0 }, "glm_asr_nano": { "mean_score": 97.0, "mean_raw": 3.0 }, "gpt_transcribe": { "mean_score": 97.0, "mean_raw": 3.0 }, "mai_transcribe_2": { "mean_score": 98.0, "mean_raw": 2.0 }, "mimo": { "mean_score": 98.0, "mean_raw": 2.0 }, "mimo_stt": { "mean_score": 98.0, "mean_raw": 2.0 }, "moss_audio": { "mean_score": 97.0, "mean_raw": 3.0 }, "qwen3_oss": { "mean_score": 97.0, "mean_raw": 3.0 }, "qwen_audio_3_1_asr": { "mean_score": 98.0, "mean_raw": 2.0 }, "r2t2": { "mean_score": 93.0, "mean_raw": 7.0 }, "soniox": { "mean_score": 97.0, "mean_raw": 3.0 }, "stepfun": { "mean_score": 98.0, "mean_raw": 2.0 }, "vibevoice": { "mean_score": 90.0, "mean_raw": 10.0 }, "voxtral": { "mean_score": 97.0, "mean_raw": 3.0 }, "whisper": { "mean_score": 99.0, "mean_raw": 1.0 }, "whisper_v3_turbo": { "mean_score": 97.0, "mean_raw": 3.0 } } }, "6": { "duration_s": 17, "model_scores": { "apple_speechanalyzer": { "mean_score": 90.0, "mean_raw": 10.0 }, "doubao": { "mean_score": 84.0, "mean_raw": 16.0 }, "elevenlabs": { "mean_score": 99.0, "mean_raw": 1.0 }, "firered": { "mean_score": 86.0, "mean_raw": 14.0 }, "firered_aed": { "mean_score": 99.0, "mean_raw": 1.0 }, "flash": { "mean_score": 97.0, "mean_raw": 3.0 }, "funasr_nano": { "mean_score": 0.0, "mean_raw": 100.0 }, "gemini_3_5_flash_lite_stt": { "mean_score": 91.0, "mean_raw": 9.0 }, "gemini_3_5_flash_stt": { "mean_score": 86.0, "mean_raw": 14.0 }, "gemini_3_5_transcribe": { "mean_score": 87.0, "mean_raw": 13.0 }, "gemini_3_6_flash_stt": { "mean_score": 94.0, "mean_raw": 6.0 }, "gemini_3_7_flash_stt": { "mean_score": 98.0, "mean_raw": 2.0 }, "gemini_3_8_flash_stt": { "mean_score": 92.0, "mean_raw": 8.0 }, "gemini_flash_stt": { "mean_score": 89.0, "mean_raw": 11.0 }, "gemini_pro_stt": { "mean_score": 97.0, "mean_raw": 3.0 }, "gemini_stt": { "mean_score": 85.0, "mean_raw": 15.0 }, "glm_asr_nano": { "mean_score": 99.0, "mean_raw": 1.0 }, "gpt_transcribe": { "mean_score": 97.0, "mean_raw": 3.0 }, "mai_transcribe_2": { "mean_score": 99.0, "mean_raw": 1.0 }, "mimo": { "mean_score": 99.0, "mean_raw": 1.0 }, "mimo_stt": { "mean_score": 99.0, "mean_raw": 1.0 }, "moss_audio": { "mean_score": 91.0, "mean_raw": 9.0 }, "qwen3_oss": { "mean_score": 100.0, "mean_raw": 0.0 }, "qwen_audio_3_1_asr": { "mean_score": 100.0, "mean_raw": 0.0 }, "r2t2": { "mean_score": 99.0, "mean_raw": 1.0 }, "soniox": { "mean_score": 99.0, "mean_raw": 1.0 }, "stepfun": { "mean_score": 89.0, "mean_raw": 11.0 }, "vibevoice": { "mean_score": 99.0, "mean_raw": 1.0 }, "voxtral": { "mean_score": 95.0, "mean_raw": 5.0 }, "whisper": { "mean_score": 99.0, "mean_raw": 1.0 }, "whisper_v3_turbo": { "mean_score": 97.0, "mean_raw": 3.0 } } }, "7": { "duration_s": 20, "model_scores": { "apple_speechanalyzer": { "mean_score": 78.0, "mean_raw": 22.0 }, "doubao": { "mean_score": 99.0, "mean_raw": 1.0 }, "elevenlabs": { "mean_score": 98.0, "mean_raw": 2.0 }, "firered": { "mean_score": 77.0, "mean_raw": 23.0 }, "firered_aed": { "mean_score": 81.0, "mean_raw": 19.0 }, "flash": { "mean_score": 97.0, "mean_raw": 3.0 }, "funasr_nano": { "mean_score": 90.0, "mean_raw": 10.0 }, "gemini_3_5_flash_lite_stt": { "mean_score": 91.0, "mean_raw": 9.0 }, "gemini_3_5_flash_stt": { "mean_score": 97.0, "mean_raw": 3.0 }, "gemini_3_5_transcribe": { "mean_score": 99.0, "mean_raw": 1.0 }, "gemini_3_6_flash_stt": { "mean_score": 97.0, "mean_raw": 3.0 }, "gemini_3_7_flash_stt": { "mean_score": 98.0, "mean_raw": 2.0 }, "gemini_3_8_flash_stt": { "mean_score": 99.0, "mean_raw": 1.0 }, "gemini_flash_stt": { "mean_score": 94.0, "mean_raw": 6.0 }, "gemini_pro_stt": { "mean_score": 99.0, "mean_raw": 1.0 }, "gemini_stt": { "mean_score": 97.0, "mean_raw": 3.0 }, "glm_asr_nano": { "mean_score": 98.0, "mean_raw": 2.0 }, "gpt_transcribe": { "mean_score": 79.0, "mean_raw": 21.0 }, "mai_transcribe_2": { "mean_score": 99.0, "mean_raw": 1.0 }, "mimo": { "mean_score": 99.0, "mean_raw": 1.0 }, "mimo_stt": { "mean_score": 99.0, "mean_raw": 1.0 }, "moss_audio": { "mean_score": 97.0, "mean_raw": 3.0 }, "qwen3_oss": { "mean_score": 99.0, "mean_raw": 1.0 }, "qwen_audio_3_1_asr": { "mean_score": 100.0, "mean_raw": 0.0 }, "r2t2": { "mean_score": 99.0, "mean_raw": 1.0 }, "soniox": { "mean_score": 88.0, "mean_raw": 12.0 }, "stepfun": { "mean_score": 88.0, "mean_raw": 12.0 }, "vibevoice": { "mean_score": 98.0, "mean_raw": 2.0 }, "voxtral": { "mean_score": 89.0, "mean_raw": 11.0 }, "whisper": { "mean_score": 94.0, "mean_raw": 6.0 }, "whisper_v3_turbo": { "mean_score": 85.0, "mean_raw": 15.0 } } }, "8": { "duration_s": 30, "model_scores": { "apple_speechanalyzer": { "mean_score": 92.0, "mean_raw": 8.0 }, "doubao": { "mean_score": 100.0, "mean_raw": 0.0 }, "elevenlabs": { "mean_score": 100.0, "mean_raw": 0.0 }, "firered": { "mean_score": 92.0, "mean_raw": 8.0 }, "firered_aed": { "mean_score": 100.0, "mean_raw": 0.0 }, "flash": { "mean_score": 100.0, "mean_raw": 0.0 }, "funasr_nano": { "mean_score": 99.0, "mean_raw": 1.0 }, "gemini_3_5_flash_lite_stt": { "mean_score": 92.0, "mean_raw": 8.0 }, "gemini_3_5_flash_stt": { "mean_score": 100.0, "mean_raw": 0.0 }, "gemini_3_5_transcribe": { "mean_score": 99.0, "mean_raw": 1.0 }, "gemini_3_6_flash_stt": { "mean_score": 100.0, "mean_raw": 0.0 }, "gemini_3_7_flash_stt": { "mean_score": 99.0, "mean_raw": 1.0 }, "gemini_3_8_flash_stt": { "mean_score": 99.0, "mean_raw": 1.0 }, "gemini_flash_stt": { "mean_score": 99.0, "mean_raw": 1.0 }, "gemini_pro_stt": { "mean_score": 100.0, "mean_raw": 0.0 }, "gemini_stt": { "mean_score": 99.0, "mean_raw": 1.0 }, "glm_asr_nano": { "mean_score": 100.0, "mean_raw": 0.0 }, "gpt_transcribe": { "mean_score": 100.0, "mean_raw": 0.0 }, "mai_transcribe_2": { "mean_score": 100.0, "mean_raw": 0.0 }, "mimo": { "mean_score": 100.0, "mean_raw": 0.0 }, "mimo_stt": { "mean_score": 99.0, "mean_raw": 1.0 }, "moss_audio": { "mean_score": 99.0, "mean_raw": 1.0 }, "qwen3_oss": { "mean_score": 100.0, "mean_raw": 0.0 }, "qwen_audio_3_1_asr": { "mean_score": 99.0, "mean_raw": 1.0 }, "r2t2": { "mean_score": 97.0, "mean_raw": 3.0 }, "soniox": { "mean_score": 99.0, "mean_raw": 1.0 }, "stepfun": { "mean_score": 92.0, "mean_raw": 8.0 }, "vibevoice": { "mean_score": 99.0, "mean_raw": 1.0 }, "voxtral": { "mean_score": 92.0, "mean_raw": 8.0 }, "whisper": { "mean_score": 99.0, "mean_raw": 1.0 }, "whisper_v3_turbo": { "mean_score": 95.0, "mean_raw": 5.0 } } }, "9": { "duration_s": 32, "model_scores": { "apple_speechanalyzer": { "mean_score": 83.0, "mean_raw": 17.0 }, "doubao": { "mean_score": 98.0, "mean_raw": 2.0 }, "elevenlabs": { "mean_score": 96.0, "mean_raw": 4.0 }, "firered": { "mean_score": 90.0, "mean_raw": 10.0 }, "firered_aed": { "mean_score": 73.0, "mean_raw": 27.0 }, "flash": { "mean_score": 98.0, "mean_raw": 2.0 }, "funasr_nano": { "mean_score": 98.0, "mean_raw": 2.0 }, "gemini_3_5_flash_lite_stt": { "mean_score": 95.0, "mean_raw": 5.0 }, "gemini_3_5_flash_stt": { "mean_score": 94.0, "mean_raw": 6.0 }, "gemini_3_5_transcribe": { "mean_score": 97.0, "mean_raw": 3.0 }, "gemini_3_6_flash_stt": { "mean_score": 96.0, "mean_raw": 4.0 }, "gemini_3_7_flash_stt": { "mean_score": 98.0, "mean_raw": 2.0 }, "gemini_3_8_flash_stt": { "mean_score": 87.0, "mean_raw": 13.0 }, "gemini_flash_stt": { "mean_score": 96.0, "mean_raw": 4.0 }, "gemini_pro_stt": { "mean_score": 95.0, "mean_raw": 5.0 }, "gemini_stt": { "mean_score": 96.0, "mean_raw": 4.0 }, "glm_asr_nano": { "mean_score": 95.0, "mean_raw": 5.0 }, "gpt_transcribe": { "mean_score": 98.0, "mean_raw": 2.0 }, "mai_transcribe_2": { "mean_score": 94.0, "mean_raw": 6.0 }, "mimo": { "mean_score": 98.0, "mean_raw": 2.0 }, "mimo_stt": { "mean_score": 98.0, "mean_raw": 2.0 }, "moss_audio": { "mean_score": 93.0, "mean_raw": 7.0 }, "qwen3_oss": { "mean_score": 96.0, "mean_raw": 4.0 }, "qwen_audio_3_1_asr": { "mean_score": 98.0, "mean_raw": 2.0 }, "r2t2": { "mean_score": 98.0, "mean_raw": 2.0 }, "soniox": { "mean_score": 98.0, "mean_raw": 2.0 }, "stepfun": { "mean_score": 95.0, "mean_raw": 5.0 }, "vibevoice": { "mean_score": 98.0, "mean_raw": 2.0 }, "voxtral": { "mean_score": 90.0, "mean_raw": 10.0 }, "whisper": { "mean_score": 98.0, "mean_raw": 2.0 }, "whisper_v3_turbo": { "mean_score": 95.0, "mean_raw": 5.0 } } }, "10": { "duration_s": 35, "model_scores": { "apple_speechanalyzer": { "mean_score": 98.0, "mean_raw": 2.0 }, "doubao": { "mean_score": 99.0, "mean_raw": 1.0 }, "elevenlabs": { "mean_score": 99.0, "mean_raw": 1.0 }, "firered": { "mean_score": 87.0, "mean_raw": 13.0 }, "firered_aed": { "mean_score": 98.0, "mean_raw": 2.0 }, "flash": { "mean_score": 99.0, "mean_raw": 1.0 }, "funasr_nano": { "mean_score": 99.0, "mean_raw": 1.0 }, "gemini_3_5_flash_lite_stt": { "mean_score": 91.0, "mean_raw": 9.0 }, "gemini_3_5_flash_stt": { "mean_score": 97.0, "mean_raw": 3.0 }, "gemini_3_5_transcribe": { "mean_score": 99.0, "mean_raw": 1.0 }, "gemini_3_6_flash_stt": { "mean_score": 90.0, "mean_raw": 10.0 }, "gemini_3_7_flash_stt": { "mean_score": 97.0, "mean_raw": 3.0 }, "gemini_3_8_flash_stt": { "mean_score": 96.0, "mean_raw": 4.0 }, "gemini_flash_stt": { "mean_score": 94.0, "mean_raw": 6.0 }, "gemini_pro_stt": { "mean_score": 94.0, "mean_raw": 6.0 }, "gemini_stt": { "mean_score": 96.0, "mean_raw": 4.0 }, "glm_asr_nano": { "mean_score": 99.0, "mean_raw": 1.0 }, "gpt_transcribe": { "mean_score": 97.0, "mean_raw": 3.0 }, "mai_transcribe_2": { "mean_score": 100.0, "mean_raw": 0.0 }, "mimo": { "mean_score": 99.0, "mean_raw": 1.0 }, "mimo_stt": { "mean_score": 92.0, "mean_raw": 8.0 }, "moss_audio": { "mean_score": 95.0, "mean_raw": 5.0 }, "qwen3_oss": { "mean_score": 99.0, "mean_raw": 1.0 }, "qwen_audio_3_1_asr": { "mean_score": 99.0, "mean_raw": 1.0 }, "r2t2": { "mean_score": 99.0, "mean_raw": 1.0 }, "soniox": { "mean_score": 98.0, "mean_raw": 2.0 }, "stepfun": { "mean_score": 92.0, "mean_raw": 8.0 }, "vibevoice": { "mean_score": 99.0, "mean_raw": 1.0 }, "voxtral": { "mean_score": 98.0, "mean_raw": 2.0 }, "whisper": { "mean_score": 98.0, "mean_raw": 2.0 }, "whisper_v3_turbo": { "mean_score": 97.0, "mean_raw": 3.0 } } }, "11": { "duration_s": 40, "model_scores": { "apple_speechanalyzer": { "mean_score": 58.0, "mean_raw": 42.0 }, "doubao": { "mean_score": 79.0, "mean_raw": 21.0 }, "elevenlabs": { "mean_score": 91.0, "mean_raw": 9.0 }, "firered": { "mean_score": 73.0, "mean_raw": 27.0 }, "firered_aed": { "mean_score": 82.0, "mean_raw": 18.0 }, "flash": { "mean_score": 77.0, "mean_raw": 23.0 }, "funasr_nano": { "mean_score": 73.0, "mean_raw": 27.0 }, "gemini_3_5_flash_lite_stt": { "mean_score": 66.0, "mean_raw": 34.0 }, "gemini_3_5_flash_stt": { "mean_score": 90.0, "mean_raw": 10.0 }, "gemini_3_5_transcribe": { "mean_score": 85.0, "mean_raw": 15.0 }, "gemini_3_6_flash_stt": { "mean_score": 96.0, "mean_raw": 4.0 }, "gemini_3_7_flash_stt": { "mean_score": 88.0, "mean_raw": 12.0 }, "gemini_3_8_flash_stt": { "mean_score": 95.0, "mean_raw": 5.0 }, "gemini_flash_stt": { "mean_score": 95.0, "mean_raw": 5.0 }, "gemini_pro_stt": { "mean_score": 92.0, "mean_raw": 8.0 }, "gemini_stt": { "mean_score": 85.0, "mean_raw": 15.0 }, "glm_asr_nano": { "mean_score": 83.0, "mean_raw": 17.0 }, "gpt_transcribe": { "mean_score": 90.0, "mean_raw": 10.0 }, "mai_transcribe_2": { "mean_score": 96.0, "mean_raw": 4.0 }, "mimo": { "mean_score": 86.0, "mean_raw": 14.0 }, "mimo_stt": { "mean_score": 84.0, "mean_raw": 16.0 }, "moss_audio": { "mean_score": 0.0, "mean_raw": 100.0 }, "qwen3_oss": { "mean_score": 83.0, "mean_raw": 17.0 }, "qwen_audio_3_1_asr": { "mean_score": 81.0, "mean_raw": 19.0 }, "r2t2": { "mean_score": 82.0, "mean_raw": 18.0 }, "soniox": { "mean_score": 79.0, "mean_raw": 21.0 }, "stepfun": { "mean_score": 77.0, "mean_raw": 23.0 }, "vibevoice": { "mean_score": 83.0, "mean_raw": 17.0 }, "voxtral": { "mean_score": 77.0, "mean_raw": 23.0 }, "whisper": { "mean_score": 87.0, "mean_raw": 13.0 }, "whisper_v3_turbo": { "mean_score": 88.0, "mean_raw": 12.0 } } }, "12": { "duration_s": 46, "model_scores": { "apple_speechanalyzer": { "mean_score": 77.0, "mean_raw": 23.0 }, "doubao": { "mean_score": 98.0, "mean_raw": 2.0 }, "elevenlabs": { "mean_score": 92.0, "mean_raw": 8.0 }, "firered": { "mean_score": 57.0, "mean_raw": 43.0 }, "firered_aed": { "mean_score": 96.0, "mean_raw": 4.0 }, "flash": { "mean_score": 98.0, "mean_raw": 2.0 }, "funasr_nano": { "mean_score": 97.0, "mean_raw": 3.0 }, "gemini_3_5_flash_lite_stt": { "mean_score": 87.0, "mean_raw": 13.0 }, "gemini_3_5_flash_stt": { "mean_score": 88.0, "mean_raw": 12.0 }, "gemini_3_5_transcribe": { "mean_score": 91.0, "mean_raw": 9.0 }, "gemini_3_6_flash_stt": { "mean_score": 82.0, "mean_raw": 18.0 }, "gemini_3_7_flash_stt": { "mean_score": 95.0, "mean_raw": 5.0 }, "gemini_3_8_flash_stt": { "mean_score": 93.0, "mean_raw": 7.0 }, "gemini_flash_stt": { "mean_score": 87.0, "mean_raw": 13.0 }, "gemini_pro_stt": { "mean_score": 95.0, "mean_raw": 5.0 }, "gemini_stt": { "mean_score": 86.0, "mean_raw": 14.0 }, "glm_asr_nano": { "mean_score": 75.0, "mean_raw": 25.0 }, "gpt_transcribe": { "mean_score": 85.0, "mean_raw": 15.0 }, "mai_transcribe_2": { "mean_score": 97.0, "mean_raw": 3.0 }, "mimo": { "mean_score": 100.0, "mean_raw": 0.0 }, "mimo_stt": { "mean_score": 91.0, "mean_raw": 9.0 }, "moss_audio": { "mean_score": 53.0, "mean_raw": 47.0 }, "qwen3_oss": { "mean_score": 89.0, "mean_raw": 11.0 }, "qwen_audio_3_1_asr": { "mean_score": 96.0, "mean_raw": 4.0 }, "r2t2": { "mean_score": 98.0, "mean_raw": 2.0 }, "soniox": { "mean_score": 85.0, "mean_raw": 15.0 }, "stepfun": { "mean_score": 96.0, "mean_raw": 4.0 }, "vibevoice": { "mean_score": 98.0, "mean_raw": 2.0 }, "voxtral": { "mean_score": 68.0, "mean_raw": 32.0 }, "whisper": { "mean_score": 96.0, "mean_raw": 4.0 }, "whisper_v3_turbo": { "mean_score": 95.0, "mean_raw": 5.0 } } }, "13": { "duration_s": 60, "model_scores": { "apple_speechanalyzer": { "mean_score": 44.0, "mean_raw": 56.0 }, "doubao": { "mean_score": 80.0, "mean_raw": 20.0 }, "elevenlabs": { "mean_score": 68.0, "mean_raw": 32.0 }, "firered": { "mean_score": 10.0, "mean_raw": 90.0 }, "firered_aed": { "mean_score": 51.0, "mean_raw": 49.0 }, "flash": { "mean_score": 70.0, "mean_raw": 30.0 }, "funasr_nano": { "mean_score": 70.0, "mean_raw": 30.0 }, "gemini_3_5_flash_lite_stt": { "mean_score": 60.0, "mean_raw": 40.0 }, "gemini_3_5_flash_stt": { "mean_score": 55.0, "mean_raw": 45.0 }, "gemini_3_5_transcribe": { "mean_score": 78.0, "mean_raw": 22.0 }, "gemini_3_6_flash_stt": { "mean_score": 70.0, "mean_raw": 30.0 }, "gemini_3_7_flash_stt": { "mean_score": 83.0, "mean_raw": 17.0 }, "gemini_3_8_flash_stt": { "mean_score": 97.0, "mean_raw": 3.0 }, "gemini_flash_stt": { "mean_score": 78.0, "mean_raw": 22.0 }, "gemini_pro_stt": { "mean_score": 82.0, "mean_raw": 18.0 }, "gemini_stt": { "mean_score": 78.0, "mean_raw": 22.0 }, "glm_asr_nano": { "mean_score": 61.0, "mean_raw": 39.0 }, "gpt_transcribe": { "mean_score": 83.0, "mean_raw": 17.0 }, "mai_transcribe_2": { "mean_score": 77.0, "mean_raw": 23.0 }, "mimo": { "mean_score": 70.0, "mean_raw": 30.0 }, "mimo_stt": { "mean_score": 70.0, "mean_raw": 30.0 }, "moss_audio": { "mean_score": 57.0, "mean_raw": 43.0 }, "qwen3_oss": { "mean_score": 59.0, "mean_raw": 41.0 }, "qwen_audio_3_1_asr": { "mean_score": 71.0, "mean_raw": 29.0 }, "r2t2": { "mean_score": 59.0, "mean_raw": 41.0 }, "soniox": { "mean_score": 70.0, "mean_raw": 30.0 }, "stepfun": { "mean_score": 68.0, "mean_raw": 32.0 }, "vibevoice": { "mean_score": 65.0, "mean_raw": 35.0 }, "voxtral": { "mean_score": 60.0, "mean_raw": 40.0 }, "whisper": { "mean_score": 65.0, "mean_raw": 35.0 }, "whisper_v3_turbo": { "mean_score": 67.0, "mean_raw": 33.0 } } }, "14": { "duration_s": 64, "model_scores": { "apple_speechanalyzer": { "mean_score": 73.0, "mean_raw": 27.0 }, "doubao": { "mean_score": 88.0, "mean_raw": 12.0 }, "elevenlabs": { "mean_score": 99.0, "mean_raw": 1.0 }, "firered": { "mean_score": 63.0, "mean_raw": 37.0 }, "firered_aed": { "mean_score": 67.0, "mean_raw": 33.0 }, "flash": { "mean_score": 82.0, "mean_raw": 18.0 }, "funasr_nano": { "mean_score": 86.0, "mean_raw": 14.0 }, "gemini_3_5_flash_lite_stt": { "mean_score": 62.0, "mean_raw": 38.0 }, "gemini_3_5_flash_stt": { "mean_score": 81.0, "mean_raw": 19.0 }, "gemini_3_5_transcribe": { "mean_score": 90.0, "mean_raw": 10.0 }, "gemini_3_6_flash_stt": { "mean_score": 73.0, "mean_raw": 27.0 }, "gemini_3_7_flash_stt": { "mean_score": 85.0, "mean_raw": 15.0 }, "gemini_3_8_flash_stt": { "mean_score": 91.0, "mean_raw": 9.0 }, "gemini_flash_stt": { "mean_score": 64.0, "mean_raw": 36.0 }, "gemini_pro_stt": { "mean_score": 70.0, "mean_raw": 30.0 }, "gemini_stt": { "mean_score": 77.0, "mean_raw": 23.0 }, "glm_asr_nano": { "mean_score": 92.0, "mean_raw": 8.0 }, "gpt_transcribe": { "mean_score": 82.0, "mean_raw": 18.0 }, "mai_transcribe_2": { "mean_score": 95.0, "mean_raw": 5.0 }, "mimo": { "mean_score": 84.0, "mean_raw": 16.0 }, "mimo_stt": { "mean_score": 78.0, "mean_raw": 22.0 }, "moss_audio": { "mean_score": 70.0, "mean_raw": 30.0 }, "qwen3_oss": { "mean_score": 80.0, "mean_raw": 20.0 }, "qwen_audio_3_1_asr": { "mean_score": 71.0, "mean_raw": 29.0 }, "r2t2": { "mean_score": 90.0, "mean_raw": 10.0 }, "soniox": { "mean_score": 80.0, "mean_raw": 20.0 }, "stepfun": { "mean_score": 67.0, "mean_raw": 33.0 }, "vibevoice": { "mean_score": 78.0, "mean_raw": 22.0 }, "voxtral": { "mean_score": 36.0, "mean_raw": 64.0 }, "whisper": { "mean_score": 82.0, "mean_raw": 18.0 }, "whisper_v3_turbo": { "mean_score": 77.0, "mean_raw": 23.0 } } }, "15": { "duration_s": 69, "model_scores": { "apple_speechanalyzer": { "mean_score": 88.0, "mean_raw": 12.0 }, "doubao": { "mean_score": 100.0, "mean_raw": 0.0 }, "elevenlabs": { "mean_score": 97.0, "mean_raw": 3.0 }, "firered": { "mean_score": 60.0, "mean_raw": 40.0 }, "firered_aed": { "mean_score": 66.0, "mean_raw": 34.0 }, "flash": { "mean_score": 97.0, "mean_raw": 3.0 }, "gemini_3_5_flash_lite_stt": { "mean_score": 95.0, "mean_raw": 5.0 }, "gemini_3_5_flash_stt": { "mean_score": 82.0, "mean_raw": 18.0 }, "gemini_3_5_transcribe": { "mean_score": 97.0, "mean_raw": 3.0 }, "gemini_3_6_flash_stt": { "mean_score": 91.0, "mean_raw": 9.0 }, "gemini_3_7_flash_stt": { "mean_score": 90.0, "mean_raw": 10.0 }, "gemini_3_8_flash_stt": { "mean_score": 86.0, "mean_raw": 14.0 }, "gemini_flash_stt": { "mean_score": 85.0, "mean_raw": 15.0 }, "gemini_pro_stt": { "mean_score": 90.0, "mean_raw": 10.0 }, "gemini_stt": { "mean_score": 95.0, "mean_raw": 5.0 }, "glm_asr_nano": { "mean_score": 95.0, "mean_raw": 5.0 }, "gpt_transcribe": { "mean_score": 95.0, "mean_raw": 5.0 }, "mai_transcribe_2": { "mean_score": 98.0, "mean_raw": 2.0 }, "mimo": { "mean_score": 97.0, "mean_raw": 3.0 }, "mimo_stt": { "mean_score": 98.0, "mean_raw": 2.0 }, "moss_audio": { "mean_score": 5.0, "mean_raw": 95.0 }, "qwen3_oss": { "mean_score": 98.0, "mean_raw": 2.0 }, "qwen_audio_3_1_asr": { "mean_score": 98.0, "mean_raw": 2.0 }, "r2t2": { "mean_score": 99.0, "mean_raw": 1.0 }, "soniox": { "mean_score": 96.0, "mean_raw": 4.0 }, "stepfun": { "mean_score": 98.0, "mean_raw": 2.0 }, "vibevoice": { "mean_score": 98.0, "mean_raw": 2.0 }, "voxtral": { "mean_score": 88.0, "mean_raw": 12.0 }, "whisper": { "mean_score": 97.0, "mean_raw": 3.0 }, "whisper_v3_turbo": { "mean_score": 70.0, "mean_raw": 30.0 } } }, "16": { "duration_s": 81, "model_scores": { "apple_speechanalyzer": { "mean_score": 58.0, "mean_raw": 42.0 }, "doubao": { "mean_score": 77.0, "mean_raw": 23.0 }, "elevenlabs": { "mean_score": 84.0, "mean_raw": 16.0 }, "firered": { "mean_score": 50.0, "mean_raw": 50.0 }, "firered_aed": { "mean_score": 74.0, "mean_raw": 26.0 }, "flash": { "mean_score": 92.0, "mean_raw": 8.0 }, "funasr_nano": { "mean_score": 65.0, "mean_raw": 35.0 }, "gemini_3_5_flash_lite_stt": { "mean_score": 78.0, "mean_raw": 22.0 }, "gemini_3_5_flash_stt": { "mean_score": 90.0, "mean_raw": 10.0 }, "gemini_3_5_transcribe": { "mean_score": 81.0, "mean_raw": 19.0 }, "gemini_3_6_flash_stt": { "mean_score": 85.0, "mean_raw": 15.0 }, "gemini_3_7_flash_stt": { "mean_score": 89.0, "mean_raw": 11.0 }, "gemini_3_8_flash_stt": { "mean_score": 83.0, "mean_raw": 17.0 }, "gemini_flash_stt": { "mean_score": 85.0, "mean_raw": 15.0 }, "gemini_pro_stt": { "mean_score": 84.0, "mean_raw": 16.0 }, "gemini_stt": { "mean_score": 74.0, "mean_raw": 26.0 }, "glm_asr_nano": { "mean_score": 79.0, "mean_raw": 21.0 }, "gpt_transcribe": { "mean_score": 89.0, "mean_raw": 11.0 }, "mai_transcribe_2": { "mean_score": 94.0, "mean_raw": 6.0 }, "mimo": { "mean_score": 93.0, "mean_raw": 7.0 }, "mimo_stt": { "mean_score": 90.0, "mean_raw": 10.0 }, "moss_audio": { "mean_score": 50.0, "mean_raw": 50.0 }, "qwen3_oss": { "mean_score": 74.0, "mean_raw": 26.0 }, "qwen_audio_3_1_asr": { "mean_score": 82.0, "mean_raw": 18.0 }, "r2t2": { "mean_score": 86.0, "mean_raw": 14.0 }, "soniox": { "mean_score": 91.0, "mean_raw": 9.0 }, "stepfun": { "mean_score": 73.0, "mean_raw": 27.0 }, "vibevoice": { "mean_score": 91.0, "mean_raw": 9.0 }, "voxtral": { "mean_score": 69.0, "mean_raw": 31.0 }, "whisper": { "mean_score": 84.0, "mean_raw": 16.0 }, "whisper_v3_turbo": { "mean_score": 72.0, "mean_raw": 28.0 } } }, "17": { "duration_s": 100, "model_scores": { "apple_speechanalyzer": { "mean_score": 63.0, "mean_raw": 37.0 }, "doubao": { "mean_score": 71.0, "mean_raw": 29.0 }, "elevenlabs": { "mean_score": 98.0, "mean_raw": 2.0 }, "firered": { "mean_score": 19.0, "mean_raw": 81.0 }, "firered_aed": { "mean_score": 48.0, "mean_raw": 52.0 }, "flash": { "mean_score": 80.0, "mean_raw": 20.0 }, "funasr_nano": { "mean_score": 36.0, "mean_raw": 64.0 }, "gemini_3_5_flash_lite_stt": { "mean_score": 80.0, "mean_raw": 20.0 }, "gemini_3_5_flash_stt": { "mean_score": 90.0, "mean_raw": 10.0 }, "gemini_3_5_transcribe": { "mean_score": 85.0, "mean_raw": 15.0 }, "gemini_3_6_flash_stt": { "mean_score": 89.0, "mean_raw": 11.0 }, "gemini_3_7_flash_stt": { "mean_score": 77.0, "mean_raw": 23.0 }, "gemini_3_8_flash_stt": { "mean_score": 95.0, "mean_raw": 5.0 }, "gemini_flash_stt": { "mean_score": 87.0, "mean_raw": 13.0 }, "gemini_pro_stt": { "mean_score": 77.0, "mean_raw": 23.0 }, "gemini_stt": { "mean_score": 70.0, "mean_raw": 30.0 }, "glm_asr_nano": { "mean_score": 81.0, "mean_raw": 19.0 }, "gpt_transcribe": { "mean_score": 95.0, "mean_raw": 5.0 }, "mai_transcribe_2": { "mean_score": 88.0, "mean_raw": 12.0 }, "mimo": { "mean_score": 95.0, "mean_raw": 5.0 }, "mimo_stt": { "mean_score": 90.0, "mean_raw": 10.0 }, "moss_audio": { "mean_score": 60.0, "mean_raw": 40.0 }, "qwen3_oss": { "mean_score": 77.0, "mean_raw": 23.0 }, "qwen_audio_3_1_asr": { "mean_score": 100.0, "mean_raw": 0.0 }, "r2t2": { "mean_score": 87.0, "mean_raw": 13.0 }, "soniox": { "mean_score": 73.0, "mean_raw": 27.0 }, "stepfun": { "mean_score": 80.0, "mean_raw": 20.0 }, "vibevoice": { "mean_score": 88.0, "mean_raw": 12.0 }, "voxtral": { "mean_score": 77.0, "mean_raw": 23.0 }, "whisper": { "mean_score": 85.0, "mean_raw": 15.0 }, "whisper_v3_turbo": { "mean_score": 70.0, "mean_raw": 30.0 } } }, "18": { "duration_s": 125, "model_scores": { "apple_speechanalyzer": { "mean_score": 43.0, "mean_raw": 57.0 }, "doubao": { "mean_score": 81.0, "mean_raw": 19.0 }, "elevenlabs": { "mean_score": 100.0, "mean_raw": 0.0 }, "firered": { "mean_score": 54.0, "mean_raw": 46.0 }, "firered_aed": { "mean_score": 82.0, "mean_raw": 18.0 }, "flash": { "mean_score": 91.0, "mean_raw": 9.0 }, "funasr_nano": { "mean_score": 66.0, "mean_raw": 34.0 }, "gemini_3_5_flash_lite_stt": { "mean_score": 55.0, "mean_raw": 45.0 }, "gemini_3_5_flash_stt": { "mean_score": 81.0, "mean_raw": 19.0 }, "gemini_3_5_transcribe": { "mean_score": 82.0, "mean_raw": 18.0 }, "gemini_3_6_flash_stt": { "mean_score": 73.0, "mean_raw": 27.0 }, "gemini_3_7_flash_stt": { "mean_score": 75.0, "mean_raw": 25.0 }, "gemini_3_8_flash_stt": { "mean_score": 99.0, "mean_raw": 1.0 }, "gemini_flash_stt": { "mean_score": 90.0, "mean_raw": 10.0 }, "gemini_pro_stt": { "mean_score": 88.0, "mean_raw": 12.0 }, "gemini_stt": { "mean_score": 67.0, "mean_raw": 33.0 }, "glm_asr_nano": { "mean_score": 87.0, "mean_raw": 13.0 }, "gpt_transcribe": { "mean_score": 84.0, "mean_raw": 16.0 }, "mai_transcribe_2": { "mean_score": 90.0, "mean_raw": 10.0 }, "mimo": { "mean_score": 87.0, "mean_raw": 13.0 }, "mimo_stt": { "mean_score": 86.0, "mean_raw": 14.0 }, "moss_audio": { "mean_score": 82.0, "mean_raw": 18.0 }, "qwen3_oss": { "mean_score": 80.0, "mean_raw": 20.0 }, "qwen_audio_3_1_asr": { "mean_score": 88.0, "mean_raw": 12.0 }, "r2t2": { "mean_score": 76.0, "mean_raw": 24.0 }, "soniox": { "mean_score": 80.0, "mean_raw": 20.0 }, "stepfun": { "mean_score": 81.0, "mean_raw": 19.0 }, "vibevoice": { "mean_score": 86.0, "mean_raw": 14.0 }, "voxtral": { "mean_score": 60.0, "mean_raw": 40.0 }, "whisper": { "mean_score": 75.0, "mean_raw": 25.0 }, "whisper_v3_turbo": { "mean_score": 25.0, "mean_raw": 75.0 } } }, "19": { "duration_s": 133, "model_scores": { "apple_speechanalyzer": { "mean_score": 0.0, "mean_raw": 131.0 }, "doubao": { "mean_score": 84.0, "mean_raw": 16.0 }, "elevenlabs": { "mean_score": 90.0, "mean_raw": 10.0 }, "firered": { "mean_score": 0.0, "mean_raw": 156.0 }, "firered_aed": { "mean_score": 1.0, "mean_raw": 99.0 }, "flash": { "mean_score": 75.0, "mean_raw": 25.0 }, "funasr_nano": { "mean_score": 0.0, "mean_raw": 100.0 }, "gemini_3_5_flash_lite_stt": { "mean_score": 39.0, "mean_raw": 61.0 }, "gemini_3_5_flash_stt": { "mean_score": 60.0, "mean_raw": 40.0 }, "gemini_3_5_transcribe": { "mean_score": 48.0, "mean_raw": 52.0 }, "gemini_3_6_flash_stt": { "mean_score": 57.0, "mean_raw": 43.0 }, "gemini_3_7_flash_stt": { "mean_score": 70.0, "mean_raw": 30.0 }, "gemini_3_8_flash_stt": { "mean_score": 72.0, "mean_raw": 28.0 }, "gemini_flash_stt": { "mean_score": 65.0, "mean_raw": 35.0 }, "gemini_pro_stt": { "mean_score": 87.0, "mean_raw": 13.0 }, "gemini_stt": { "mean_score": 55.0, "mean_raw": 45.0 }, "glm_asr_nano": { "mean_score": 47.0, "mean_raw": 53.0 }, "gpt_transcribe": { "mean_score": 77.0, "mean_raw": 23.0 }, "mai_transcribe_2": { "mean_score": 70.0, "mean_raw": 30.0 }, "mimo": { "mean_score": 59.0, "mean_raw": 41.0 }, "mimo_stt": { "mean_score": 47.0, "mean_raw": 53.0 }, "qwen3_oss": { "mean_score": 63.0, "mean_raw": 37.0 }, "qwen_audio_3_1_asr": { "mean_score": 54.0, "mean_raw": 46.0 }, "r2t2": { "mean_score": 62.0, "mean_raw": 38.0 }, "soniox": { "mean_score": 55.0, "mean_raw": 45.0 }, "stepfun": { "mean_score": 44.0, "mean_raw": 56.0 }, "vibevoice": { "mean_score": 61.0, "mean_raw": 39.0 }, "voxtral": { "mean_score": 67.0, "mean_raw": 33.0 }, "whisper": { "mean_score": 62.0, "mean_raw": 38.0 }, "whisper_v3_turbo": { "mean_score": 34.0, "mean_raw": 66.0 } } }, "20": { "duration_s": 164, "model_scores": { "apple_speechanalyzer": { "mean_score": 81.0, "mean_raw": 19.0 }, "doubao": { "mean_score": 96.0, "mean_raw": 4.0 }, "elevenlabs": { "mean_score": 98.0, "mean_raw": 2.0 }, "firered": { "mean_score": 65.0, "mean_raw": 35.0 }, "firered_aed": { "mean_score": 93.0, "mean_raw": 7.0 }, "flash": { "mean_score": 90.0, "mean_raw": 10.0 }, "funasr_nano": { "mean_score": 0.0, "mean_raw": 100.0 }, "gemini_3_5_flash_lite_stt": { "mean_score": 0.0, "mean_raw": 124.0 }, "gemini_3_5_flash_stt": { "mean_score": 67.0, "mean_raw": 33.0 }, "gemini_3_5_transcribe": { "mean_score": 56.0, "mean_raw": 44.0 }, "gemini_3_6_flash_stt": { "mean_score": 75.0, "mean_raw": 25.0 }, "gemini_3_7_flash_stt": { "mean_score": 82.0, "mean_raw": 18.0 }, "gemini_3_8_flash_stt": { "mean_score": 89.0, "mean_raw": 11.0 }, "gemini_flash_stt": { "mean_score": 89.0, "mean_raw": 11.0 }, "gemini_pro_stt": { "mean_score": 82.0, "mean_raw": 18.0 }, "gemini_stt": { "mean_score": 50.0, "mean_raw": 50.0 }, "glm_asr_nano": { "mean_score": 94.0, "mean_raw": 6.0 }, "gpt_transcribe": { "mean_score": 80.0, "mean_raw": 20.0 }, "mai_transcribe_2": { "mean_score": 100.0, "mean_raw": 0.0 }, "mimo": { "mean_score": 91.0, "mean_raw": 9.0 }, "mimo_stt": { "mean_score": 91.0, "mean_raw": 9.0 }, "moss_audio": { "mean_score": 82.0, "mean_raw": 18.0 }, "qwen3_oss": { "mean_score": 80.0, "mean_raw": 20.0 }, "qwen_audio_3_1_asr": { "mean_score": 97.0, "mean_raw": 3.0 }, "r2t2": { "mean_score": 93.0, "mean_raw": 7.0 }, "soniox": { "mean_score": 78.0, "mean_raw": 22.0 }, "stepfun": { "mean_score": 95.0, "mean_raw": 5.0 }, "vibevoice": { "mean_score": 98.0, "mean_raw": 2.0 }, "voxtral": { "mean_score": 64.0, "mean_raw": 36.0 }, "whisper": { "mean_score": 83.0, "mean_raw": 17.0 }, "whisper_v3_turbo": { "mean_score": 84.0, "mean_raw": 16.0 } } }, "21": { "duration_s": 5, "model_scores": { "apple_speechanalyzer": { "mean_score": 92.0, "mean_raw": 8.0 }, "doubao": { "mean_score": 100.0, "mean_raw": 0.0 }, "elevenlabs": { "mean_score": 100.0, "mean_raw": 0.0 }, "firered": { "mean_score": 91.0, "mean_raw": 9.0 }, "firered_aed": { "mean_score": 99.0, "mean_raw": 1.0 }, "flash": { "mean_score": 99.0, "mean_raw": 1.0 }, "funasr_nano": { "mean_score": 99.0, "mean_raw": 1.0 }, "gemini_3_5_flash_lite_stt": { "mean_score": 100.0, "mean_raw": 0.0 }, "gemini_3_5_flash_stt": { "mean_score": 99.0, "mean_raw": 1.0 }, "gemini_3_5_transcribe": { "mean_score": 100.0, "mean_raw": 0.0 }, "gemini_3_6_flash_stt": { "mean_score": 100.0, "mean_raw": 0.0 }, "gemini_3_7_flash_stt": { "mean_score": 99.0, "mean_raw": 1.0 }, "gemini_3_8_flash_stt": { "mean_score": 99.0, "mean_raw": 1.0 }, "gemini_flash_stt": { "mean_score": 100.0, "mean_raw": 0.0 }, "gemini_pro_stt": { "mean_score": 100.0, "mean_raw": 0.0 }, "gemini_stt": { "mean_score": 100.0, "mean_raw": 0.0 }, "glm_asr_nano": { "mean_score": 99.0, "mean_raw": 1.0 }, "gpt_transcribe": { "mean_score": 100.0, "mean_raw": 0.0 }, "mai_transcribe_2": { "mean_score": 100.0, "mean_raw": 0.0 }, "mimo": { "mean_score": 99.0, "mean_raw": 1.0 }, "mimo_stt": { "mean_score": 99.0, "mean_raw": 1.0 }, "moss_audio": { "mean_score": 99.0, "mean_raw": 1.0 }, "qwen_audio_3_1_asr": { "mean_score": 100.0, "mean_raw": 0.0 }, "r2t2": { "mean_score": 99.0, "mean_raw": 1.0 }, "soniox": { "mean_score": 99.0, "mean_raw": 1.0 }, "stepfun": { "mean_score": 100.0, "mean_raw": 0.0 }, "vibevoice": { "mean_score": 100.0, "mean_raw": 0.0 }, "voxtral": { "mean_score": 100.0, "mean_raw": 0.0 }, "whisper": { "mean_score": 99.0, "mean_raw": 1.0 }, "whisper_v3_turbo": { "mean_score": 100.0, "mean_raw": 0.0 } } }, "22": { "duration_s": 7, "model_scores": { "apple_speechanalyzer": { "mean_score": 90.0, "mean_raw": 10.0 }, "doubao": { "mean_score": 90.0, "mean_raw": 10.0 }, "elevenlabs": { "mean_score": 94.0, "mean_raw": 6.0 }, "firered": { "mean_score": 94.0, "mean_raw": 6.0 }, "firered_aed": { "mean_score": 94.0, "mean_raw": 6.0 }, "flash": { "mean_score": 90.0, "mean_raw": 10.0 }, "funasr_nano": { "mean_score": 90.0, "mean_raw": 10.0 }, "gemini_3_5_flash_lite_stt": { "mean_score": 82.0, "mean_raw": 18.0 }, "gemini_3_5_flash_stt": { "mean_score": 86.0, "mean_raw": 14.0 }, "gemini_3_5_transcribe": { "mean_score": 88.0, "mean_raw": 12.0 }, "gemini_3_6_flash_stt": { "mean_score": 86.0, "mean_raw": 14.0 }, "gemini_3_7_flash_stt": { "mean_score": 87.0, "mean_raw": 13.0 }, "gemini_3_8_flash_stt": { "mean_score": 86.0, "mean_raw": 14.0 }, "gemini_flash_stt": { "mean_score": 85.0, "mean_raw": 15.0 }, "gemini_pro_stt": { "mean_score": 90.0, "mean_raw": 10.0 }, "gemini_stt": { "mean_score": 86.0, "mean_raw": 14.0 }, "glm_asr_nano": { "mean_score": 90.0, "mean_raw": 10.0 }, "gpt_transcribe": { "mean_score": 94.0, "mean_raw": 6.0 }, "mai_transcribe_2": { "mean_score": 100.0, "mean_raw": 0.0 }, "mimo": { "mean_score": 94.0, "mean_raw": 6.0 }, "mimo_stt": { "mean_score": 88.0, "mean_raw": 12.0 }, "moss_audio": { "mean_score": 89.0, "mean_raw": 11.0 }, "qwen3_oss": { "mean_score": 94.0, "mean_raw": 6.0 }, "qwen_audio_3_1_asr": { "mean_score": 94.0, "mean_raw": 6.0 }, "r2t2": { "mean_score": 90.0, "mean_raw": 10.0 }, "soniox": { "mean_score": 90.0, "mean_raw": 10.0 }, "stepfun": { "mean_score": 94.0, "mean_raw": 6.0 }, "vibevoice": { "mean_score": 95.0, "mean_raw": 5.0 }, "voxtral": { "mean_score": 94.0, "mean_raw": 6.0 }, "whisper": { "mean_score": 98.0, "mean_raw": 2.0 }, "whisper_v3_turbo": { "mean_score": 93.0, "mean_raw": 7.0 } } }, "23": { "duration_s": 10, "model_scores": { "apple_speechanalyzer": { "mean_score": 92.0, "mean_raw": 8.0 }, "doubao": { "mean_score": 100.0, "mean_raw": 0.0 }, "elevenlabs": { "mean_score": 100.0, "mean_raw": 0.0 }, "firered": { "mean_score": 92.0, "mean_raw": 8.0 }, "firered_aed": { "mean_score": 100.0, "mean_raw": 0.0 }, "flash": { "mean_score": 100.0, "mean_raw": 0.0 }, "funasr_nano": { "mean_score": 100.0, "mean_raw": 0.0 }, "gemini_3_5_flash_lite_stt": { "mean_score": 99.0, "mean_raw": 1.0 }, "gemini_3_5_flash_stt": { "mean_score": 100.0, "mean_raw": 0.0 }, "gemini_3_5_transcribe": { "mean_score": 100.0, "mean_raw": 0.0 }, "gemini_3_6_flash_stt": { "mean_score": 100.0, "mean_raw": 0.0 }, "gemini_3_7_flash_stt": { "mean_score": 100.0, "mean_raw": 0.0 }, "gemini_3_8_flash_stt": { "mean_score": 99.0, "mean_raw": 1.0 }, "gemini_flash_stt": { "mean_score": 100.0, "mean_raw": 0.0 }, "gemini_pro_stt": { "mean_score": 100.0, "mean_raw": 0.0 }, "gemini_stt": { "mean_score": 98.0, "mean_raw": 2.0 }, "glm_asr_nano": { "mean_score": 100.0, "mean_raw": 0.0 }, "gpt_transcribe": { "mean_score": 100.0, "mean_raw": 0.0 }, "mai_transcribe_2": { "mean_score": 100.0, "mean_raw": 0.0 }, "mimo": { "mean_score": 100.0, "mean_raw": 0.0 }, "mimo_stt": { "mean_score": 100.0, "mean_raw": 0.0 }, "moss_audio": { "mean_score": 100.0, "mean_raw": 0.0 }, "qwen3_oss": { "mean_score": 100.0, "mean_raw": 0.0 }, "qwen_audio_3_1_asr": { "mean_score": 100.0, "mean_raw": 0.0 }, "r2t2": { "mean_score": 100.0, "mean_raw": 0.0 }, "soniox": { "mean_score": 100.0, "mean_raw": 0.0 }, "stepfun": { "mean_score": 100.0, "mean_raw": 0.0 }, "vibevoice": { "mean_score": 100.0, "mean_raw": 0.0 }, "voxtral": { "mean_score": 100.0, "mean_raw": 0.0 }, "whisper": { "mean_score": 100.0, "mean_raw": 0.0 }, "whisper_v3_turbo": { "mean_score": 100.0, "mean_raw": 0.0 } } }, "24": { "duration_s": 14, "model_scores": { "apple_speechanalyzer": { "mean_score": 90.0, "mean_raw": 10.0 }, "doubao": { "mean_score": 90.0, "mean_raw": 10.0 }, "elevenlabs": { "mean_score": 91.0, "mean_raw": 9.0 }, "firered": { "mean_score": 90.0, "mean_raw": 10.0 }, "firered_aed": { "mean_score": 90.0, "mean_raw": 10.0 }, "flash": { "mean_score": 90.0, "mean_raw": 10.0 }, "funasr_nano": { "mean_score": 90.0, "mean_raw": 10.0 }, "gemini_3_5_flash_lite_stt": { "mean_score": 77.0, "mean_raw": 23.0 }, "gemini_3_5_flash_stt": { "mean_score": 85.0, "mean_raw": 15.0 }, "gemini_3_5_transcribe": { "mean_score": 89.0, "mean_raw": 11.0 }, "gemini_3_6_flash_stt": { "mean_score": 95.0, "mean_raw": 5.0 }, "gemini_3_7_flash_stt": { "mean_score": 99.0, "mean_raw": 1.0 }, "gemini_3_8_flash_stt": { "mean_score": 91.0, "mean_raw": 9.0 }, "gemini_flash_stt": { "mean_score": 88.0, "mean_raw": 12.0 }, "gemini_pro_stt": { "mean_score": 89.0, "mean_raw": 11.0 }, "gemini_stt": { "mean_score": 89.0, "mean_raw": 11.0 }, "glm_asr_nano": { "mean_score": 99.0, "mean_raw": 1.0 }, "gpt_transcribe": { "mean_score": 89.0, "mean_raw": 11.0 }, "mai_transcribe_2": { "mean_score": 92.0, "mean_raw": 8.0 }, "mimo": { "mean_score": 91.0, "mean_raw": 9.0 }, "mimo_stt": { "mean_score": 87.0, "mean_raw": 13.0 }, "moss_audio": { "mean_score": 90.0, "mean_raw": 10.0 }, "qwen3_oss": { "mean_score": 90.0, "mean_raw": 10.0 }, "qwen_audio_3_1_asr": { "mean_score": 90.0, "mean_raw": 10.0 }, "r2t2": { "mean_score": 97.0, "mean_raw": 3.0 }, "soniox": { "mean_score": 90.0, "mean_raw": 10.0 }, "stepfun": { "mean_score": 90.0, "mean_raw": 10.0 }, "vibevoice": { "mean_score": 90.0, "mean_raw": 10.0 }, "voxtral": { "mean_score": 75.0, "mean_raw": 25.0 }, "whisper": { "mean_score": 89.0, "mean_raw": 11.0 }, "whisper_v3_turbo": { "mean_score": 89.0, "mean_raw": 11.0 } } }, "25": { "duration_s": 16, "model_scores": { "apple_speechanalyzer": { "mean_score": 91.0, "mean_raw": 9.0 }, "doubao": { "mean_score": 92.0, "mean_raw": 8.0 }, "elevenlabs": { "mean_score": 100.0, "mean_raw": 0.0 }, "firered": { "mean_score": 88.0, "mean_raw": 12.0 }, "firered_aed": { "mean_score": 91.0, "mean_raw": 9.0 }, "flash": { "mean_score": 95.0, "mean_raw": 5.0 }, "funasr_nano": { "mean_score": 92.0, "mean_raw": 8.0 }, "gemini_3_5_flash_lite_stt": { "mean_score": 87.0, "mean_raw": 13.0 }, "gemini_3_5_flash_stt": { "mean_score": 99.0, "mean_raw": 1.0 }, "gemini_3_5_transcribe": { "mean_score": 98.0, "mean_raw": 2.0 }, "gemini_3_6_flash_stt": { "mean_score": 77.0, "mean_raw": 23.0 }, "gemini_3_7_flash_stt": { "mean_score": 98.0, "mean_raw": 2.0 }, "gemini_3_8_flash_stt": { "mean_score": 98.0, "mean_raw": 2.0 }, "gemini_flash_stt": { "mean_score": 87.0, "mean_raw": 13.0 }, "gemini_pro_stt": { "mean_score": 100.0, "mean_raw": 0.0 }, "gemini_stt": { "mean_score": 89.0, "mean_raw": 11.0 }, "glm_asr_nano": { "mean_score": 92.0, "mean_raw": 8.0 }, "gpt_transcribe": { "mean_score": 96.0, "mean_raw": 4.0 }, "mai_transcribe_2": { "mean_score": 100.0, "mean_raw": 0.0 }, "mimo": { "mean_score": 92.0, "mean_raw": 8.0 }, "mimo_stt": { "mean_score": 89.0, "mean_raw": 11.0 }, "moss_audio": { "mean_score": 92.0, "mean_raw": 8.0 }, "qwen3_oss": { "mean_score": 92.0, "mean_raw": 8.0 }, "qwen_audio_3_1_asr": { "mean_score": 100.0, "mean_raw": 0.0 }, "r2t2": { "mean_score": 100.0, "mean_raw": 0.0 }, "soniox": { "mean_score": 100.0, "mean_raw": 0.0 }, "stepfun": { "mean_score": 84.0, "mean_raw": 16.0 }, "vibevoice": { "mean_score": 92.0, "mean_raw": 8.0 }, "voxtral": { "mean_score": 87.0, "mean_raw": 13.0 }, "whisper": { "mean_score": 97.0, "mean_raw": 3.0 }, "whisper_v3_turbo": { "mean_score": 84.0, "mean_raw": 16.0 } } }, "26": { "duration_s": 23, "model_scores": { "apple_speechanalyzer": { "mean_score": 91.0, "mean_raw": 9.0 }, "doubao": { "mean_score": 99.0, "mean_raw": 1.0 }, "elevenlabs": { "mean_score": 99.0, "mean_raw": 1.0 }, "firered": { "mean_score": 84.0, "mean_raw": 16.0 }, "firered_aed": { "mean_score": 99.0, "mean_raw": 1.0 }, "flash": { "mean_score": 100.0, "mean_raw": 0.0 }, "funasr_nano": { "mean_score": 100.0, "mean_raw": 0.0 }, "gemini_3_5_flash_lite_stt": { "mean_score": 99.0, "mean_raw": 1.0 }, "gemini_3_5_flash_stt": { "mean_score": 100.0, "mean_raw": 0.0 }, "gemini_3_5_transcribe": { "mean_score": 96.0, "mean_raw": 4.0 }, "gemini_3_6_flash_stt": { "mean_score": 97.0, "mean_raw": 3.0 }, "gemini_3_7_flash_stt": { "mean_score": 62.0, "mean_raw": 38.0 }, "gemini_3_8_flash_stt": { "mean_score": 98.0, "mean_raw": 2.0 }, "gemini_flash_stt": { "mean_score": 96.0, "mean_raw": 4.0 }, "gemini_pro_stt": { "mean_score": 100.0, "mean_raw": 0.0 }, "gemini_stt": { "mean_score": 99.0, "mean_raw": 1.0 }, "glm_asr_nano": { "mean_score": 100.0, "mean_raw": 0.0 }, "gpt_transcribe": { "mean_score": 97.0, "mean_raw": 3.0 }, "mai_transcribe_2": { "mean_score": 100.0, "mean_raw": 0.0 }, "mimo": { "mean_score": 100.0, "mean_raw": 0.0 }, "mimo_stt": { "mean_score": 92.0, "mean_raw": 8.0 }, "moss_audio": { "mean_score": 64.0, "mean_raw": 36.0 }, "qwen3_oss": { "mean_score": 100.0, "mean_raw": 0.0 }, "qwen_audio_3_1_asr": { "mean_score": 100.0, "mean_raw": 0.0 }, "r2t2": { "mean_score": 100.0, "mean_raw": 0.0 }, "soniox": { "mean_score": 100.0, "mean_raw": 0.0 }, "stepfun": { "mean_score": 100.0, "mean_raw": 0.0 }, "vibevoice": { "mean_score": 99.0, "mean_raw": 1.0 }, "voxtral": { "mean_score": 92.0, "mean_raw": 8.0 }, "whisper": { "mean_score": 96.0, "mean_raw": 4.0 }, "whisper_v3_turbo": { "mean_score": 98.0, "mean_raw": 2.0 } } }, "27": { "duration_s": 25, "model_scores": { "apple_speechanalyzer": { "mean_score": 93.0, "mean_raw": 7.0 }, "doubao": { "mean_score": 100.0, "mean_raw": 0.0 }, "elevenlabs": { "mean_score": 97.0, "mean_raw": 3.0 }, "firered": { "mean_score": 96.0, "mean_raw": 4.0 }, "firered_aed": { "mean_score": 100.0, "mean_raw": 0.0 }, "flash": { "mean_score": 100.0, "mean_raw": 0.0 }, "funasr_nano": { "mean_score": 100.0, "mean_raw": 0.0 }, "gemini_3_5_flash_lite_stt": { "mean_score": 93.0, "mean_raw": 7.0 }, "gemini_3_5_flash_stt": { "mean_score": 96.0, "mean_raw": 4.0 }, "gemini_3_5_transcribe": { "mean_score": 91.0, "mean_raw": 9.0 }, "gemini_3_6_flash_stt": { "mean_score": 93.0, "mean_raw": 7.0 }, "gemini_3_7_flash_stt": { "mean_score": 98.0, "mean_raw": 2.0 }, "gemini_3_8_flash_stt": { "mean_score": 98.0, "mean_raw": 2.0 }, "gemini_flash_stt": { "mean_score": 97.0, "mean_raw": 3.0 }, "gemini_pro_stt": { "mean_score": 99.0, "mean_raw": 1.0 }, "gemini_stt": { "mean_score": 90.0, "mean_raw": 10.0 }, "glm_asr_nano": { "mean_score": 100.0, "mean_raw": 0.0 }, "gpt_transcribe": { "mean_score": 100.0, "mean_raw": 0.0 }, "mai_transcribe_2": { "mean_score": 100.0, "mean_raw": 0.0 }, "mimo": { "mean_score": 100.0, "mean_raw": 0.0 }, "mimo_stt": { "mean_score": 98.0, "mean_raw": 2.0 }, "moss_audio": { "mean_score": 100.0, "mean_raw": 0.0 }, "qwen3_oss": { "mean_score": 100.0, "mean_raw": 0.0 }, "qwen_audio_3_1_asr": { "mean_score": 100.0, "mean_raw": 0.0 }, "r2t2": { "mean_score": 100.0, "mean_raw": 0.0 }, "soniox": { "mean_score": 97.0, "mean_raw": 3.0 }, "stepfun": { "mean_score": 98.0, "mean_raw": 2.0 }, "vibevoice": { "mean_score": 100.0, "mean_raw": 0.0 }, "voxtral": { "mean_score": 95.0, "mean_raw": 5.0 }, "whisper": { "mean_score": 98.0, "mean_raw": 2.0 }, "whisper_v3_turbo": { "mean_score": 99.0, "mean_raw": 1.0 } } }, "28": { "duration_s": 30, "model_scores": { "apple_speechanalyzer": { "mean_score": 90.0, "mean_raw": 10.0 }, "doubao": { "mean_score": 39.0, "mean_raw": 61.0 }, "elevenlabs": { "mean_score": 97.0, "mean_raw": 3.0 }, "firered": { "mean_score": 84.0, "mean_raw": 16.0 }, "firered_aed": { "mean_score": 74.0, "mean_raw": 26.0 }, "flash": { "mean_score": 98.0, "mean_raw": 2.0 }, "funasr_nano": { "mean_score": 94.0, "mean_raw": 6.0 }, "gemini_3_5_flash_lite_stt": { "mean_score": 82.0, "mean_raw": 18.0 }, "gemini_3_5_flash_stt": { "mean_score": 95.0, "mean_raw": 5.0 }, "gemini_3_5_transcribe": { "mean_score": 85.0, "mean_raw": 15.0 }, "gemini_3_6_flash_stt": { "mean_score": 82.0, "mean_raw": 18.0 }, "gemini_3_7_flash_stt": { "mean_score": 87.0, "mean_raw": 13.0 }, "gemini_3_8_flash_stt": { "mean_score": 88.0, "mean_raw": 12.0 }, "gemini_flash_stt": { "mean_score": 89.0, "mean_raw": 11.0 }, "gemini_pro_stt": { "mean_score": 93.0, "mean_raw": 7.0 }, "gemini_stt": { "mean_score": 78.0, "mean_raw": 22.0 }, "glm_asr_nano": { "mean_score": 89.0, "mean_raw": 11.0 }, "gpt_transcribe": { "mean_score": 93.0, "mean_raw": 7.0 }, "mai_transcribe_2": { "mean_score": 91.0, "mean_raw": 9.0 }, "mimo": { "mean_score": 97.0, "mean_raw": 3.0 }, "mimo_stt": { "mean_score": 95.0, "mean_raw": 5.0 }, "moss_audio": { "mean_score": 97.0, "mean_raw": 3.0 }, "qwen3_oss": { "mean_score": 99.0, "mean_raw": 1.0 }, "qwen_audio_3_1_asr": { "mean_score": 99.0, "mean_raw": 1.0 }, "r2t2": { "mean_score": 96.0, "mean_raw": 4.0 }, "soniox": { "mean_score": 94.0, "mean_raw": 6.0 }, "stepfun": { "mean_score": 93.0, "mean_raw": 7.0 }, "vibevoice": { "mean_score": 98.0, "mean_raw": 2.0 }, "voxtral": { "mean_score": 78.0, "mean_raw": 22.0 }, "whisper": { "mean_score": 90.0, "mean_raw": 10.0 }, "whisper_v3_turbo": { "mean_score": 63.0, "mean_raw": 37.0 } } }, "29": { "duration_s": 34, "model_scores": { "apple_speechanalyzer": { "mean_score": 71.0, "mean_raw": 29.0 }, "doubao": { "mean_score": 80.0, "mean_raw": 20.0 }, "elevenlabs": { "mean_score": 90.0, "mean_raw": 10.0 }, "firered": { "mean_score": 59.0, "mean_raw": 41.0 }, "firered_aed": { "mean_score": 67.0, "mean_raw": 33.0 }, "flash": { "mean_score": 79.0, "mean_raw": 21.0 }, "funasr_nano": { "mean_score": 90.0, "mean_raw": 10.0 }, "gemini_3_5_flash_lite_stt": { "mean_score": 84.0, "mean_raw": 16.0 }, "gemini_3_5_flash_stt": { "mean_score": 98.0, "mean_raw": 2.0 }, "gemini_3_5_transcribe": { "mean_score": 87.0, "mean_raw": 13.0 }, "gemini_3_6_flash_stt": { "mean_score": 95.0, "mean_raw": 5.0 }, "gemini_3_7_flash_stt": { "mean_score": 98.0, "mean_raw": 2.0 }, "gemini_3_8_flash_stt": { "mean_score": 99.0, "mean_raw": 1.0 }, "gemini_flash_stt": { "mean_score": 95.0, "mean_raw": 5.0 }, "gemini_pro_stt": { "mean_score": 88.0, "mean_raw": 12.0 }, "gemini_stt": { "mean_score": 84.0, "mean_raw": 16.0 }, "glm_asr_nano": { "mean_score": 69.0, "mean_raw": 31.0 }, "gpt_transcribe": { "mean_score": 97.0, "mean_raw": 3.0 }, "mai_transcribe_2": { "mean_score": 92.0, "mean_raw": 8.0 }, "mimo": { "mean_score": 99.0, "mean_raw": 1.0 }, "mimo_stt": { "mean_score": 89.0, "mean_raw": 11.0 }, "moss_audio": { "mean_score": 54.0, "mean_raw": 46.0 }, "qwen3_oss": { "mean_score": 89.0, "mean_raw": 11.0 }, "qwen_audio_3_1_asr": { "mean_score": 100.0, "mean_raw": 0.0 }, "r2t2": { "mean_score": 90.0, "mean_raw": 10.0 }, "soniox": { "mean_score": 99.0, "mean_raw": 1.0 }, "stepfun": { "mean_score": 81.0, "mean_raw": 19.0 }, "vibevoice": { "mean_score": 90.0, "mean_raw": 10.0 }, "voxtral": { "mean_score": 79.0, "mean_raw": 21.0 }, "whisper": { "mean_score": 86.0, "mean_raw": 14.0 }, "whisper_v3_turbo": { "mean_score": 52.0, "mean_raw": 48.0 } } }, "30": { "duration_s": 38, "model_scores": { "apple_speechanalyzer": { "mean_score": 90.0, "mean_raw": 10.0 }, "doubao": { "mean_score": 64.0, "mean_raw": 36.0 }, "elevenlabs": { "mean_score": 99.0, "mean_raw": 1.0 }, "firered": { "mean_score": 62.0, "mean_raw": 38.0 }, "firered_aed": { "mean_score": 55.0, "mean_raw": 45.0 }, "flash": { "mean_score": 99.0, "mean_raw": 1.0 }, "funasr_nano": { "mean_score": 62.0, "mean_raw": 38.0 }, "gemini_3_5_flash_lite_stt": { "mean_score": 87.0, "mean_raw": 13.0 }, "gemini_3_5_flash_stt": { "mean_score": 91.0, "mean_raw": 9.0 }, "gemini_3_5_transcribe": { "mean_score": 84.0, "mean_raw": 16.0 }, "gemini_3_6_flash_stt": { "mean_score": 95.0, "mean_raw": 5.0 }, "gemini_3_7_flash_stt": { "mean_score": 91.0, "mean_raw": 9.0 }, "gemini_3_8_flash_stt": { "mean_score": 92.0, "mean_raw": 8.0 }, "gemini_flash_stt": { "mean_score": 94.0, "mean_raw": 6.0 }, "gemini_pro_stt": { "mean_score": 95.0, "mean_raw": 5.0 }, "gemini_stt": { "mean_score": 90.0, "mean_raw": 10.0 }, "glm_asr_nano": { "mean_score": 65.0, "mean_raw": 35.0 }, "gpt_transcribe": { "mean_score": 96.0, "mean_raw": 4.0 }, "mai_transcribe_2": { "mean_score": 99.0, "mean_raw": 1.0 }, "mimo": { "mean_score": 100.0, "mean_raw": 0.0 }, "mimo_stt": { "mean_score": 93.0, "mean_raw": 7.0 }, "moss_audio": { "mean_score": 82.0, "mean_raw": 18.0 }, "qwen3_oss": { "mean_score": 56.0, "mean_raw": 44.0 }, "qwen_audio_3_1_asr": { "mean_score": 95.0, "mean_raw": 5.0 }, "r2t2": { "mean_score": 97.0, "mean_raw": 3.0 }, "soniox": { "mean_score": 89.0, "mean_raw": 11.0 }, "stepfun": { "mean_score": 54.0, "mean_raw": 46.0 }, "vibevoice": { "mean_score": 99.0, "mean_raw": 1.0 }, "voxtral": { "mean_score": 49.0, "mean_raw": 51.0 }, "whisper": { "mean_score": 62.0, "mean_raw": 38.0 }, "whisper_v3_turbo": { "mean_score": 50.0, "mean_raw": 50.0 } } }, "31": { "duration_s": 44, "model_scores": { "apple_speechanalyzer": { "mean_score": 39.0, "mean_raw": 61.0 }, "doubao": { "mean_score": 89.0, "mean_raw": 11.0 }, "elevenlabs": { "mean_score": 100.0, "mean_raw": 0.0 }, "firered": { "mean_score": 59.0, "mean_raw": 41.0 }, "firered_aed": { "mean_score": 97.0, "mean_raw": 3.0 }, "flash": { "mean_score": 99.0, "mean_raw": 1.0 }, "funasr_nano": { "mean_score": 55.0, "mean_raw": 45.0 }, "gemini_3_5_flash_lite_stt": { "mean_score": 87.0, "mean_raw": 13.0 }, "gemini_3_5_flash_stt": { "mean_score": 97.0, "mean_raw": 3.0 }, "gemini_3_5_transcribe": { "mean_score": 98.0, "mean_raw": 2.0 }, "gemini_3_6_flash_stt": { "mean_score": 79.0, "mean_raw": 21.0 }, "gemini_3_7_flash_stt": { "mean_score": 87.0, "mean_raw": 13.0 }, "gemini_3_8_flash_stt": { "mean_score": 96.0, "mean_raw": 4.0 }, "gemini_flash_stt": { "mean_score": 97.0, "mean_raw": 3.0 }, "gemini_pro_stt": { "mean_score": 98.0, "mean_raw": 2.0 }, "gemini_stt": { "mean_score": 95.0, "mean_raw": 5.0 }, "glm_asr_nano": { "mean_score": 63.0, "mean_raw": 37.0 }, "gpt_transcribe": { "mean_score": 95.0, "mean_raw": 5.0 }, "mai_transcribe_2": { "mean_score": 100.0, "mean_raw": 0.0 }, "mimo": { "mean_score": 97.0, "mean_raw": 3.0 }, "mimo_stt": { "mean_score": 98.0, "mean_raw": 2.0 }, "moss_audio": { "mean_score": 47.0, "mean_raw": 53.0 }, "qwen3_oss": { "mean_score": 99.0, "mean_raw": 1.0 }, "qwen_audio_3_1_asr": { "mean_score": 64.0, "mean_raw": 36.0 }, "r2t2": { "mean_score": 97.0, "mean_raw": 3.0 }, "soniox": { "mean_score": 100.0, "mean_raw": 0.0 }, "stepfun": { "mean_score": 52.0, "mean_raw": 48.0 }, "vibevoice": { "mean_score": 65.0, "mean_raw": 35.0 }, "voxtral": { "mean_score": 62.0, "mean_raw": 38.0 }, "whisper": { "mean_score": 61.0, "mean_raw": 39.0 }, "whisper_v3_turbo": { "mean_score": 93.0, "mean_raw": 7.0 } } }, "32": { "duration_s": 50, "model_scores": { "apple_speechanalyzer": { "mean_score": 98.0, "mean_raw": 2.0 }, "doubao": { "mean_score": 100.0, "mean_raw": 0.0 }, "elevenlabs": { "mean_score": 97.0, "mean_raw": 3.0 }, "firered": { "mean_score": 60.0, "mean_raw": 40.0 }, "firered_aed": { "mean_score": 95.0, "mean_raw": 5.0 }, "flash": { "mean_score": 100.0, "mean_raw": 0.0 }, "funasr_nano": { "mean_score": 99.0, "mean_raw": 1.0 }, "gemini_3_5_flash_lite_stt": { "mean_score": 56.0, "mean_raw": 44.0 }, "gemini_3_5_flash_stt": { "mean_score": 95.0, "mean_raw": 5.0 }, "gemini_3_5_transcribe": { "mean_score": 91.0, "mean_raw": 9.0 }, "gemini_3_6_flash_stt": { "mean_score": 95.0, "mean_raw": 5.0 }, "gemini_3_7_flash_stt": { "mean_score": 95.0, "mean_raw": 5.0 }, "gemini_3_8_flash_stt": { "mean_score": 80.0, "mean_raw": 20.0 }, "gemini_flash_stt": { "mean_score": 90.0, "mean_raw": 10.0 }, "gemini_pro_stt": { "mean_score": 77.0, "mean_raw": 23.0 }, "gemini_stt": { "mean_score": 88.0, "mean_raw": 12.0 }, "glm_asr_nano": { "mean_score": 99.0, "mean_raw": 1.0 }, "gpt_transcribe": { "mean_score": 82.0, "mean_raw": 18.0 }, "mai_transcribe_2": { "mean_score": 99.0, "mean_raw": 1.0 }, "mimo": { "mean_score": 98.0, "mean_raw": 2.0 }, "mimo_stt": { "mean_score": 96.0, "mean_raw": 4.0 }, "moss_audio": { "mean_score": 58.0, "mean_raw": 42.0 }, "qwen3_oss": { "mean_score": 93.0, "mean_raw": 7.0 }, "qwen_audio_3_1_asr": { "mean_score": 98.0, "mean_raw": 2.0 }, "r2t2": { "mean_score": 96.0, "mean_raw": 4.0 }, "soniox": { "mean_score": 95.0, "mean_raw": 5.0 }, "stepfun": { "mean_score": 97.0, "mean_raw": 3.0 }, "vibevoice": { "mean_score": 98.0, "mean_raw": 2.0 }, "voxtral": { "mean_score": 62.0, "mean_raw": 38.0 }, "whisper": { "mean_score": 85.0, "mean_raw": 15.0 }, "whisper_v3_turbo": { "mean_score": 95.0, "mean_raw": 5.0 } } }, "33": { "duration_s": 62, "model_scores": { "apple_speechanalyzer": { "mean_score": 34.0, "mean_raw": 66.0 }, "doubao": { "mean_score": 81.0, "mean_raw": 19.0 }, "elevenlabs": { "mean_score": 79.0, "mean_raw": 21.0 }, "firered": { "mean_score": 28.0, "mean_raw": 72.0 }, "firered_aed": { "mean_score": 61.0, "mean_raw": 39.0 }, "flash": { "mean_score": 81.0, "mean_raw": 19.0 }, "funasr_nano": { "mean_score": 82.0, "mean_raw": 18.0 }, "gemini_3_5_flash_lite_stt": { "mean_score": 61.0, "mean_raw": 39.0 }, "gemini_3_5_flash_stt": { "mean_score": 92.0, "mean_raw": 8.0 }, "gemini_3_5_transcribe": { "mean_score": 82.0, "mean_raw": 18.0 }, "gemini_3_6_flash_stt": { "mean_score": 90.0, "mean_raw": 10.0 }, "gemini_3_7_flash_stt": { "mean_score": 95.0, "mean_raw": 5.0 }, "gemini_3_8_flash_stt": { "mean_score": 95.0, "mean_raw": 5.0 }, "gemini_flash_stt": { "mean_score": 87.0, "mean_raw": 13.0 }, "gemini_pro_stt": { "mean_score": 95.0, "mean_raw": 5.0 }, "gemini_stt": { "mean_score": 79.0, "mean_raw": 21.0 }, "glm_asr_nano": { "mean_score": 76.0, "mean_raw": 24.0 }, "gpt_transcribe": { "mean_score": 89.0, "mean_raw": 11.0 }, "mai_transcribe_2": { "mean_score": 94.0, "mean_raw": 6.0 }, "mimo": { "mean_score": 83.0, "mean_raw": 17.0 }, "mimo_stt": { "mean_score": 89.0, "mean_raw": 11.0 }, "moss_audio": { "mean_score": 64.0, "mean_raw": 36.0 }, "qwen3_oss": { "mean_score": 75.0, "mean_raw": 25.0 }, "qwen_audio_3_1_asr": { "mean_score": 89.0, "mean_raw": 11.0 }, "r2t2": { "mean_score": 79.0, "mean_raw": 21.0 }, "soniox": { "mean_score": 91.0, "mean_raw": 9.0 }, "stepfun": { "mean_score": 85.0, "mean_raw": 15.0 }, "vibevoice": { "mean_score": 80.0, "mean_raw": 20.0 }, "voxtral": { "mean_score": 77.0, "mean_raw": 23.0 }, "whisper": { "mean_score": 79.0, "mean_raw": 21.0 }, "whisper_v3_turbo": { "mean_score": 79.0, "mean_raw": 21.0 } } }, "34": { "duration_s": 66, "model_scores": { "apple_speechanalyzer": { "mean_score": 24.0, "mean_raw": 76.0 }, "doubao": { "mean_score": 98.0, "mean_raw": 2.0 }, "elevenlabs": { "mean_score": 98.0, "mean_raw": 2.0 }, "firered": { "mean_score": 23.0, "mean_raw": 77.0 }, "firered_aed": { "mean_score": 77.0, "mean_raw": 23.0 }, "flash": { "mean_score": 64.0, "mean_raw": 36.0 }, "funasr_nano": { "mean_score": 58.0, "mean_raw": 42.0 }, "gemini_3_5_flash_lite_stt": { "mean_score": 45.0, "mean_raw": 55.0 }, "gemini_3_5_flash_stt": { "mean_score": 79.0, "mean_raw": 21.0 }, "gemini_3_5_transcribe": { "mean_score": 70.0, "mean_raw": 30.0 }, "gemini_3_6_flash_stt": { "mean_score": 85.0, "mean_raw": 15.0 }, "gemini_3_7_flash_stt": { "mean_score": 80.0, "mean_raw": 20.0 }, "gemini_3_8_flash_stt": { "mean_score": 84.0, "mean_raw": 16.0 }, "gemini_flash_stt": { "mean_score": 84.0, "mean_raw": 16.0 }, "gemini_pro_stt": { "mean_score": 95.0, "mean_raw": 5.0 }, "gemini_stt": { "mean_score": 89.0, "mean_raw": 11.0 }, "glm_asr_nano": { "mean_score": 62.0, "mean_raw": 38.0 }, "gpt_transcribe": { "mean_score": 89.0, "mean_raw": 11.0 }, "mai_transcribe_2": { "mean_score": 97.0, "mean_raw": 3.0 }, "mimo": { "mean_score": 98.0, "mean_raw": 2.0 }, "mimo_stt": { "mean_score": 58.0, "mean_raw": 42.0 }, "moss_audio": { "mean_score": 48.0, "mean_raw": 52.0 }, "qwen3_oss": { "mean_score": 78.0, "mean_raw": 22.0 }, "qwen_audio_3_1_asr": { "mean_score": 96.0, "mean_raw": 4.0 }, "r2t2": { "mean_score": 65.0, "mean_raw": 35.0 }, "soniox": { "mean_score": 89.0, "mean_raw": 11.0 }, "stepfun": { "mean_score": 60.0, "mean_raw": 40.0 }, "vibevoice": { "mean_score": 59.0, "mean_raw": 41.0 }, "voxtral": { "mean_score": 44.0, "mean_raw": 56.0 }, "whisper": { "mean_score": 85.0, "mean_raw": 15.0 }, "whisper_v3_turbo": { "mean_score": 34.0, "mean_raw": 66.0 } } }, "35": { "duration_s": 77, "model_scores": { "apple_speechanalyzer": { "mean_score": 39.0, "mean_raw": 61.0 }, "doubao": { "mean_score": 82.0, "mean_raw": 18.0 }, "elevenlabs": { "mean_score": 97.0, "mean_raw": 3.0 }, "firered": { "mean_score": 37.0, "mean_raw": 63.0 }, "firered_aed": { "mean_score": 66.0, "mean_raw": 34.0 }, "flash": { "mean_score": 96.0, "mean_raw": 4.0 }, "funasr_nano": { "mean_score": 74.0, "mean_raw": 26.0 }, "gemini_3_5_flash_lite_stt": { "mean_score": 79.0, "mean_raw": 21.0 }, "gemini_3_5_flash_stt": { "mean_score": 91.0, "mean_raw": 9.0 }, "gemini_3_5_transcribe": { "mean_score": 87.0, "mean_raw": 13.0 }, "gemini_3_6_flash_stt": { "mean_score": 77.0, "mean_raw": 23.0 }, "gemini_3_7_flash_stt": { "mean_score": 93.0, "mean_raw": 7.0 }, "gemini_3_8_flash_stt": { "mean_score": 87.0, "mean_raw": 13.0 }, "gemini_flash_stt": { "mean_score": 87.0, "mean_raw": 13.0 }, "gemini_pro_stt": { "mean_score": 74.0, "mean_raw": 26.0 }, "gemini_stt": { "mean_score": 71.0, "mean_raw": 29.0 }, "glm_asr_nano": { "mean_score": 89.0, "mean_raw": 11.0 }, "gpt_transcribe": { "mean_score": 88.0, "mean_raw": 12.0 }, "mai_transcribe_2": { "mean_score": 97.0, "mean_raw": 3.0 }, "mimo": { "mean_score": 95.0, "mean_raw": 5.0 }, "mimo_stt": { "mean_score": 81.0, "mean_raw": 19.0 }, "moss_audio": { "mean_score": 51.0, "mean_raw": 49.0 }, "qwen3_oss": { "mean_score": 67.0, "mean_raw": 33.0 }, "qwen_audio_3_1_asr": { "mean_score": 94.0, "mean_raw": 6.0 }, "r2t2": { "mean_score": 96.0, "mean_raw": 4.0 }, "soniox": { "mean_score": 90.0, "mean_raw": 10.0 }, "stepfun": { "mean_score": 88.0, "mean_raw": 12.0 }, "vibevoice": { "mean_score": 85.0, "mean_raw": 15.0 }, "voxtral": { "mean_score": 79.0, "mean_raw": 21.0 }, "whisper": { "mean_score": 99.0, "mean_raw": 1.0 }, "whisper_v3_turbo": { "mean_score": 61.0, "mean_raw": 39.0 } } }, "36": { "duration_s": 94, "model_scores": { "apple_speechanalyzer": { "mean_score": 60.0, "mean_raw": 40.0 }, "doubao": { "mean_score": 97.0, "mean_raw": 3.0 }, "elevenlabs": { "mean_score": 96.0, "mean_raw": 4.0 }, "firered": { "mean_score": 63.0, "mean_raw": 37.0 }, "firered_aed": { "mean_score": 97.0, "mean_raw": 3.0 }, "flash": { "mean_score": 98.0, "mean_raw": 2.0 }, "funasr_nano": { "mean_score": 89.0, "mean_raw": 11.0 }, "gemini_3_5_flash_lite_stt": { "mean_score": 55.0, "mean_raw": 45.0 }, "gemini_3_5_flash_stt": { "mean_score": 84.0, "mean_raw": 16.0 }, "gemini_3_5_transcribe": { "mean_score": 87.0, "mean_raw": 13.0 }, "gemini_3_6_flash_stt": { "mean_score": 82.0, "mean_raw": 18.0 }, "gemini_3_7_flash_stt": { "mean_score": 98.0, "mean_raw": 2.0 }, "gemini_3_8_flash_stt": { "mean_score": 89.0, "mean_raw": 11.0 }, "gemini_flash_stt": { "mean_score": 80.0, "mean_raw": 20.0 }, "gemini_pro_stt": { "mean_score": 53.0, "mean_raw": 47.0 }, "gemini_stt": { "mean_score": 95.0, "mean_raw": 5.0 }, "glm_asr_nano": { "mean_score": 96.0, "mean_raw": 4.0 }, "gpt_transcribe": { "mean_score": 95.0, "mean_raw": 5.0 }, "mai_transcribe_2": { "mean_score": 100.0, "mean_raw": 0.0 }, "mimo": { "mean_score": 89.0, "mean_raw": 11.0 }, "mimo_stt": { "mean_score": 96.0, "mean_raw": 4.0 }, "moss_audio": { "mean_score": 57.0, "mean_raw": 43.0 }, "qwen3_oss": { "mean_score": 100.0, "mean_raw": 0.0 }, "qwen_audio_3_1_asr": { "mean_score": 98.0, "mean_raw": 2.0 }, "r2t2": { "mean_score": 99.0, "mean_raw": 1.0 }, "soniox": { "mean_score": 95.0, "mean_raw": 5.0 }, "stepfun": { "mean_score": 98.0, "mean_raw": 2.0 }, "vibevoice": { "mean_score": 96.0, "mean_raw": 4.0 }, "voxtral": { "mean_score": 61.0, "mean_raw": 39.0 }, "whisper": { "mean_score": 95.0, "mean_raw": 5.0 }, "whisper_v3_turbo": { "mean_score": 52.0, "mean_raw": 48.0 } } }, "37": { "duration_s": 106, "model_scores": { "apple_speechanalyzer": { "mean_score": 67.0, "mean_raw": 33.0 }, "doubao": { "mean_score": 99.0, "mean_raw": 1.0 }, "elevenlabs": { "mean_score": 100.0, "mean_raw": 0.0 }, "firered": { "mean_score": 76.0, "mean_raw": 24.0 }, "firered_aed": { "mean_score": 98.0, "mean_raw": 2.0 }, "flash": { "mean_score": 99.0, "mean_raw": 1.0 }, "funasr_nano": { "mean_score": 72.0, "mean_raw": 28.0 }, "gemini_3_5_flash_lite_stt": { "mean_score": 78.0, "mean_raw": 22.0 }, "gemini_3_5_flash_stt": { "mean_score": 93.0, "mean_raw": 7.0 }, "gemini_3_5_transcribe": { "mean_score": 87.0, "mean_raw": 13.0 }, "gemini_3_6_flash_stt": { "mean_score": 78.0, "mean_raw": 22.0 }, "gemini_3_7_flash_stt": { "mean_score": 90.0, "mean_raw": 10.0 }, "gemini_3_8_flash_stt": { "mean_score": 90.0, "mean_raw": 10.0 }, "gemini_flash_stt": { "mean_score": 95.0, "mean_raw": 5.0 }, "gemini_pro_stt": { "mean_score": 95.0, "mean_raw": 5.0 }, "gemini_stt": { "mean_score": 79.0, "mean_raw": 21.0 }, "glm_asr_nano": { "mean_score": 90.0, "mean_raw": 10.0 }, "gpt_transcribe": { "mean_score": 95.0, "mean_raw": 5.0 }, "mai_transcribe_2": { "mean_score": 100.0, "mean_raw": 0.0 }, "mimo": { "mean_score": 98.0, "mean_raw": 2.0 }, "mimo_stt": { "mean_score": 95.0, "mean_raw": 5.0 }, "qwen3_oss": { "mean_score": 98.0, "mean_raw": 2.0 }, "qwen_audio_3_1_asr": { "mean_score": 96.0, "mean_raw": 4.0 }, "r2t2": { "mean_score": 98.0, "mean_raw": 2.0 }, "soniox": { "mean_score": 98.0, "mean_raw": 2.0 }, "stepfun": { "mean_score": 97.0, "mean_raw": 3.0 }, "vibevoice": { "mean_score": 100.0, "mean_raw": 0.0 }, "voxtral": { "mean_score": 95.0, "mean_raw": 5.0 }, "whisper": { "mean_score": 93.0, "mean_raw": 7.0 }, "whisper_v3_turbo": { "mean_score": 90.0, "mean_raw": 10.0 } } }, "38": { "duration_s": 126, "model_scores": { "apple_speechanalyzer": { "mean_score": 3.0, "mean_raw": 97.0 }, "doubao": { "mean_score": 86.0, "mean_raw": 14.0 }, "elevenlabs": { "mean_score": 86.0, "mean_raw": 14.0 }, "firered": { "mean_score": 10.0, "mean_raw": 90.0 }, "firered_aed": { "mean_score": 79.0, "mean_raw": 21.0 }, "flash": { "mean_score": 87.0, "mean_raw": 13.0 }, "funasr_nano": { "mean_score": 64.0, "mean_raw": 36.0 }, "gemini_3_5_flash_lite_stt": { "mean_score": 56.0, "mean_raw": 44.0 }, "gemini_3_5_flash_stt": { "mean_score": 93.0, "mean_raw": 7.0 }, "gemini_3_5_transcribe": { "mean_score": 75.0, "mean_raw": 25.0 }, "gemini_3_6_flash_stt": { "mean_score": 79.0, "mean_raw": 21.0 }, "gemini_3_7_flash_stt": { "mean_score": 88.0, "mean_raw": 12.0 }, "gemini_3_8_flash_stt": { "mean_score": 93.0, "mean_raw": 7.0 }, "gemini_flash_stt": { "mean_score": 84.0, "mean_raw": 16.0 }, "gemini_pro_stt": { "mean_score": 83.0, "mean_raw": 17.0 }, "gemini_stt": { "mean_score": 70.0, "mean_raw": 30.0 }, "glm_asr_nano": { "mean_score": 70.0, "mean_raw": 30.0 }, "gpt_transcribe": { "mean_score": 85.0, "mean_raw": 15.0 }, "mai_transcribe_2": { "mean_score": 95.0, "mean_raw": 5.0 }, "mimo": { "mean_score": 78.0, "mean_raw": 22.0 }, "mimo_stt": { "mean_score": 93.0, "mean_raw": 7.0 }, "moss_audio": { "mean_score": 65.0, "mean_raw": 35.0 }, "qwen3_oss": { "mean_score": 76.0, "mean_raw": 24.0 }, "qwen_audio_3_1_asr": { "mean_score": 83.0, "mean_raw": 17.0 }, "r2t2": { "mean_score": 76.0, "mean_raw": 24.0 }, "soniox": { "mean_score": 82.0, "mean_raw": 18.0 }, "stepfun": { "mean_score": 93.0, "mean_raw": 7.0 }, "vibevoice": { "mean_score": 85.0, "mean_raw": 15.0 }, "voxtral": { "mean_score": 26.0, "mean_raw": 74.0 }, "whisper": { "mean_score": 77.0, "mean_raw": 23.0 }, "whisper_v3_turbo": { "mean_score": 80.0, "mean_raw": 20.0 } } }, "39": { "duration_s": 158, "model_scores": { "apple_speechanalyzer": { "mean_score": 44.0, "mean_raw": 56.0 }, "doubao": { "mean_score": 99.0, "mean_raw": 1.0 }, "elevenlabs": { "mean_score": 85.0, "mean_raw": 15.0 }, "firered": { "mean_score": 0.0, "mean_raw": 101.0 }, "firered_aed": { "mean_score": 51.0, "mean_raw": 49.0 }, "flash": { "mean_score": 74.0, "mean_raw": 26.0 }, "funasr_nano": { "mean_score": 0.0, "mean_raw": 100.0 }, "gemini_3_5_flash_lite_stt": { "mean_score": 35.0, "mean_raw": 65.0 }, "gemini_3_5_flash_stt": { "mean_score": 71.0, "mean_raw": 29.0 }, "gemini_3_5_transcribe": { "mean_score": 70.0, "mean_raw": 30.0 }, "gemini_3_6_flash_stt": { "mean_score": 76.0, "mean_raw": 24.0 }, "gemini_3_7_flash_stt": { "mean_score": 85.0, "mean_raw": 15.0 }, "gemini_3_8_flash_stt": { "mean_score": 89.0, "mean_raw": 11.0 }, "gemini_flash_stt": { "mean_score": 71.0, "mean_raw": 29.0 }, "gemini_pro_stt": { "mean_score": 75.0, "mean_raw": 25.0 }, "gemini_stt": { "mean_score": 46.0, "mean_raw": 54.0 }, "glm_asr_nano": { "mean_score": 60.0, "mean_raw": 40.0 }, "gpt_transcribe": { "mean_score": 61.0, "mean_raw": 39.0 }, "mai_transcribe_2": { "mean_score": 86.0, "mean_raw": 14.0 }, "mimo": { "mean_score": 95.0, "mean_raw": 5.0 }, "mimo_stt": { "mean_score": 93.0, "mean_raw": 7.0 }, "moss_audio": { "mean_score": 72.0, "mean_raw": 28.0 }, "qwen3_oss": { "mean_score": 71.0, "mean_raw": 29.0 }, "qwen_audio_3_1_asr": { "mean_score": 94.0, "mean_raw": 6.0 }, "r2t2": { "mean_score": 79.0, "mean_raw": 21.0 }, "soniox": { "mean_score": 79.0, "mean_raw": 21.0 }, "stepfun": { "mean_score": 59.0, "mean_raw": 41.0 }, "vibevoice": { "mean_score": 89.0, "mean_raw": 11.0 }, "voxtral": { "mean_score": 57.0, "mean_raw": 43.0 }, "whisper": { "mean_score": 81.0, "mean_raw": 19.0 }, "whisper_v3_turbo": { "mean_score": 62.0, "mean_raw": 38.0 } } }, "40": { "duration_s": 279, "model_scores": { "apple_speechanalyzer": { "mean_score": 12.0, "mean_raw": 88.0 }, "doubao": { "mean_score": 90.0, "mean_raw": 10.0 }, "elevenlabs": { "mean_score": 70.0, "mean_raw": 30.0 }, "firered_aed": { "mean_score": 65.0, "mean_raw": 35.0 }, "flash": { "mean_score": 74.0, "mean_raw": 26.0 }, "funasr_nano": { "mean_score": 0.0, "mean_raw": 100.0 }, "gemini_3_5_flash_lite_stt": { "mean_score": 0.0, "mean_raw": 112.0 }, "gemini_3_5_flash_stt": { "mean_score": 36.0, "mean_raw": 64.0 }, "gemini_3_5_transcribe": { "mean_score": 17.0, "mean_raw": 83.0 }, "gemini_3_6_flash_stt": { "mean_score": 36.0, "mean_raw": 64.0 }, "gemini_3_7_flash_stt": { "mean_score": 32.0, "mean_raw": 68.0 }, "gemini_3_8_flash_stt": { "mean_score": 49.0, "mean_raw": 51.0 }, "gemini_flash_stt": { "mean_score": 37.0, "mean_raw": 63.0 }, "gemini_pro_stt": { "mean_score": 54.0, "mean_raw": 46.0 }, "gemini_stt": { "mean_score": 14.0, "mean_raw": 86.0 }, "glm_asr_nano": { "mean_score": 87.0, "mean_raw": 13.0 }, "gpt_transcribe": { "mean_score": 87.0, "mean_raw": 13.0 }, "mai_transcribe_2": { "mean_score": 80.0, "mean_raw": 20.0 }, "mimo": { "mean_score": 77.0, "mean_raw": 23.0 }, "mimo_stt": { "mean_score": 86.0, "mean_raw": 14.0 }, "moss_audio": { "mean_score": 36.0, "mean_raw": 64.0 }, "qwen3_oss": { "mean_score": 13.0, "mean_raw": 87.0 }, "qwen_audio_3_1_asr": { "mean_score": 91.0, "mean_raw": 9.0 }, "r2t2": { "mean_score": 88.0, "mean_raw": 12.0 }, "soniox": { "mean_score": 67.0, "mean_raw": 33.0 }, "stepfun": { "mean_score": 66.0, "mean_raw": 34.0 }, "vibevoice": { "mean_score": 79.0, "mean_raw": 21.0 }, "voxtral": { "mean_score": 60.0, "mean_raw": 40.0 }, "whisper": { "mean_score": 62.0, "mean_raw": 38.0 }, "whisper_v3_turbo": { "mean_score": 28.0, "mean_raw": 72.0 } } }, "41": { "duration_s": 34, "model_scores": { "apple_speechanalyzer": { "mean_score": 69.0, "mean_raw": 31.0 }, "doubao": { "mean_score": 100.0, "mean_raw": 0.0 }, "elevenlabs": { "mean_score": 87.0, "mean_raw": 13.0 }, "firered": { "mean_score": 71.0, "mean_raw": 29.0 }, "firered_aed": { "mean_score": 89.0, "mean_raw": 11.0 }, "flash": { "mean_score": 100.0, "mean_raw": 0.0 }, "funasr_nano": { "mean_score": 97.0, "mean_raw": 3.0 }, "gemini_3_5_flash_lite_stt": { "mean_score": 78.0, "mean_raw": 22.0 }, "gemini_3_5_flash_stt": { "mean_score": 92.0, "mean_raw": 8.0 }, "gemini_3_5_transcribe": { "mean_score": 93.0, "mean_raw": 7.0 }, "gemini_3_6_flash_stt": { "mean_score": 84.0, "mean_raw": 16.0 }, "gemini_3_7_flash_stt": { "mean_score": 89.0, "mean_raw": 11.0 }, "gemini_3_8_flash_stt": { "mean_score": 96.0, "mean_raw": 4.0 }, "gemini_flash_stt": { "mean_score": 97.0, "mean_raw": 3.0 }, "gemini_pro_stt": { "mean_score": 92.0, "mean_raw": 8.0 }, "gemini_stt": { "mean_score": 93.0, "mean_raw": 7.0 }, "glm_asr_nano": { "mean_score": 100.0, "mean_raw": 0.0 }, "gpt_transcribe": { "mean_score": 92.0, "mean_raw": 8.0 }, "mai_transcribe_2": { "mean_score": 100.0, "mean_raw": 0.0 }, "mimo": { "mean_score": 100.0, "mean_raw": 0.0 }, "mimo_stt": { "mean_score": 97.0, "mean_raw": 3.0 }, "moss_audio": { "mean_score": 98.0, "mean_raw": 2.0 }, "qwen3_oss": { "mean_score": 100.0, "mean_raw": 0.0 }, "qwen_audio_3_1_asr": { "mean_score": 98.0, "mean_raw": 2.0 }, "r2t2": { "mean_score": 100.0, "mean_raw": 0.0 }, "soniox": { "mean_score": 67.0, "mean_raw": 33.0 }, "stepfun": { "mean_score": 100.0, "mean_raw": 0.0 }, "vibevoice": { "mean_score": 100.0, "mean_raw": 0.0 }, "voxtral": { "mean_score": 55.0, "mean_raw": 45.0 }, "whisper": { "mean_score": 57.0, "mean_raw": 43.0 }, "whisper_v3_turbo": { "mean_score": 55.0, "mean_raw": 45.0 } } }, "42": { "duration_s": 43, "model_scores": { "apple_speechanalyzer": { "mean_score": 78.0, "mean_raw": 22.0 }, "doubao": { "mean_score": 80.0, "mean_raw": 20.0 }, "elevenlabs": { "mean_score": 87.0, "mean_raw": 13.0 }, "firered": { "mean_score": 52.0, "mean_raw": 48.0 }, "firered_aed": { "mean_score": 92.0, "mean_raw": 8.0 }, "flash": { "mean_score": 100.0, "mean_raw": 0.0 }, "funasr_nano": { "mean_score": 99.0, "mean_raw": 1.0 }, "gemini_3_5_flash_lite_stt": { "mean_score": 61.0, "mean_raw": 39.0 }, "gemini_3_5_flash_stt": { "mean_score": 86.0, "mean_raw": 14.0 }, "gemini_3_5_transcribe": { "mean_score": 73.0, "mean_raw": 27.0 }, "gemini_3_6_flash_stt": { "mean_score": 72.0, "mean_raw": 28.0 }, "gemini_3_7_flash_stt": { "mean_score": 77.0, "mean_raw": 23.0 }, "gemini_3_8_flash_stt": { "mean_score": 97.0, "mean_raw": 3.0 }, "gemini_flash_stt": { "mean_score": 95.0, "mean_raw": 5.0 }, "gemini_pro_stt": { "mean_score": 99.0, "mean_raw": 1.0 }, "gemini_stt": { "mean_score": 72.0, "mean_raw": 28.0 }, "glm_asr_nano": { "mean_score": 90.0, "mean_raw": 10.0 }, "gpt_transcribe": { "mean_score": 95.0, "mean_raw": 5.0 }, "mai_transcribe_2": { "mean_score": 100.0, "mean_raw": 0.0 }, "mimo": { "mean_score": 94.0, "mean_raw": 6.0 }, "mimo_stt": { "mean_score": 98.0, "mean_raw": 2.0 }, "moss_audio": { "mean_score": 51.0, "mean_raw": 49.0 }, "qwen3_oss": { "mean_score": 79.0, "mean_raw": 21.0 }, "qwen_audio_3_1_asr": { "mean_score": 100.0, "mean_raw": 0.0 }, "r2t2": { "mean_score": 80.0, "mean_raw": 20.0 }, "soniox": { "mean_score": 97.0, "mean_raw": 3.0 }, "stepfun": { "mean_score": 98.0, "mean_raw": 2.0 }, "vibevoice": { "mean_score": 92.0, "mean_raw": 8.0 }, "voxtral": { "mean_score": 61.0, "mean_raw": 39.0 }, "whisper": { "mean_score": 55.0, "mean_raw": 45.0 }, "whisper_v3_turbo": { "mean_score": 66.0, "mean_raw": 34.0 } } }, "43": { "duration_s": 29, "model_scores": { "apple_speechanalyzer": { "mean_score": 67.0, "mean_raw": 33.0 }, "doubao": { "mean_score": 99.0, "mean_raw": 1.0 }, "elevenlabs": { "mean_score": 87.0, "mean_raw": 13.0 }, "firered": { "mean_score": 68.0, "mean_raw": 32.0 }, "firered_aed": { "mean_score": 82.0, "mean_raw": 18.0 }, "flash": { "mean_score": 90.0, "mean_raw": 10.0 }, "funasr_nano": { "mean_score": 89.0, "mean_raw": 11.0 }, "gemini_3_5_flash_lite_stt": { "mean_score": 61.0, "mean_raw": 39.0 }, "gemini_3_5_flash_stt": { "mean_score": 78.0, "mean_raw": 22.0 }, "gemini_3_5_transcribe": { "mean_score": 76.0, "mean_raw": 24.0 }, "gemini_3_6_flash_stt": { "mean_score": 87.0, "mean_raw": 13.0 }, "gemini_3_7_flash_stt": { "mean_score": 87.0, "mean_raw": 13.0 }, "gemini_3_8_flash_stt": { "mean_score": 90.0, "mean_raw": 10.0 }, "gemini_flash_stt": { "mean_score": 87.0, "mean_raw": 13.0 }, "gemini_pro_stt": { "mean_score": 97.0, "mean_raw": 3.0 }, "gemini_stt": { "mean_score": 72.0, "mean_raw": 28.0 }, "glm_asr_nano": { "mean_score": 89.0, "mean_raw": 11.0 }, "gpt_transcribe": { "mean_score": 89.0, "mean_raw": 11.0 }, "mai_transcribe_2": { "mean_score": 100.0, "mean_raw": 0.0 }, "mimo": { "mean_score": 91.0, "mean_raw": 9.0 }, "mimo_stt": { "mean_score": 87.0, "mean_raw": 13.0 }, "moss_audio": { "mean_score": 80.0, "mean_raw": 20.0 }, "qwen3_oss": { "mean_score": 92.0, "mean_raw": 8.0 }, "qwen_audio_3_1_asr": { "mean_score": 91.0, "mean_raw": 9.0 }, "r2t2": { "mean_score": 87.0, "mean_raw": 13.0 }, "soniox": { "mean_score": 79.0, "mean_raw": 21.0 }, "stepfun": { "mean_score": 88.0, "mean_raw": 12.0 }, "vibevoice": { "mean_score": 91.0, "mean_raw": 9.0 }, "voxtral": { "mean_score": 72.0, "mean_raw": 28.0 }, "whisper": { "mean_score": 83.0, "mean_raw": 17.0 }, "whisper_v3_turbo": { "mean_score": 71.0, "mean_raw": 29.0 } } }, "44": { "duration_s": 71, "model_scores": { "apple_speechanalyzer": { "mean_score": 50.0, "mean_raw": 50.0 }, "doubao": { "mean_score": 92.0, "mean_raw": 8.0 }, "elevenlabs": { "mean_score": 76.0, "mean_raw": 24.0 }, "firered": { "mean_score": 28.0, "mean_raw": 72.0 }, "firered_aed": { "mean_score": 85.0, "mean_raw": 15.0 }, "flash": { "mean_score": 90.0, "mean_raw": 10.0 }, "funasr_nano": { "mean_score": 76.0, "mean_raw": 24.0 }, "gemini_3_5_flash_lite_stt": { "mean_score": 54.0, "mean_raw": 46.0 }, "gemini_3_5_flash_stt": { "mean_score": 89.0, "mean_raw": 11.0 }, "gemini_3_5_transcribe": { "mean_score": 90.0, "mean_raw": 10.0 }, "gemini_3_6_flash_stt": { "mean_score": 79.0, "mean_raw": 21.0 }, "gemini_3_7_flash_stt": { "mean_score": 83.0, "mean_raw": 17.0 }, "gemini_3_8_flash_stt": { "mean_score": 91.0, "mean_raw": 9.0 }, "gemini_flash_stt": { "mean_score": 80.0, "mean_raw": 20.0 }, "gemini_pro_stt": { "mean_score": 100.0, "mean_raw": 0.0 }, "gemini_stt": { "mean_score": 62.0, "mean_raw": 38.0 }, "glm_asr_nano": { "mean_score": 84.0, "mean_raw": 16.0 }, "gpt_transcribe": { "mean_score": 89.0, "mean_raw": 11.0 }, "mai_transcribe_2": { "mean_score": 85.0, "mean_raw": 15.0 }, "mimo": { "mean_score": 100.0, "mean_raw": 0.0 }, "mimo_stt": { "mean_score": 83.0, "mean_raw": 17.0 }, "moss_audio": { "mean_score": 68.0, "mean_raw": 32.0 }, "qwen3_oss": { "mean_score": 89.0, "mean_raw": 11.0 }, "qwen_audio_3_1_asr": { "mean_score": 89.0, "mean_raw": 11.0 }, "r2t2": { "mean_score": 90.0, "mean_raw": 10.0 }, "soniox": { "mean_score": 75.0, "mean_raw": 25.0 }, "stepfun": { "mean_score": 80.0, "mean_raw": 20.0 }, "vibevoice": { "mean_score": 92.0, "mean_raw": 8.0 }, "voxtral": { "mean_score": 65.0, "mean_raw": 35.0 }, "whisper": { "mean_score": 71.0, "mean_raw": 29.0 }, "whisper_v3_turbo": { "mean_score": 41.0, "mean_raw": 59.0 } } }, "45": { "duration_s": 46, "model_scores": { "apple_speechanalyzer": { "mean_score": 27.0, "mean_raw": 73.0 }, "doubao": { "mean_score": 98.0, "mean_raw": 2.0 }, "elevenlabs": { "mean_score": 67.0, "mean_raw": 33.0 }, "firered": { "mean_score": 73.0, "mean_raw": 27.0 }, "firered_aed": { "mean_score": 96.0, "mean_raw": 4.0 }, "flash": { "mean_score": 98.0, "mean_raw": 2.0 }, "funasr_nano": { "mean_score": 56.0, "mean_raw": 44.0 }, "gemini_3_5_flash_lite_stt": { "mean_score": 0.0, "mean_raw": 100.0 }, "gemini_3_5_flash_stt": { "mean_score": 88.0, "mean_raw": 12.0 }, "gemini_3_5_transcribe": { "mean_score": 70.0, "mean_raw": 30.0 }, "gemini_3_6_flash_stt": { "mean_score": 65.0, "mean_raw": 35.0 }, "gemini_3_8_flash_stt": { "mean_score": 88.0, "mean_raw": 12.0 }, "gemini_flash_stt": { "mean_score": 65.0, "mean_raw": 35.0 }, "gemini_pro_stt": { "mean_score": 93.0, "mean_raw": 7.0 }, "gemini_stt": { "mean_score": 29.0, "mean_raw": 71.0 }, "glm_asr_nano": { "mean_score": 90.0, "mean_raw": 10.0 }, "gpt_transcribe": { "mean_score": 44.0, "mean_raw": 56.0 }, "mai_transcribe_2": { "mean_score": 93.0, "mean_raw": 7.0 }, "mimo": { "mean_score": 93.0, "mean_raw": 7.0 }, "mimo_stt": { "mean_score": 51.0, "mean_raw": 49.0 }, "moss_audio": { "mean_score": 25.0, "mean_raw": 75.0 }, "qwen3_oss": { "mean_score": 82.0, "mean_raw": 18.0 }, "qwen_audio_3_1_asr": { "mean_score": 95.0, "mean_raw": 5.0 }, "r2t2": { "mean_score": 91.0, "mean_raw": 9.0 }, "soniox": { "mean_score": 68.0, "mean_raw": 32.0 }, "stepfun": { "mean_score": 94.0, "mean_raw": 6.0 }, "vibevoice": { "mean_score": 96.0, "mean_raw": 4.0 }, "voxtral": { "mean_score": 36.0, "mean_raw": 64.0 }, "whisper": { "mean_score": 51.0, "mean_raw": 49.0 }, "whisper_v3_turbo": { "mean_score": 43.0, "mean_raw": 57.0 } } }, "46": { "duration_s": 62, "model_scores": { "apple_speechanalyzer": { "mean_score": 85.0, "mean_raw": 15.0 }, "doubao": { "mean_score": 95.0, "mean_raw": 5.0 }, "elevenlabs": { "mean_score": 87.0, "mean_raw": 13.0 }, "firered": { "mean_score": 88.0, "mean_raw": 12.0 }, "firered_aed": { "mean_score": 95.0, "mean_raw": 5.0 }, "flash": { "mean_score": 97.0, "mean_raw": 3.0 }, "funasr_nano": { "mean_score": 100.0, "mean_raw": 0.0 }, "gemini_3_5_flash_lite_stt": { "mean_score": 79.0, "mean_raw": 21.0 }, "gemini_3_5_flash_stt": { "mean_score": 88.0, "mean_raw": 12.0 }, "gemini_3_5_transcribe": { "mean_score": 95.0, "mean_raw": 5.0 }, "gemini_3_6_flash_stt": { "mean_score": 91.0, "mean_raw": 9.0 }, "gemini_3_7_flash_stt": { "mean_score": 93.0, "mean_raw": 7.0 }, "gemini_3_8_flash_stt": { "mean_score": 95.0, "mean_raw": 5.0 }, "gemini_flash_stt": { "mean_score": 83.0, "mean_raw": 17.0 }, "gemini_pro_stt": { "mean_score": 87.0, "mean_raw": 13.0 }, "gemini_stt": { "mean_score": 87.0, "mean_raw": 13.0 }, "glm_asr_nano": { "mean_score": 82.0, "mean_raw": 18.0 }, "gpt_transcribe": { "mean_score": 84.0, "mean_raw": 16.0 }, "mai_transcribe_2": { "mean_score": 90.0, "mean_raw": 10.0 }, "mimo": { "mean_score": 97.0, "mean_raw": 3.0 }, "mimo_stt": { "mean_score": 96.0, "mean_raw": 4.0 }, "moss_audio": { "mean_score": 49.0, "mean_raw": 51.0 }, "qwen3_oss": { "mean_score": 88.0, "mean_raw": 12.0 }, "qwen_audio_3_1_asr": { "mean_score": 98.0, "mean_raw": 2.0 }, "r2t2": { "mean_score": 97.0, "mean_raw": 3.0 }, "soniox": { "mean_score": 96.0, "mean_raw": 4.0 }, "stepfun": { "mean_score": 93.0, "mean_raw": 7.0 }, "vibevoice": { "mean_score": 98.0, "mean_raw": 2.0 }, "voxtral": { "mean_score": 85.0, "mean_raw": 15.0 }, "whisper": { "mean_score": 91.0, "mean_raw": 9.0 }, "whisper_v3_turbo": { "mean_score": 74.0, "mean_raw": 26.0 } } }, "47": { "duration_s": 67, "model_scores": { "apple_speechanalyzer": { "mean_score": 73.0, "mean_raw": 27.0 }, "doubao": { "mean_score": 90.0, "mean_raw": 10.0 }, "elevenlabs": { "mean_score": 87.0, "mean_raw": 13.0 }, "firered": { "mean_score": 70.0, "mean_raw": 30.0 }, "firered_aed": { "mean_score": 79.0, "mean_raw": 21.0 }, "flash": { "mean_score": 89.0, "mean_raw": 11.0 }, "funasr_nano": { "mean_score": 81.0, "mean_raw": 19.0 }, "gemini_3_5_flash_lite_stt": { "mean_score": 84.0, "mean_raw": 16.0 }, "gemini_3_5_flash_stt": { "mean_score": 96.0, "mean_raw": 4.0 }, "gemini_3_5_transcribe": { "mean_score": 84.0, "mean_raw": 16.0 }, "gemini_3_6_flash_stt": { "mean_score": 81.0, "mean_raw": 19.0 }, "gemini_3_7_flash_stt": { "mean_score": 88.0, "mean_raw": 12.0 }, "gemini_3_8_flash_stt": { "mean_score": 96.0, "mean_raw": 4.0 }, "gemini_flash_stt": { "mean_score": 93.0, "mean_raw": 7.0 }, "gemini_pro_stt": { "mean_score": 78.0, "mean_raw": 22.0 }, "gemini_stt": { "mean_score": 85.0, "mean_raw": 15.0 }, "glm_asr_nano": { "mean_score": 92.0, "mean_raw": 8.0 }, "gpt_transcribe": { "mean_score": 98.0, "mean_raw": 2.0 }, "mai_transcribe_2": { "mean_score": 90.0, "mean_raw": 10.0 }, "mimo": { "mean_score": 89.0, "mean_raw": 11.0 }, "mimo_stt": { "mean_score": 99.0, "mean_raw": 1.0 }, "moss_audio": { "mean_score": 6.0, "mean_raw": 94.0 }, "qwen3_oss": { "mean_score": 88.0, "mean_raw": 12.0 }, "qwen_audio_3_1_asr": { "mean_score": 99.0, "mean_raw": 1.0 }, "r2t2": { "mean_score": 88.0, "mean_raw": 12.0 }, "soniox": { "mean_score": 89.0, "mean_raw": 11.0 }, "stepfun": { "mean_score": 84.0, "mean_raw": 16.0 }, "vibevoice": { "mean_score": 86.0, "mean_raw": 14.0 }, "voxtral": { "mean_score": 84.0, "mean_raw": 16.0 }, "whisper": { "mean_score": 86.0, "mean_raw": 14.0 }, "whisper_v3_turbo": { "mean_score": 87.0, "mean_raw": 13.0 } } }, "48": { "duration_s": 72, "model_scores": { "apple_speechanalyzer": { "mean_score": 61.0, "mean_raw": 39.0 }, "doubao": { "mean_score": 79.0, "mean_raw": 21.0 }, "elevenlabs": { "mean_score": 77.0, "mean_raw": 23.0 }, "firered": { "mean_score": 47.0, "mean_raw": 53.0 }, "firered_aed": { "mean_score": 89.0, "mean_raw": 11.0 }, "flash": { "mean_score": 100.0, "mean_raw": 0.0 }, "funasr_nano": { "mean_score": 78.0, "mean_raw": 22.0 }, "gemini_3_5_flash_lite_stt": { "mean_score": 41.0, "mean_raw": 59.0 }, "gemini_3_5_flash_stt": { "mean_score": 78.0, "mean_raw": 22.0 }, "gemini_3_5_transcribe": { "mean_score": 85.0, "mean_raw": 15.0 }, "gemini_3_6_flash_stt": { "mean_score": 70.0, "mean_raw": 30.0 }, "gemini_3_7_flash_stt": { "mean_score": 92.0, "mean_raw": 8.0 }, "gemini_3_8_flash_stt": { "mean_score": 98.0, "mean_raw": 2.0 }, "gemini_flash_stt": { "mean_score": 77.0, "mean_raw": 23.0 }, "gemini_pro_stt": { "mean_score": 89.0, "mean_raw": 11.0 }, "gemini_stt": { "mean_score": 87.0, "mean_raw": 13.0 }, "glm_asr_nano": { "mean_score": 0.0, "mean_raw": 100.0 }, "gpt_transcribe": { "mean_score": 87.0, "mean_raw": 13.0 }, "mai_transcribe_2": { "mean_score": 100.0, "mean_raw": 0.0 }, "mimo": { "mean_score": 81.0, "mean_raw": 19.0 }, "mimo_stt": { "mean_score": 79.0, "mean_raw": 21.0 }, "moss_audio": { "mean_score": 82.0, "mean_raw": 18.0 }, "qwen3_oss": { "mean_score": 90.0, "mean_raw": 10.0 }, "qwen_audio_3_1_asr": { "mean_score": 88.0, "mean_raw": 12.0 }, "r2t2": { "mean_score": 91.0, "mean_raw": 9.0 }, "soniox": { "mean_score": 88.0, "mean_raw": 12.0 }, "stepfun": { "mean_score": 90.0, "mean_raw": 10.0 }, "vibevoice": { "mean_score": 90.0, "mean_raw": 10.0 }, "voxtral": { "mean_score": 79.0, "mean_raw": 21.0 }, "whisper": { "mean_score": 85.0, "mean_raw": 15.0 }, "whisper_v3_turbo": { "mean_score": 82.0, "mean_raw": 18.0 } } }, "49": { "duration_s": 87, "model_scores": { "apple_speechanalyzer": { "mean_score": 63.0, "mean_raw": 37.0 }, "doubao": { "mean_score": 89.0, "mean_raw": 11.0 }, "elevenlabs": { "mean_score": 83.0, "mean_raw": 17.0 }, "firered": { "mean_score": 39.0, "mean_raw": 61.0 }, "firered_aed": { "mean_score": 74.0, "mean_raw": 26.0 }, "flash": { "mean_score": 88.0, "mean_raw": 12.0 }, "funasr_nano": { "mean_score": 89.0, "mean_raw": 11.0 }, "gemini_3_5_flash_lite_stt": { "mean_score": 78.0, "mean_raw": 22.0 }, "gemini_3_5_flash_stt": { "mean_score": 89.0, "mean_raw": 11.0 }, "gemini_3_5_transcribe": { "mean_score": 92.0, "mean_raw": 8.0 }, "gemini_3_6_flash_stt": { "mean_score": 87.0, "mean_raw": 13.0 }, "gemini_3_7_flash_stt": { "mean_score": 100.0, "mean_raw": 0.0 }, "gemini_3_8_flash_stt": { "mean_score": 95.0, "mean_raw": 5.0 }, "gemini_flash_stt": { "mean_score": 90.0, "mean_raw": 10.0 }, "gemini_pro_stt": { "mean_score": 82.0, "mean_raw": 18.0 }, "gemini_stt": { "mean_score": 81.0, "mean_raw": 19.0 }, "glm_asr_nano": { "mean_score": 92.0, "mean_raw": 8.0 }, "gpt_transcribe": { "mean_score": 95.0, "mean_raw": 5.0 }, "mai_transcribe_2": { "mean_score": 96.0, "mean_raw": 4.0 }, "mimo": { "mean_score": 95.0, "mean_raw": 5.0 }, "mimo_stt": { "mean_score": 82.0, "mean_raw": 18.0 }, "moss_audio": { "mean_score": 82.0, "mean_raw": 18.0 }, "qwen3_oss": { "mean_score": 85.0, "mean_raw": 15.0 }, "qwen_audio_3_1_asr": { "mean_score": 90.0, "mean_raw": 10.0 }, "r2t2": { "mean_score": 87.0, "mean_raw": 13.0 }, "soniox": { "mean_score": 95.0, "mean_raw": 5.0 }, "stepfun": { "mean_score": 88.0, "mean_raw": 12.0 }, "vibevoice": { "mean_score": 88.0, "mean_raw": 12.0 }, "voxtral": { "mean_score": 67.0, "mean_raw": 33.0 }, "whisper": { "mean_score": 82.0, "mean_raw": 18.0 }, "whisper_v3_turbo": { "mean_score": 79.0, "mean_raw": 21.0 } } }, "50": { "duration_s": 106, "model_scores": { "apple_speechanalyzer": { "mean_score": 63.0, "mean_raw": 37.0 }, "doubao": { "mean_score": 92.0, "mean_raw": 8.0 }, "elevenlabs": { "mean_score": 95.0, "mean_raw": 5.0 }, "firered": { "mean_score": 60.0, "mean_raw": 40.0 }, "firered_aed": { "mean_score": 67.0, "mean_raw": 33.0 }, "flash": { "mean_score": 91.0, "mean_raw": 9.0 }, "funasr_nano": { "mean_score": 69.0, "mean_raw": 31.0 }, "gemini_3_5_flash_lite_stt": { "mean_score": 46.0, "mean_raw": 54.0 }, "gemini_3_5_flash_stt": { "mean_score": 82.0, "mean_raw": 18.0 }, "gemini_3_5_transcribe": { "mean_score": 75.0, "mean_raw": 25.0 }, "gemini_3_6_flash_stt": { "mean_score": 81.0, "mean_raw": 19.0 }, "gemini_3_7_flash_stt": { "mean_score": 83.0, "mean_raw": 17.0 }, "gemini_3_8_flash_stt": { "mean_score": 96.0, "mean_raw": 4.0 }, "gemini_flash_stt": { "mean_score": 82.0, "mean_raw": 18.0 }, "gemini_pro_stt": { "mean_score": 89.0, "mean_raw": 11.0 }, "gemini_stt": { "mean_score": 64.0, "mean_raw": 36.0 }, "glm_asr_nano": { "mean_score": 89.0, "mean_raw": 11.0 }, "gpt_transcribe": { "mean_score": 87.0, "mean_raw": 13.0 }, "mai_transcribe_2": { "mean_score": 80.0, "mean_raw": 20.0 }, "mimo": { "mean_score": 91.0, "mean_raw": 9.0 }, "mimo_stt": { "mean_score": 87.0, "mean_raw": 13.0 }, "moss_audio": { "mean_score": 56.0, "mean_raw": 44.0 }, "qwen3_oss": { "mean_score": 80.0, "mean_raw": 20.0 }, "qwen_audio_3_1_asr": { "mean_score": 92.0, "mean_raw": 8.0 }, "r2t2": { "mean_score": 88.0, "mean_raw": 12.0 }, "soniox": { "mean_score": 79.0, "mean_raw": 21.0 }, "stepfun": { "mean_score": 75.0, "mean_raw": 25.0 }, "vibevoice": { "mean_score": 70.0, "mean_raw": 30.0 }, "voxtral": { "mean_score": 66.0, "mean_raw": 34.0 }, "whisper": { "mean_score": 79.0, "mean_raw": 21.0 }, "whisper_v3_turbo": { "mean_score": 67.0, "mean_raw": 33.0 } } }, "51": { "duration_s": 33, "model_scores": { "apple_speechanalyzer": { "mean_score": 73.0, "mean_raw": 27.0 }, "doubao": { "mean_score": 99.0, "mean_raw": 1.0 }, "elevenlabs": { "mean_score": 94.0, "mean_raw": 6.0 }, "firered": { "mean_score": 67.0, "mean_raw": 33.0 }, "firered_aed": { "mean_score": 85.0, "mean_raw": 15.0 }, "flash": { "mean_score": 97.0, "mean_raw": 3.0 }, "funasr_nano": { "mean_score": 90.0, "mean_raw": 10.0 }, "gemini_3_5_flash_lite_stt": { "mean_score": 39.0, "mean_raw": 61.0 }, "gemini_3_5_flash_stt": { "mean_score": 95.0, "mean_raw": 5.0 }, "gemini_3_5_transcribe": { "mean_score": 90.0, "mean_raw": 10.0 }, "gemini_3_6_flash_stt": { "mean_score": 79.0, "mean_raw": 21.0 }, "gemini_3_7_flash_stt": { "mean_score": 94.0, "mean_raw": 6.0 }, "gemini_3_8_flash_stt": { "mean_score": 96.0, "mean_raw": 4.0 }, "gemini_flash_stt": { "mean_score": 85.0, "mean_raw": 15.0 }, "gemini_pro_stt": { "mean_score": 97.0, "mean_raw": 3.0 }, "gemini_stt": { "mean_score": 80.0, "mean_raw": 20.0 }, "glm_asr_nano": { "mean_score": 84.0, "mean_raw": 16.0 }, "gpt_transcribe": { "mean_score": 92.0, "mean_raw": 8.0 }, "mai_transcribe_2": { "mean_score": 97.0, "mean_raw": 3.0 }, "mimo": { "mean_score": 97.0, "mean_raw": 3.0 }, "mimo_stt": { "mean_score": 96.0, "mean_raw": 4.0 }, "moss_audio": { "mean_score": 86.0, "mean_raw": 14.0 }, "qwen3_oss": { "mean_score": 98.0, "mean_raw": 2.0 }, "qwen_audio_3_1_asr": { "mean_score": 94.0, "mean_raw": 6.0 }, "r2t2": { "mean_score": 94.0, "mean_raw": 6.0 }, "soniox": { "mean_score": 84.0, "mean_raw": 16.0 }, "stepfun": { "mean_score": 93.0, "mean_raw": 7.0 }, "vibevoice": { "mean_score": 91.0, "mean_raw": 9.0 }, "voxtral": { "mean_score": 37.0, "mean_raw": 63.0 }, "whisper": { "mean_score": 96.0, "mean_raw": 4.0 }, "whisper_v3_turbo": { "mean_score": 68.0, "mean_raw": 32.0 } } }, "52": { "duration_s": 25, "model_scores": { "apple_speechanalyzer": { "mean_score": 90.0, "mean_raw": 10.0 }, "doubao": { "mean_score": 97.0, "mean_raw": 3.0 }, "elevenlabs": { "mean_score": 98.0, "mean_raw": 2.0 }, "firered": { "mean_score": 74.0, "mean_raw": 26.0 }, "firered_aed": { "mean_score": 93.0, "mean_raw": 7.0 }, "flash": { "mean_score": 98.0, "mean_raw": 2.0 }, "funasr_nano": { "mean_score": 96.0, "mean_raw": 4.0 }, "gemini_3_5_flash_lite_stt": { "mean_score": 47.0, "mean_raw": 53.0 }, "gemini_3_5_flash_stt": { "mean_score": 95.0, "mean_raw": 5.0 }, "gemini_3_5_transcribe": { "mean_score": 89.0, "mean_raw": 11.0 }, "gemini_3_6_flash_stt": { "mean_score": 95.0, "mean_raw": 5.0 }, "gemini_3_7_flash_stt": { "mean_score": 96.0, "mean_raw": 4.0 }, "gemini_3_8_flash_stt": { "mean_score": 95.0, "mean_raw": 5.0 }, "gemini_flash_stt": { "mean_score": 95.0, "mean_raw": 5.0 }, "gemini_pro_stt": { "mean_score": 98.0, "mean_raw": 2.0 }, "gemini_stt": { "mean_score": 87.0, "mean_raw": 13.0 }, "glm_asr_nano": { "mean_score": 96.0, "mean_raw": 4.0 }, "gpt_transcribe": { "mean_score": 98.0, "mean_raw": 2.0 }, "mai_transcribe_2": { "mean_score": 97.0, "mean_raw": 3.0 }, "mimo": { "mean_score": 99.0, "mean_raw": 1.0 }, "mimo_stt": { "mean_score": 98.0, "mean_raw": 2.0 }, "moss_audio": { "mean_score": 81.0, "mean_raw": 19.0 }, "qwen3_oss": { "mean_score": 96.0, "mean_raw": 4.0 }, "qwen_audio_3_1_asr": { "mean_score": 98.0, "mean_raw": 2.0 }, "r2t2": { "mean_score": 95.0, "mean_raw": 5.0 }, "soniox": { "mean_score": 83.0, "mean_raw": 17.0 }, "stepfun": { "mean_score": 94.0, "mean_raw": 6.0 }, "vibevoice": { "mean_score": 90.0, "mean_raw": 10.0 }, "voxtral": { "mean_score": 75.0, "mean_raw": 25.0 }, "whisper": { "mean_score": 86.0, "mean_raw": 14.0 }, "whisper_v3_turbo": { "mean_score": 85.0, "mean_raw": 15.0 } } }, "53": { "duration_s": 35, "model_scores": { "apple_speechanalyzer": { "mean_score": 89.0, "mean_raw": 11.0 }, "doubao": { "mean_score": 97.0, "mean_raw": 3.0 }, "elevenlabs": { "mean_score": 99.0, "mean_raw": 1.0 }, "firered": { "mean_score": 81.0, "mean_raw": 19.0 }, "firered_aed": { "mean_score": 96.0, "mean_raw": 4.0 }, "flash": { "mean_score": 96.0, "mean_raw": 4.0 }, "funasr_nano": { "mean_score": 98.0, "mean_raw": 2.0 }, "gemini_3_5_flash_lite_stt": { "mean_score": 91.0, "mean_raw": 9.0 }, "gemini_3_5_flash_stt": { "mean_score": 90.0, "mean_raw": 10.0 }, "gemini_3_5_transcribe": { "mean_score": 96.0, "mean_raw": 4.0 }, "gemini_3_6_flash_stt": { "mean_score": 94.0, "mean_raw": 6.0 }, "gemini_3_7_flash_stt": { "mean_score": 97.0, "mean_raw": 3.0 }, "gemini_3_8_flash_stt": { "mean_score": 98.0, "mean_raw": 2.0 }, "gemini_flash_stt": { "mean_score": 94.0, "mean_raw": 6.0 }, "gemini_pro_stt": { "mean_score": 96.0, "mean_raw": 4.0 }, "gemini_stt": { "mean_score": 91.0, "mean_raw": 9.0 }, "glm_asr_nano": { "mean_score": 97.0, "mean_raw": 3.0 }, "gpt_transcribe": { "mean_score": 98.0, "mean_raw": 2.0 }, "mai_transcribe_2": { "mean_score": 100.0, "mean_raw": 0.0 }, "mimo": { "mean_score": 100.0, "mean_raw": 0.0 }, "mimo_stt": { "mean_score": 96.0, "mean_raw": 4.0 }, "moss_audio": { "mean_score": 93.0, "mean_raw": 7.0 }, "qwen3_oss": { "mean_score": 98.0, "mean_raw": 2.0 }, "qwen_audio_3_1_asr": { "mean_score": 94.0, "mean_raw": 6.0 }, "r2t2": { "mean_score": 99.0, "mean_raw": 1.0 }, "soniox": { "mean_score": 97.0, "mean_raw": 3.0 }, "stepfun": { "mean_score": 97.0, "mean_raw": 3.0 }, "vibevoice": { "mean_score": 98.0, "mean_raw": 2.0 }, "voxtral": { "mean_score": 90.0, "mean_raw": 10.0 }, "whisper": { "mean_score": 95.0, "mean_raw": 5.0 }, "whisper_v3_turbo": { "mean_score": 95.0, "mean_raw": 5.0 } } }, "54": { "duration_s": 36, "model_scores": { "apple_speechanalyzer": { "mean_score": 85.0, "mean_raw": 15.0 }, "doubao": { "mean_score": 99.0, "mean_raw": 1.0 }, "elevenlabs": { "mean_score": 98.0, "mean_raw": 2.0 }, "firered": { "mean_score": 86.0, "mean_raw": 14.0 }, "firered_aed": { "mean_score": 98.0, "mean_raw": 2.0 }, "flash": { "mean_score": 97.0, "mean_raw": 3.0 }, "funasr_nano": { "mean_score": 99.0, "mean_raw": 1.0 }, "gemini_3_5_flash_lite_stt": { "mean_score": 93.0, "mean_raw": 7.0 }, "gemini_3_5_flash_stt": { "mean_score": 87.0, "mean_raw": 13.0 }, "gemini_3_5_transcribe": { "mean_score": 97.0, "mean_raw": 3.0 }, "gemini_3_6_flash_stt": { "mean_score": 93.0, "mean_raw": 7.0 }, "gemini_flash_stt": { "mean_score": 95.0, "mean_raw": 5.0 }, "gemini_pro_stt": { "mean_score": 99.0, "mean_raw": 1.0 }, "gemini_stt": { "mean_score": 95.0, "mean_raw": 5.0 }, "glm_asr_nano": { "mean_score": 97.0, "mean_raw": 3.0 }, "gpt_transcribe": { "mean_score": 96.0, "mean_raw": 4.0 }, "mai_transcribe_2": { "mean_score": 99.0, "mean_raw": 1.0 }, "mimo": { "mean_score": 99.0, "mean_raw": 1.0 }, "mimo_stt": { "mean_score": 96.0, "mean_raw": 4.0 }, "moss_audio": { "mean_score": 91.0, "mean_raw": 9.0 }, "qwen3_oss": { "mean_score": 99.0, "mean_raw": 1.0 }, "qwen_audio_3_1_asr": { "mean_score": 98.0, "mean_raw": 2.0 }, "r2t2": { "mean_score": 97.0, "mean_raw": 3.0 }, "soniox": { "mean_score": 96.0, "mean_raw": 4.0 }, "stepfun": { "mean_score": 96.0, "mean_raw": 4.0 }, "vibevoice": { "mean_score": 97.0, "mean_raw": 3.0 }, "voxtral": { "mean_score": 90.0, "mean_raw": 10.0 }, "whisper": { "mean_score": 96.0, "mean_raw": 4.0 }, "whisper_v3_turbo": { "mean_score": 85.0, "mean_raw": 15.0 } } }, "55": { "duration_s": 33, "model_scores": { "apple_speechanalyzer": { "mean_score": 84.0, "mean_raw": 16.0 }, "doubao": { "mean_score": 90.0, "mean_raw": 10.0 }, "elevenlabs": { "mean_score": 94.0, "mean_raw": 6.0 }, "firered": { "mean_score": 84.0, "mean_raw": 16.0 }, "firered_aed": { "mean_score": 89.0, "mean_raw": 11.0 }, "flash": { "mean_score": 91.0, "mean_raw": 9.0 }, "funasr_nano": { "mean_score": 92.0, "mean_raw": 8.0 }, "gemini_3_5_flash_lite_stt": { "mean_score": 80.0, "mean_raw": 20.0 }, "gemini_3_5_flash_stt": { "mean_score": 95.0, "mean_raw": 5.0 }, "gemini_3_5_transcribe": { "mean_score": 95.0, "mean_raw": 5.0 }, "gemini_3_6_flash_stt": { "mean_score": 96.0, "mean_raw": 4.0 }, "gemini_3_7_flash_stt": { "mean_score": 90.0, "mean_raw": 10.0 }, "gemini_3_8_flash_stt": { "mean_score": 92.0, "mean_raw": 8.0 }, "gemini_flash_stt": { "mean_score": 95.0, "mean_raw": 5.0 }, "gemini_pro_stt": { "mean_score": 95.0, "mean_raw": 5.0 }, "gemini_stt": { "mean_score": 91.0, "mean_raw": 9.0 }, "glm_asr_nano": { "mean_score": 92.0, "mean_raw": 8.0 }, "gpt_transcribe": { "mean_score": 95.0, "mean_raw": 5.0 }, "mai_transcribe_2": { "mean_score": 95.0, "mean_raw": 5.0 }, "mimo": { "mean_score": 95.0, "mean_raw": 5.0 }, "mimo_stt": { "mean_score": 95.0, "mean_raw": 5.0 }, "moss_audio": { "mean_score": 75.0, "mean_raw": 25.0 }, "qwen3_oss": { "mean_score": 89.0, "mean_raw": 11.0 }, "qwen_audio_3_1_asr": { "mean_score": 93.0, "mean_raw": 7.0 }, "r2t2": { "mean_score": 89.0, "mean_raw": 11.0 }, "soniox": { "mean_score": 95.0, "mean_raw": 5.0 }, "stepfun": { "mean_score": 77.0, "mean_raw": 23.0 }, "vibevoice": { "mean_score": 93.0, "mean_raw": 7.0 }, "voxtral": { "mean_score": 95.0, "mean_raw": 5.0 }, "whisper": { "mean_score": 95.0, "mean_raw": 5.0 }, "whisper_v3_turbo": { "mean_score": 74.0, "mean_raw": 26.0 } } }, "56": { "duration_s": 15, "model_scores": { "apple_speechanalyzer": { "mean_score": 70.0, "mean_raw": 30.0 }, "doubao": { "mean_score": 79.0, "mean_raw": 21.0 }, "elevenlabs": { "mean_score": 98.0, "mean_raw": 2.0 }, "firered": { "mean_score": 68.0, "mean_raw": 32.0 }, "firered_aed": { "mean_score": 70.0, "mean_raw": 30.0 }, "flash": { "mean_score": 89.0, "mean_raw": 11.0 }, "funasr_nano": { "mean_score": 79.0, "mean_raw": 21.0 }, "gemini_3_5_flash_lite_stt": { "mean_score": 89.0, "mean_raw": 11.0 }, "gemini_3_5_flash_stt": { "mean_score": 89.0, "mean_raw": 11.0 }, "gemini_3_5_transcribe": { "mean_score": 79.0, "mean_raw": 21.0 }, "gemini_3_6_flash_stt": { "mean_score": 96.0, "mean_raw": 4.0 }, "gemini_3_7_flash_stt": { "mean_score": 57.0, "mean_raw": 43.0 }, "gemini_3_8_flash_stt": { "mean_score": 97.0, "mean_raw": 3.0 }, "gemini_flash_stt": { "mean_score": 98.0, "mean_raw": 2.0 }, "gemini_pro_stt": { "mean_score": 98.0, "mean_raw": 2.0 }, "gemini_stt": { "mean_score": 98.0, "mean_raw": 2.0 }, "glm_asr_nano": { "mean_score": 90.0, "mean_raw": 10.0 }, "gpt_transcribe": { "mean_score": 87.0, "mean_raw": 13.0 }, "mai_transcribe_2": { "mean_score": 100.0, "mean_raw": 0.0 }, "mimo": { "mean_score": 89.0, "mean_raw": 11.0 }, "mimo_stt": { "mean_score": 98.0, "mean_raw": 2.0 }, "moss_audio": { "mean_score": 76.0, "mean_raw": 24.0 }, "qwen3_oss": { "mean_score": 89.0, "mean_raw": 11.0 }, "qwen_audio_3_1_asr": { "mean_score": 86.0, "mean_raw": 14.0 }, "r2t2": { "mean_score": 86.0, "mean_raw": 14.0 }, "soniox": { "mean_score": 89.0, "mean_raw": 11.0 }, "stepfun": { "mean_score": 90.0, "mean_raw": 10.0 }, "vibevoice": { "mean_score": 75.0, "mean_raw": 25.0 }, "voxtral": { "mean_score": 77.0, "mean_raw": 23.0 }, "whisper": { "mean_score": 88.0, "mean_raw": 12.0 }, "whisper_v3_turbo": { "mean_score": 66.0, "mean_raw": 34.0 } } }, "57": { "duration_s": 38, "model_scores": { "apple_speechanalyzer": { "mean_score": 52.0, "mean_raw": 48.0 }, "doubao": { "mean_score": 92.0, "mean_raw": 8.0 }, "elevenlabs": { "mean_score": 74.0, "mean_raw": 26.0 }, "firered": { "mean_score": 42.0, "mean_raw": 58.0 }, "firered_aed": { "mean_score": 66.0, "mean_raw": 34.0 }, "flash": { "mean_score": 97.0, "mean_raw": 3.0 }, "funasr_nano": { "mean_score": 90.0, "mean_raw": 10.0 }, "gemini_3_5_flash_lite_stt": { "mean_score": 56.0, "mean_raw": 44.0 }, "gemini_3_5_flash_stt": { "mean_score": 59.0, "mean_raw": 41.0 }, "gemini_3_5_transcribe": { "mean_score": 85.0, "mean_raw": 15.0 }, "gemini_3_6_flash_stt": { "mean_score": 84.0, "mean_raw": 16.0 }, "gemini_3_7_flash_stt": { "mean_score": 60.0, "mean_raw": 40.0 }, "gemini_3_8_flash_stt": { "mean_score": 64.0, "mean_raw": 36.0 }, "gemini_flash_stt": { "mean_score": 59.0, "mean_raw": 41.0 }, "gemini_pro_stt": { "mean_score": 98.0, "mean_raw": 2.0 }, "gemini_stt": { "mean_score": 54.0, "mean_raw": 46.0 }, "glm_asr_nano": { "mean_score": 95.0, "mean_raw": 5.0 }, "gpt_transcribe": { "mean_score": 89.0, "mean_raw": 11.0 }, "mai_transcribe_2": { "mean_score": 97.0, "mean_raw": 3.0 }, "mimo": { "mean_score": 99.0, "mean_raw": 1.0 }, "mimo_stt": { "mean_score": 98.0, "mean_raw": 2.0 }, "moss_audio": { "mean_score": 55.0, "mean_raw": 45.0 }, "qwen3_oss": { "mean_score": 98.0, "mean_raw": 2.0 }, "qwen_audio_3_1_asr": { "mean_score": 99.0, "mean_raw": 1.0 }, "r2t2": { "mean_score": 95.0, "mean_raw": 5.0 }, "soniox": { "mean_score": 97.0, "mean_raw": 3.0 }, "stepfun": { "mean_score": 80.0, "mean_raw": 20.0 }, "vibevoice": { "mean_score": 96.0, "mean_raw": 4.0 }, "voxtral": { "mean_score": 38.0, "mean_raw": 62.0 }, "whisper": { "mean_score": 98.0, "mean_raw": 2.0 }, "whisper_v3_turbo": { "mean_score": 43.0, "mean_raw": 57.0 } } }, "58": { "duration_s": 15, "model_scores": { "apple_speechanalyzer": { "mean_score": 82.0, "mean_raw": 18.0 }, "doubao": { "mean_score": 98.0, "mean_raw": 2.0 }, "elevenlabs": { "mean_score": 97.0, "mean_raw": 3.0 }, "firered": { "mean_score": 79.0, "mean_raw": 21.0 }, "firered_aed": { "mean_score": 89.0, "mean_raw": 11.0 }, "flash": { "mean_score": 98.0, "mean_raw": 2.0 }, "funasr_nano": { "mean_score": 98.0, "mean_raw": 2.0 }, "gemini_3_5_flash_lite_stt": { "mean_score": 77.0, "mean_raw": 23.0 }, "gemini_3_5_flash_stt": { "mean_score": 84.0, "mean_raw": 16.0 }, "gemini_3_5_transcribe": { "mean_score": 91.0, "mean_raw": 9.0 }, "gemini_3_6_flash_stt": { "mean_score": 97.0, "mean_raw": 3.0 }, "gemini_3_7_flash_stt": { "mean_score": 97.0, "mean_raw": 3.0 }, "gemini_3_8_flash_stt": { "mean_score": 98.0, "mean_raw": 2.0 }, "gemini_flash_stt": { "mean_score": 99.0, "mean_raw": 1.0 }, "gemini_pro_stt": { "mean_score": 100.0, "mean_raw": 0.0 }, "gemini_stt": { "mean_score": 47.0, "mean_raw": 53.0 }, "glm_asr_nano": { "mean_score": 99.0, "mean_raw": 1.0 }, "gpt_transcribe": { "mean_score": 95.0, "mean_raw": 5.0 }, "mai_transcribe_2": { "mean_score": 100.0, "mean_raw": 0.0 }, "mimo": { "mean_score": 98.0, "mean_raw": 2.0 }, "mimo_stt": { "mean_score": 93.0, "mean_raw": 7.0 }, "moss_audio": { "mean_score": 89.0, "mean_raw": 11.0 }, "qwen3_oss": { "mean_score": 98.0, "mean_raw": 2.0 }, "qwen_audio_3_1_asr": { "mean_score": 98.0, "mean_raw": 2.0 }, "r2t2": { "mean_score": 100.0, "mean_raw": 0.0 }, "soniox": { "mean_score": 94.0, "mean_raw": 6.0 }, "stepfun": { "mean_score": 98.0, "mean_raw": 2.0 }, "vibevoice": { "mean_score": 98.0, "mean_raw": 2.0 }, "voxtral": { "mean_score": 90.0, "mean_raw": 10.0 }, "whisper": { "mean_score": 94.0, "mean_raw": 6.0 }, "whisper_v3_turbo": { "mean_score": 98.0, "mean_raw": 2.0 } } }, "59": { "duration_s": 28, "model_scores": { "apple_speechanalyzer": { "mean_score": 97.0, "mean_raw": 3.0 }, "doubao": { "mean_score": 100.0, "mean_raw": 0.0 }, "elevenlabs": { "mean_score": 93.0, "mean_raw": 7.0 }, "firered": { "mean_score": 79.0, "mean_raw": 21.0 }, "firered_aed": { "mean_score": 79.0, "mean_raw": 21.0 }, "flash": { "mean_score": 98.0, "mean_raw": 2.0 }, "funasr_nano": { "mean_score": 88.0, "mean_raw": 12.0 }, "gemini_3_5_flash_lite_stt": { "mean_score": 85.0, "mean_raw": 15.0 }, "gemini_3_5_flash_stt": { "mean_score": 92.0, "mean_raw": 8.0 }, "gemini_3_5_transcribe": { "mean_score": 59.0, "mean_raw": 41.0 }, "gemini_3_6_flash_stt": { "mean_score": 88.0, "mean_raw": 12.0 }, "gemini_3_7_flash_stt": { "mean_score": 94.0, "mean_raw": 6.0 }, "gemini_3_8_flash_stt": { "mean_score": 90.0, "mean_raw": 10.0 }, "gemini_flash_stt": { "mean_score": 87.0, "mean_raw": 13.0 }, "gemini_pro_stt": { "mean_score": 97.0, "mean_raw": 3.0 }, "gemini_stt": { "mean_score": 88.0, "mean_raw": 12.0 }, "glm_asr_nano": { "mean_score": 96.0, "mean_raw": 4.0 }, "gpt_transcribe": { "mean_score": 88.0, "mean_raw": 12.0 }, "mai_transcribe_2": { "mean_score": 97.0, "mean_raw": 3.0 }, "mimo": { "mean_score": 97.0, "mean_raw": 3.0 }, "mimo_stt": { "mean_score": 94.0, "mean_raw": 6.0 }, "moss_audio": { "mean_score": 97.0, "mean_raw": 3.0 }, "qwen3_oss": { "mean_score": 95.0, "mean_raw": 5.0 }, "qwen_audio_3_1_asr": { "mean_score": 97.0, "mean_raw": 3.0 }, "r2t2": { "mean_score": 96.0, "mean_raw": 4.0 }, "soniox": { "mean_score": 98.0, "mean_raw": 2.0 }, "stepfun": { "mean_score": 92.0, "mean_raw": 8.0 }, "vibevoice": { "mean_score": 80.0, "mean_raw": 20.0 }, "voxtral": { "mean_score": 64.0, "mean_raw": 36.0 }, "whisper": { "mean_score": 94.0, "mean_raw": 6.0 }, "whisper_v3_turbo": { "mean_score": 91.0, "mean_raw": 9.0 } } }, "60": { "duration_s": 18, "model_scores": { "apple_speechanalyzer": { "mean_score": 51.0, "mean_raw": 49.0 }, "doubao": { "mean_score": 100.0, "mean_raw": 0.0 }, "elevenlabs": { "mean_score": 64.0, "mean_raw": 36.0 }, "firered": { "mean_score": 40.0, "mean_raw": 60.0 }, "firered_aed": { "mean_score": 62.0, "mean_raw": 38.0 }, "flash": { "mean_score": 100.0, "mean_raw": 0.0 }, "funasr_nano": { "mean_score": 63.0, "mean_raw": 37.0 }, "gemini_3_5_flash_lite_stt": { "mean_score": 52.0, "mean_raw": 48.0 }, "gemini_3_5_flash_stt": { "mean_score": 97.0, "mean_raw": 3.0 }, "gemini_3_5_transcribe": { "mean_score": 62.0, "mean_raw": 38.0 }, "gemini_3_6_flash_stt": { "mean_score": 61.0, "mean_raw": 39.0 }, "gemini_3_7_flash_stt": { "mean_score": 62.0, "mean_raw": 38.0 }, "gemini_3_8_flash_stt": { "mean_score": 63.0, "mean_raw": 37.0 }, "gemini_flash_stt": { "mean_score": 59.0, "mean_raw": 41.0 }, "gemini_pro_stt": { "mean_score": 62.0, "mean_raw": 38.0 }, "gemini_stt": { "mean_score": 33.0, "mean_raw": 67.0 }, "glm_asr_nano": { "mean_score": 48.0, "mean_raw": 52.0 }, "gpt_transcribe": { "mean_score": 63.0, "mean_raw": 37.0 }, "mai_transcribe_2": { "mean_score": 100.0, "mean_raw": 0.0 }, "mimo": { "mean_score": 63.0, "mean_raw": 37.0 }, "mimo_stt": { "mean_score": 64.0, "mean_raw": 36.0 }, "moss_audio": { "mean_score": 63.0, "mean_raw": 37.0 }, "qwen3_oss": { "mean_score": 44.0, "mean_raw": 56.0 }, "qwen_audio_3_1_asr": { "mean_score": 64.0, "mean_raw": 36.0 }, "r2t2": { "mean_score": 62.0, "mean_raw": 38.0 }, "soniox": { "mean_score": 97.0, "mean_raw": 3.0 }, "stepfun": { "mean_score": 79.0, "mean_raw": 21.0 }, "vibevoice": { "mean_score": 63.0, "mean_raw": 37.0 }, "voxtral": { "mean_score": 58.0, "mean_raw": 42.0 }, "whisper": { "mean_score": 93.0, "mean_raw": 7.0 }, "whisper_v3_turbo": { "mean_score": 85.0, "mean_raw": 15.0 } } }, "61": { "duration_s": 19, "model_scores": { "apple_speechanalyzer": { "mean_score": 94.0, "mean_raw": 6.0 }, "doubao": { "mean_score": 85.0, "mean_raw": 15.0 }, "elevenlabs": { "mean_score": 100.0, "mean_raw": 0.0 }, "firered": { "mean_score": 67.0, "mean_raw": 33.0 }, "firered_aed": { "mean_score": 100.0, "mean_raw": 0.0 }, "flash": { "mean_score": 100.0, "mean_raw": 0.0 }, "funasr_nano": { "mean_score": 100.0, "mean_raw": 0.0 }, "gemini_3_5_flash_lite_stt": { "mean_score": 84.0, "mean_raw": 16.0 }, "gemini_3_5_flash_stt": { "mean_score": 99.0, "mean_raw": 1.0 }, "gemini_3_5_transcribe": { "mean_score": 96.0, "mean_raw": 4.0 }, "gemini_3_6_flash_stt": { "mean_score": 96.0, "mean_raw": 4.0 }, "gemini_3_7_flash_stt": { "mean_score": 100.0, "mean_raw": 0.0 }, "gemini_3_8_flash_stt": { "mean_score": 99.0, "mean_raw": 1.0 }, "gemini_flash_stt": { "mean_score": 95.0, "mean_raw": 5.0 }, "gemini_pro_stt": { "mean_score": 95.0, "mean_raw": 5.0 }, "gemini_stt": { "mean_score": 82.0, "mean_raw": 18.0 }, "glm_asr_nano": { "mean_score": 100.0, "mean_raw": 0.0 }, "gpt_transcribe": { "mean_score": 100.0, "mean_raw": 0.0 }, "mai_transcribe_2": { "mean_score": 100.0, "mean_raw": 0.0 }, "mimo": { "mean_score": 85.0, "mean_raw": 15.0 }, "mimo_stt": { "mean_score": 97.0, "mean_raw": 3.0 }, "moss_audio": { "mean_score": 85.0, "mean_raw": 15.0 }, "qwen3_oss": { "mean_score": 89.0, "mean_raw": 11.0 }, "qwen_audio_3_1_asr": { "mean_score": 100.0, "mean_raw": 0.0 }, "r2t2": { "mean_score": 100.0, "mean_raw": 0.0 }, "soniox": { "mean_score": 95.0, "mean_raw": 5.0 }, "stepfun": { "mean_score": 95.0, "mean_raw": 5.0 }, "vibevoice": { "mean_score": 95.0, "mean_raw": 5.0 }, "voxtral": { "mean_score": 82.0, "mean_raw": 18.0 }, "whisper": { "mean_score": 94.0, "mean_raw": 6.0 }, "whisper_v3_turbo": { "mean_score": 89.0, "mean_raw": 11.0 } } }, "62": { "duration_s": 17, "model_scores": { "apple_speechanalyzer": { "mean_score": 100.0, "mean_raw": 0.0 }, "doubao": { "mean_score": 99.0, "mean_raw": 1.0 }, "elevenlabs": { "mean_score": 100.0, "mean_raw": 0.0 }, "firered": { "mean_score": 100.0, "mean_raw": 0.0 }, "firered_aed": { "mean_score": 100.0, "mean_raw": 0.0 }, "flash": { "mean_score": 100.0, "mean_raw": 0.0 }, "funasr_nano": { "mean_score": 100.0, "mean_raw": 0.0 }, "gemini_3_5_flash_lite_stt": { "mean_score": 100.0, "mean_raw": 0.0 }, "gemini_3_5_flash_stt": { "mean_score": 100.0, "mean_raw": 0.0 }, "gemini_3_5_transcribe": { "mean_score": 100.0, "mean_raw": 0.0 }, "gemini_3_6_flash_stt": { "mean_score": 100.0, "mean_raw": 0.0 }, "gemini_3_7_flash_stt": { "mean_score": 100.0, "mean_raw": 0.0 }, "gemini_3_8_flash_stt": { "mean_score": 100.0, "mean_raw": 0.0 }, "gemini_flash_stt": { "mean_score": 99.0, "mean_raw": 1.0 }, "gemini_pro_stt": { "mean_score": 100.0, "mean_raw": 0.0 }, "gemini_stt": { "mean_score": 98.0, "mean_raw": 2.0 }, "glm_asr_nano": { "mean_score": 100.0, "mean_raw": 0.0 }, "gpt_transcribe": { "mean_score": 100.0, "mean_raw": 0.0 }, "mai_transcribe_2": { "mean_score": 100.0, "mean_raw": 0.0 }, "mimo": { "mean_score": 100.0, "mean_raw": 0.0 }, "mimo_stt": { "mean_score": 100.0, "mean_raw": 0.0 }, "moss_audio": { "mean_score": 100.0, "mean_raw": 0.0 }, "qwen3_oss": { "mean_score": 100.0, "mean_raw": 0.0 }, "qwen_audio_3_1_asr": { "mean_score": 100.0, "mean_raw": 0.0 }, "r2t2": { "mean_score": 100.0, "mean_raw": 0.0 }, "soniox": { "mean_score": 100.0, "mean_raw": 0.0 }, "stepfun": { "mean_score": 100.0, "mean_raw": 0.0 }, "vibevoice": { "mean_score": 100.0, "mean_raw": 0.0 }, "voxtral": { "mean_score": 70.0, "mean_raw": 30.0 }, "whisper": { "mean_score": 100.0, "mean_raw": 0.0 }, "whisper_v3_turbo": { "mean_score": 100.0, "mean_raw": 0.0 } } }, "63": { "duration_s": 21, "model_scores": { "apple_speechanalyzer": { "mean_score": 86.0, "mean_raw": 14.0 }, "doubao": { "mean_score": 99.0, "mean_raw": 1.0 }, "elevenlabs": { "mean_score": 99.0, "mean_raw": 1.0 }, "firered": { "mean_score": 91.0, "mean_raw": 9.0 }, "firered_aed": { "mean_score": 96.0, "mean_raw": 4.0 }, "flash": { "mean_score": 99.0, "mean_raw": 1.0 }, "funasr_nano": { "mean_score": 98.0, "mean_raw": 2.0 }, "gemini_3_5_flash_lite_stt": { "mean_score": 96.0, "mean_raw": 4.0 }, "gemini_3_5_flash_stt": { "mean_score": 97.0, "mean_raw": 3.0 }, "gemini_3_5_transcribe": { "mean_score": 97.0, "mean_raw": 3.0 }, "gemini_3_6_flash_stt": { "mean_score": 82.0, "mean_raw": 18.0 }, "gemini_3_7_flash_stt": { "mean_score": 95.0, "mean_raw": 5.0 }, "gemini_3_8_flash_stt": { "mean_score": 98.0, "mean_raw": 2.0 }, "gemini_flash_stt": { "mean_score": 96.0, "mean_raw": 4.0 }, "gemini_pro_stt": { "mean_score": 97.0, "mean_raw": 3.0 }, "gemini_stt": { "mean_score": 79.0, "mean_raw": 21.0 }, "glm_asr_nano": { "mean_score": 98.0, "mean_raw": 2.0 }, "gpt_transcribe": { "mean_score": 98.0, "mean_raw": 2.0 }, "mai_transcribe_2": { "mean_score": 100.0, "mean_raw": 0.0 }, "mimo": { "mean_score": 100.0, "mean_raw": 0.0 }, "mimo_stt": { "mean_score": 96.0, "mean_raw": 4.0 }, "moss_audio": { "mean_score": 100.0, "mean_raw": 0.0 }, "qwen3_oss": { "mean_score": 98.0, "mean_raw": 2.0 }, "qwen_audio_3_1_asr": { "mean_score": 97.0, "mean_raw": 3.0 }, "r2t2": { "mean_score": 97.0, "mean_raw": 3.0 }, "soniox": { "mean_score": 99.0, "mean_raw": 1.0 }, "stepfun": { "mean_score": 98.0, "mean_raw": 2.0 }, "vibevoice": { "mean_score": 90.0, "mean_raw": 10.0 }, "voxtral": { "mean_score": 97.0, "mean_raw": 3.0 }, "whisper": { "mean_score": 99.0, "mean_raw": 1.0 }, "whisper_v3_turbo": { "mean_score": 88.0, "mean_raw": 12.0 } } }, "64": { "duration_s": 23, "model_scores": { "apple_speechanalyzer": { "mean_score": 97.0, "mean_raw": 3.0 }, "doubao": { "mean_score": 96.0, "mean_raw": 4.0 }, "elevenlabs": { "mean_score": 95.0, "mean_raw": 5.0 }, "firered": { "mean_score": 92.0, "mean_raw": 8.0 }, "firered_aed": { "mean_score": 94.0, "mean_raw": 6.0 }, "flash": { "mean_score": 95.0, "mean_raw": 5.0 }, "funasr_nano": { "mean_score": 96.0, "mean_raw": 4.0 }, "gemini_3_5_flash_lite_stt": { "mean_score": 88.0, "mean_raw": 12.0 }, "gemini_3_5_flash_stt": { "mean_score": 95.0, "mean_raw": 5.0 }, "gemini_3_5_transcribe": { "mean_score": 96.0, "mean_raw": 4.0 }, "gemini_3_6_flash_stt": { "mean_score": 94.0, "mean_raw": 6.0 }, "gemini_3_7_flash_stt": { "mean_score": 96.0, "mean_raw": 4.0 }, "gemini_3_8_flash_stt": { "mean_score": 96.0, "mean_raw": 4.0 }, "gemini_flash_stt": { "mean_score": 97.0, "mean_raw": 3.0 }, "gemini_pro_stt": { "mean_score": 98.0, "mean_raw": 2.0 }, "gemini_stt": { "mean_score": 98.0, "mean_raw": 2.0 }, "glm_asr_nano": { "mean_score": 96.0, "mean_raw": 4.0 }, "gpt_transcribe": { "mean_score": 95.0, "mean_raw": 5.0 }, "mai_transcribe_2": { "mean_score": 97.0, "mean_raw": 3.0 }, "mimo": { "mean_score": 96.0, "mean_raw": 4.0 }, "mimo_stt": { "mean_score": 90.0, "mean_raw": 10.0 }, "moss_audio": { "mean_score": 93.0, "mean_raw": 7.0 }, "qwen3_oss": { "mean_score": 95.0, "mean_raw": 5.0 }, "qwen_audio_3_1_asr": { "mean_score": 97.0, "mean_raw": 3.0 }, "r2t2": { "mean_score": 98.0, "mean_raw": 2.0 }, "soniox": { "mean_score": 80.0, "mean_raw": 20.0 }, "stepfun": { "mean_score": 95.0, "mean_raw": 5.0 }, "vibevoice": { "mean_score": 95.0, "mean_raw": 5.0 }, "voxtral": { "mean_score": 95.0, "mean_raw": 5.0 }, "whisper": { "mean_score": 98.0, "mean_raw": 2.0 }, "whisper_v3_turbo": { "mean_score": 96.0, "mean_raw": 4.0 } } }, "65": { "duration_s": 20, "model_scores": { "apple_speechanalyzer": { "mean_score": 98.0, "mean_raw": 2.0 }, "doubao": { "mean_score": 100.0, "mean_raw": 0.0 }, "elevenlabs": { "mean_score": 99.0, "mean_raw": 1.0 }, "firered": { "mean_score": 91.0, "mean_raw": 9.0 }, "firered_aed": { "mean_score": 100.0, "mean_raw": 0.0 }, "flash": { "mean_score": 99.0, "mean_raw": 1.0 }, "funasr_nano": { "mean_score": 100.0, "mean_raw": 0.0 }, "gemini_3_5_flash_lite_stt": { "mean_score": 99.0, "mean_raw": 1.0 }, "gemini_3_5_flash_stt": { "mean_score": 99.0, "mean_raw": 1.0 }, "gemini_3_5_transcribe": { "mean_score": 100.0, "mean_raw": 0.0 }, "gemini_3_6_flash_stt": { "mean_score": 98.0, "mean_raw": 2.0 }, "gemini_3_7_flash_stt": { "mean_score": 100.0, "mean_raw": 0.0 }, "gemini_3_8_flash_stt": { "mean_score": 97.0, "mean_raw": 3.0 }, "gemini_flash_stt": { "mean_score": 98.0, "mean_raw": 2.0 }, "gemini_pro_stt": { "mean_score": 99.0, "mean_raw": 1.0 }, "gemini_stt": { "mean_score": 98.0, "mean_raw": 2.0 }, "glm_asr_nano": { "mean_score": 100.0, "mean_raw": 0.0 }, "gpt_transcribe": { "mean_score": 100.0, "mean_raw": 0.0 }, "mai_transcribe_2": { "mean_score": 99.0, "mean_raw": 1.0 }, "mimo": { "mean_score": 100.0, "mean_raw": 0.0 }, "mimo_stt": { "mean_score": 99.0, "mean_raw": 1.0 }, "moss_audio": { "mean_score": 100.0, "mean_raw": 0.0 }, "qwen3_oss": { "mean_score": 100.0, "mean_raw": 0.0 }, "qwen_audio_3_1_asr": { "mean_score": 99.0, "mean_raw": 1.0 }, "r2t2": { "mean_score": 100.0, "mean_raw": 0.0 }, "soniox": { "mean_score": 99.0, "mean_raw": 1.0 }, "stepfun": { "mean_score": 100.0, "mean_raw": 0.0 }, "vibevoice": { "mean_score": 100.0, "mean_raw": 0.0 }, "voxtral": { "mean_score": 98.0, "mean_raw": 2.0 }, "whisper": { "mean_score": 100.0, "mean_raw": 0.0 }, "whisper_v3_turbo": { "mean_score": 99.0, "mean_raw": 1.0 } } }, "66": { "duration_s": 24, "model_scores": { "apple_speechanalyzer": { "mean_score": 83.0, "mean_raw": 17.0 }, "doubao": { "mean_score": 84.0, "mean_raw": 16.0 }, "elevenlabs": { "mean_score": 91.0, "mean_raw": 9.0 }, "firered": { "mean_score": 65.0, "mean_raw": 35.0 }, "firered_aed": { "mean_score": 92.0, "mean_raw": 8.0 }, "flash": { "mean_score": 89.0, "mean_raw": 11.0 }, "funasr_nano": { "mean_score": 91.0, "mean_raw": 9.0 }, "gemini_3_5_flash_lite_stt": { "mean_score": 91.0, "mean_raw": 9.0 }, "gemini_3_5_flash_stt": { "mean_score": 95.0, "mean_raw": 5.0 }, "gemini_3_5_transcribe": { "mean_score": 86.0, "mean_raw": 14.0 }, "gemini_3_6_flash_stt": { "mean_score": 93.0, "mean_raw": 7.0 }, "gemini_3_7_flash_stt": { "mean_score": 85.0, "mean_raw": 15.0 }, "gemini_3_8_flash_stt": { "mean_score": 95.0, "mean_raw": 5.0 }, "gemini_flash_stt": { "mean_score": 92.0, "mean_raw": 8.0 }, "gemini_pro_stt": { "mean_score": 95.0, "mean_raw": 5.0 }, "gemini_stt": { "mean_score": 87.0, "mean_raw": 13.0 }, "glm_asr_nano": { "mean_score": 90.0, "mean_raw": 10.0 }, "gpt_transcribe": { "mean_score": 87.0, "mean_raw": 13.0 }, "mai_transcribe_2": { "mean_score": 99.0, "mean_raw": 1.0 }, "mimo": { "mean_score": 96.0, "mean_raw": 4.0 }, "mimo_stt": { "mean_score": 90.0, "mean_raw": 10.0 }, "moss_audio": { "mean_score": 90.0, "mean_raw": 10.0 }, "qwen3_oss": { "mean_score": 91.0, "mean_raw": 9.0 }, "qwen_audio_3_1_asr": { "mean_score": 91.0, "mean_raw": 9.0 }, "r2t2": { "mean_score": 91.0, "mean_raw": 9.0 }, "soniox": { "mean_score": 90.0, "mean_raw": 10.0 }, "stepfun": { "mean_score": 91.0, "mean_raw": 9.0 }, "vibevoice": { "mean_score": 75.0, "mean_raw": 25.0 }, "voxtral": { "mean_score": 94.0, "mean_raw": 6.0 }, "whisper": { "mean_score": 91.0, "mean_raw": 9.0 }, "whisper_v3_turbo": { "mean_score": 97.0, "mean_raw": 3.0 } } }, "67": { "duration_s": 16, "model_scores": { "apple_speechanalyzer": { "mean_score": 97.0, "mean_raw": 3.0 }, "doubao": { "mean_score": 100.0, "mean_raw": 0.0 }, "elevenlabs": { "mean_score": 94.0, "mean_raw": 6.0 }, "firered": { "mean_score": 93.0, "mean_raw": 7.0 }, "firered_aed": { "mean_score": 98.0, "mean_raw": 2.0 }, "flash": { "mean_score": 98.0, "mean_raw": 2.0 }, "funasr_nano": { "mean_score": 99.0, "mean_raw": 1.0 }, "gemini_3_5_flash_lite_stt": { "mean_score": 94.0, "mean_raw": 6.0 }, "gemini_3_5_flash_stt": { "mean_score": 98.0, "mean_raw": 2.0 }, "gemini_3_5_transcribe": { "mean_score": 89.0, "mean_raw": 11.0 }, "gemini_3_6_flash_stt": { "mean_score": 95.0, "mean_raw": 5.0 }, "gemini_3_7_flash_stt": { "mean_score": 97.0, "mean_raw": 3.0 }, "gemini_3_8_flash_stt": { "mean_score": 99.0, "mean_raw": 1.0 }, "gemini_flash_stt": { "mean_score": 99.0, "mean_raw": 1.0 }, "gemini_pro_stt": { "mean_score": 100.0, "mean_raw": 0.0 }, "gemini_stt": { "mean_score": 93.0, "mean_raw": 7.0 }, "glm_asr_nano": { "mean_score": 99.0, "mean_raw": 1.0 }, "gpt_transcribe": { "mean_score": 98.0, "mean_raw": 2.0 }, "mai_transcribe_2": { "mean_score": 99.0, "mean_raw": 1.0 }, "mimo": { "mean_score": 100.0, "mean_raw": 0.0 }, "mimo_stt": { "mean_score": 100.0, "mean_raw": 0.0 }, "moss_audio": { "mean_score": 99.0, "mean_raw": 1.0 }, "qwen3_oss": { "mean_score": 91.0, "mean_raw": 9.0 }, "qwen_audio_3_1_asr": { "mean_score": 92.0, "mean_raw": 8.0 }, "r2t2": { "mean_score": 98.0, "mean_raw": 2.0 }, "soniox": { "mean_score": 98.0, "mean_raw": 2.0 }, "stepfun": { "mean_score": 99.0, "mean_raw": 1.0 }, "vibevoice": { "mean_score": 99.0, "mean_raw": 1.0 }, "voxtral": { "mean_score": 97.0, "mean_raw": 3.0 }, "whisper": { "mean_score": 98.0, "mean_raw": 2.0 }, "whisper_v3_turbo": { "mean_score": 97.0, "mean_raw": 3.0 } } }, "68": { "duration_s": 18, "model_scores": { "apple_speechanalyzer": { "mean_score": 89.0, "mean_raw": 11.0 }, "doubao": { "mean_score": 90.0, "mean_raw": 10.0 }, "elevenlabs": { "mean_score": 100.0, "mean_raw": 0.0 }, "firered": { "mean_score": 87.0, "mean_raw": 13.0 }, "firered_aed": { "mean_score": 90.0, "mean_raw": 10.0 }, "flash": { "mean_score": 85.0, "mean_raw": 15.0 }, "funasr_nano": { "mean_score": 89.0, "mean_raw": 11.0 }, "gemini_3_5_flash_lite_stt": { "mean_score": 86.0, "mean_raw": 14.0 }, "gemini_3_5_flash_stt": { "mean_score": 99.0, "mean_raw": 1.0 }, "gemini_3_5_transcribe": { "mean_score": 94.0, "mean_raw": 6.0 }, "gemini_3_6_flash_stt": { "mean_score": 87.0, "mean_raw": 13.0 }, "gemini_3_7_flash_stt": { "mean_score": 100.0, "mean_raw": 0.0 }, "gemini_3_8_flash_stt": { "mean_score": 100.0, "mean_raw": 0.0 }, "gemini_flash_stt": { "mean_score": 99.0, "mean_raw": 1.0 }, "gemini_pro_stt": { "mean_score": 99.0, "mean_raw": 1.0 }, "gemini_stt": { "mean_score": 89.0, "mean_raw": 11.0 }, "glm_asr_nano": { "mean_score": 90.0, "mean_raw": 10.0 }, "gpt_transcribe": { "mean_score": 85.0, "mean_raw": 15.0 }, "mai_transcribe_2": { "mean_score": 100.0, "mean_raw": 0.0 }, "mimo": { "mean_score": 100.0, "mean_raw": 0.0 }, "mimo_stt": { "mean_score": 100.0, "mean_raw": 0.0 }, "moss_audio": { "mean_score": 88.0, "mean_raw": 12.0 }, "qwen3_oss": { "mean_score": 89.0, "mean_raw": 11.0 }, "qwen_audio_3_1_asr": { "mean_score": 99.0, "mean_raw": 1.0 }, "r2t2": { "mean_score": 89.0, "mean_raw": 11.0 }, "soniox": { "mean_score": 90.0, "mean_raw": 10.0 }, "stepfun": { "mean_score": 90.0, "mean_raw": 10.0 }, "vibevoice": { "mean_score": 100.0, "mean_raw": 0.0 }, "voxtral": { "mean_score": 72.0, "mean_raw": 28.0 }, "whisper": { "mean_score": 99.0, "mean_raw": 1.0 }, "whisper_v3_turbo": { "mean_score": 90.0, "mean_raw": 10.0 } } }, "69": { "duration_s": 18, "model_scores": { "apple_speechanalyzer": { "mean_score": 98.0, "mean_raw": 2.0 }, "doubao": { "mean_score": 100.0, "mean_raw": 0.0 }, "elevenlabs": { "mean_score": 99.0, "mean_raw": 1.0 }, "firered": { "mean_score": 89.0, "mean_raw": 11.0 }, "firered_aed": { "mean_score": 94.0, "mean_raw": 6.0 }, "flash": { "mean_score": 100.0, "mean_raw": 0.0 }, "funasr_nano": { "mean_score": 100.0, "mean_raw": 0.0 }, "gemini_3_5_flash_lite_stt": { "mean_score": 100.0, "mean_raw": 0.0 }, "gemini_3_5_flash_stt": { "mean_score": 94.0, "mean_raw": 6.0 }, "gemini_3_5_transcribe": { "mean_score": 89.0, "mean_raw": 11.0 }, "gemini_3_6_flash_stt": { "mean_score": 95.0, "mean_raw": 5.0 }, "gemini_3_7_flash_stt": { "mean_score": 93.0, "mean_raw": 7.0 }, "gemini_3_8_flash_stt": { "mean_score": 95.0, "mean_raw": 5.0 }, "gemini_flash_stt": { "mean_score": 97.0, "mean_raw": 3.0 }, "gemini_pro_stt": { "mean_score": 95.0, "mean_raw": 5.0 }, "gemini_stt": { "mean_score": 85.0, "mean_raw": 15.0 }, "glm_asr_nano": { "mean_score": 99.0, "mean_raw": 1.0 }, "gpt_transcribe": { "mean_score": 98.0, "mean_raw": 2.0 }, "mai_transcribe_2": { "mean_score": 100.0, "mean_raw": 0.0 }, "mimo": { "mean_score": 100.0, "mean_raw": 0.0 }, "mimo_stt": { "mean_score": 96.0, "mean_raw": 4.0 }, "moss_audio": { "mean_score": 99.0, "mean_raw": 1.0 }, "qwen3_oss": { "mean_score": 100.0, "mean_raw": 0.0 }, "qwen_audio_3_1_asr": { "mean_score": 100.0, "mean_raw": 0.0 }, "r2t2": { "mean_score": 99.0, "mean_raw": 1.0 }, "soniox": { "mean_score": 97.0, "mean_raw": 3.0 }, "stepfun": { "mean_score": 100.0, "mean_raw": 0.0 }, "vibevoice": { "mean_score": 84.0, "mean_raw": 16.0 }, "voxtral": { "mean_score": 81.0, "mean_raw": 19.0 }, "whisper": { "mean_score": 100.0, "mean_raw": 0.0 }, "whisper_v3_turbo": { "mean_score": 91.0, "mean_raw": 9.0 } } }, "70": { "duration_s": 17, "model_scores": { "apple_speechanalyzer": { "mean_score": 89.0, "mean_raw": 11.0 }, "doubao": { "mean_score": 90.0, "mean_raw": 10.0 }, "elevenlabs": { "mean_score": 59.0, "mean_raw": 41.0 }, "firered": { "mean_score": 56.0, "mean_raw": 44.0 }, "firered_aed": { "mean_score": 90.0, "mean_raw": 10.0 }, "flash": { "mean_score": 64.0, "mean_raw": 36.0 }, "funasr_nano": { "mean_score": 55.0, "mean_raw": 45.0 }, "gemini_3_5_flash_lite_stt": { "mean_score": 50.0, "mean_raw": 50.0 }, "gemini_3_5_flash_stt": { "mean_score": 89.0, "mean_raw": 11.0 }, "gemini_3_5_transcribe": { "mean_score": 85.0, "mean_raw": 15.0 }, "gemini_3_6_flash_stt": { "mean_score": 80.0, "mean_raw": 20.0 }, "gemini_3_7_flash_stt": { "mean_score": 98.0, "mean_raw": 2.0 }, "gemini_3_8_flash_stt": { "mean_score": 90.0, "mean_raw": 10.0 }, "gemini_flash_stt": { "mean_score": 97.0, "mean_raw": 3.0 }, "gemini_pro_stt": { "mean_score": 91.0, "mean_raw": 9.0 }, "gemini_stt": { "mean_score": 83.0, "mean_raw": 17.0 }, "glm_asr_nano": { "mean_score": 64.0, "mean_raw": 36.0 }, "gpt_transcribe": { "mean_score": 98.0, "mean_raw": 2.0 }, "mai_transcribe_2": { "mean_score": 99.0, "mean_raw": 1.0 }, "mimo": { "mean_score": 99.0, "mean_raw": 1.0 }, "mimo_stt": { "mean_score": 62.0, "mean_raw": 38.0 }, "moss_audio": { "mean_score": 55.0, "mean_raw": 45.0 }, "qwen3_oss": { "mean_score": 56.0, "mean_raw": 44.0 }, "qwen_audio_3_1_asr": { "mean_score": 56.0, "mean_raw": 44.0 }, "r2t2": { "mean_score": 56.0, "mean_raw": 44.0 }, "soniox": { "mean_score": 99.0, "mean_raw": 1.0 }, "stepfun": { "mean_score": 22.0, "mean_raw": 78.0 }, "vibevoice": { "mean_score": 98.0, "mean_raw": 2.0 }, "voxtral": { "mean_score": 89.0, "mean_raw": 11.0 }, "whisper": { "mean_score": 90.0, "mean_raw": 10.0 }, "whisper_v3_turbo": { "mean_score": 93.0, "mean_raw": 7.0 } } }, "71": { "duration_s": 51, "model_scores": { "apple_speechanalyzer": { "mean_score": 59.0, "mean_raw": 41.0 }, "doubao": { "mean_score": 98.0, "mean_raw": 2.0 }, "elevenlabs": { "mean_score": 92.0, "mean_raw": 8.0 }, "firered": { "mean_score": 52.0, "mean_raw": 48.0 }, "firered_aed": { "mean_score": 84.0, "mean_raw": 16.0 }, "flash": { "mean_score": 96.0, "mean_raw": 4.0 }, "funasr_nano": { "mean_score": 88.0, "mean_raw": 12.0 }, "gemini_3_5_flash_lite_stt": { "mean_score": 49.0, "mean_raw": 51.0 }, "gemini_3_5_flash_stt": { "mean_score": 85.0, "mean_raw": 15.0 }, "gemini_3_5_transcribe": { "mean_score": 95.0, "mean_raw": 5.0 }, "gemini_3_6_flash_stt": { "mean_score": 77.0, "mean_raw": 23.0 }, "gemini_3_7_flash_stt": { "mean_score": 94.0, "mean_raw": 6.0 }, "gemini_3_8_flash_stt": { "mean_score": 96.0, "mean_raw": 4.0 }, "gemini_flash_stt": { "mean_score": 88.0, "mean_raw": 12.0 }, "gemini_pro_stt": { "mean_score": 88.0, "mean_raw": 12.0 }, "gemini_stt": { "mean_score": 78.0, "mean_raw": 22.0 }, "glm_asr_nano": { "mean_score": 79.0, "mean_raw": 21.0 }, "gpt_transcribe": { "mean_score": 77.0, "mean_raw": 23.0 }, "mai_transcribe_2": { "mean_score": 97.0, "mean_raw": 3.0 }, "mimo": { "mean_score": 95.0, "mean_raw": 5.0 }, "mimo_stt": { "mean_score": 94.0, "mean_raw": 6.0 }, "moss_audio": { "mean_score": 60.0, "mean_raw": 40.0 }, "qwen3_oss": { "mean_score": 77.0, "mean_raw": 23.0 }, "qwen_audio_3_1_asr": { "mean_score": 96.0, "mean_raw": 4.0 }, "r2t2": { "mean_score": 95.0, "mean_raw": 5.0 }, "soniox": { "mean_score": 90.0, "mean_raw": 10.0 }, "stepfun": { "mean_score": 72.0, "mean_raw": 28.0 }, "vibevoice": { "mean_score": 96.0, "mean_raw": 4.0 }, "voxtral": { "mean_score": 64.0, "mean_raw": 36.0 }, "whisper": { "mean_score": 75.0, "mean_raw": 25.0 }, "whisper_v3_turbo": { "mean_score": 51.0, "mean_raw": 49.0 } } }, "72": { "duration_s": 54, "model_scores": { "apple_speechanalyzer": { "mean_score": 86.0, "mean_raw": 14.0 }, "doubao": { "mean_score": 94.0, "mean_raw": 6.0 }, "elevenlabs": { "mean_score": 99.0, "mean_raw": 1.0 }, "firered": { "mean_score": 79.0, "mean_raw": 21.0 }, "firered_aed": { "mean_score": 91.0, "mean_raw": 9.0 }, "flash": { "mean_score": 100.0, "mean_raw": 0.0 }, "funasr_nano": { "mean_score": 92.0, "mean_raw": 8.0 }, "gemini_3_5_flash_lite_stt": { "mean_score": 88.0, "mean_raw": 12.0 }, "gemini_3_5_flash_stt": { "mean_score": 89.0, "mean_raw": 11.0 }, "gemini_3_5_transcribe": { "mean_score": 98.0, "mean_raw": 2.0 }, "gemini_3_6_flash_stt": { "mean_score": 85.0, "mean_raw": 15.0 }, "gemini_3_7_flash_stt": { "mean_score": 92.0, "mean_raw": 8.0 }, "gemini_3_8_flash_stt": { "mean_score": 97.0, "mean_raw": 3.0 }, "gemini_flash_stt": { "mean_score": 97.0, "mean_raw": 3.0 }, "gemini_pro_stt": { "mean_score": 91.0, "mean_raw": 9.0 }, "gemini_stt": { "mean_score": 87.0, "mean_raw": 13.0 }, "glm_asr_nano": { "mean_score": 95.0, "mean_raw": 5.0 }, "gpt_transcribe": { "mean_score": 80.0, "mean_raw": 20.0 }, "mai_transcribe_2": { "mean_score": 99.0, "mean_raw": 1.0 }, "mimo": { "mean_score": 99.0, "mean_raw": 1.0 }, "mimo_stt": { "mean_score": 93.0, "mean_raw": 7.0 }, "moss_audio": { "mean_score": 42.0, "mean_raw": 58.0 }, "qwen3_oss": { "mean_score": 91.0, "mean_raw": 9.0 }, "qwen_audio_3_1_asr": { "mean_score": 98.0, "mean_raw": 2.0 }, "r2t2": { "mean_score": 91.0, "mean_raw": 9.0 }, "soniox": { "mean_score": 99.0, "mean_raw": 1.0 }, "stepfun": { "mean_score": 93.0, "mean_raw": 7.0 }, "vibevoice": { "mean_score": 92.0, "mean_raw": 8.0 }, "voxtral": { "mean_score": 89.0, "mean_raw": 11.0 }, "whisper": { "mean_score": 94.0, "mean_raw": 6.0 }, "whisper_v3_turbo": { "mean_score": 90.0, "mean_raw": 10.0 } } }, "73": { "duration_s": 56, "model_scores": { "apple_speechanalyzer": { "mean_score": 49.0, "mean_raw": 51.0 }, "doubao": { "mean_score": 93.0, "mean_raw": 7.0 }, "elevenlabs": { "mean_score": 90.0, "mean_raw": 10.0 }, "firered": { "mean_score": 58.0, "mean_raw": 42.0 }, "firered_aed": { "mean_score": 90.0, "mean_raw": 10.0 }, "flash": { "mean_score": 97.0, "mean_raw": 3.0 }, "funasr_nano": { "mean_score": 86.0, "mean_raw": 14.0 }, "gemini_3_5_flash_lite_stt": { "mean_score": 61.0, "mean_raw": 39.0 }, "gemini_3_5_flash_stt": { "mean_score": 68.0, "mean_raw": 32.0 }, "gemini_3_5_transcribe": { "mean_score": 81.0, "mean_raw": 19.0 }, "gemini_3_6_flash_stt": { "mean_score": 71.0, "mean_raw": 29.0 }, "gemini_3_7_flash_stt": { "mean_score": 79.0, "mean_raw": 21.0 }, "gemini_3_8_flash_stt": { "mean_score": 80.0, "mean_raw": 20.0 }, "gemini_flash_stt": { "mean_score": 86.0, "mean_raw": 14.0 }, "gemini_pro_stt": { "mean_score": 93.0, "mean_raw": 7.0 }, "gemini_stt": { "mean_score": 67.0, "mean_raw": 33.0 }, "glm_asr_nano": { "mean_score": 89.0, "mean_raw": 11.0 }, "gpt_transcribe": { "mean_score": 87.0, "mean_raw": 13.0 }, "mai_transcribe_2": { "mean_score": 85.0, "mean_raw": 15.0 }, "mimo": { "mean_score": 87.0, "mean_raw": 13.0 }, "mimo_stt": { "mean_score": 88.0, "mean_raw": 12.0 }, "moss_audio": { "mean_score": 59.0, "mean_raw": 41.0 }, "qwen3_oss": { "mean_score": 97.0, "mean_raw": 3.0 }, "qwen_audio_3_1_asr": { "mean_score": 99.0, "mean_raw": 1.0 }, "r2t2": { "mean_score": 91.0, "mean_raw": 9.0 }, "soniox": { "mean_score": 81.0, "mean_raw": 19.0 }, "stepfun": { "mean_score": 84.0, "mean_raw": 16.0 }, "vibevoice": { "mean_score": 100.0, "mean_raw": 0.0 }, "voxtral": { "mean_score": 100.0, "mean_raw": 0.0 }, "whisper": { "mean_score": 85.0, "mean_raw": 15.0 }, "whisper_v3_turbo": { "mean_score": 85.0, "mean_raw": 15.0 } } }, "74": { "duration_s": 45, "model_scores": { "apple_speechanalyzer": { "mean_score": 89.0, "mean_raw": 11.0 }, "doubao": { "mean_score": 95.0, "mean_raw": 5.0 }, "elevenlabs": { "mean_score": 99.0, "mean_raw": 1.0 }, "firered": { "mean_score": 80.0, "mean_raw": 20.0 }, "firered_aed": { "mean_score": 97.0, "mean_raw": 3.0 }, "flash": { "mean_score": 97.0, "mean_raw": 3.0 }, "funasr_nano": { "mean_score": 97.0, "mean_raw": 3.0 }, "gemini_3_5_flash_lite_stt": { "mean_score": 90.0, "mean_raw": 10.0 }, "gemini_3_5_flash_stt": { "mean_score": 95.0, "mean_raw": 5.0 }, "gemini_3_5_transcribe": { "mean_score": 95.0, "mean_raw": 5.0 }, "gemini_3_6_flash_stt": { "mean_score": 96.0, "mean_raw": 4.0 }, "gemini_3_7_flash_stt": { "mean_score": 95.0, "mean_raw": 5.0 }, "gemini_3_8_flash_stt": { "mean_score": 97.0, "mean_raw": 3.0 }, "gemini_flash_stt": { "mean_score": 95.0, "mean_raw": 5.0 }, "gemini_pro_stt": { "mean_score": 97.0, "mean_raw": 3.0 }, "gemini_stt": { "mean_score": 99.0, "mean_raw": 1.0 }, "glm_asr_nano": { "mean_score": 100.0, "mean_raw": 0.0 }, "gpt_transcribe": { "mean_score": 97.0, "mean_raw": 3.0 }, "mai_transcribe_2": { "mean_score": 97.0, "mean_raw": 3.0 }, "mimo": { "mean_score": 98.0, "mean_raw": 2.0 }, "mimo_stt": { "mean_score": 99.0, "mean_raw": 1.0 }, "moss_audio": { "mean_score": 75.0, "mean_raw": 25.0 }, "qwen3_oss": { "mean_score": 93.0, "mean_raw": 7.0 }, "qwen_audio_3_1_asr": { "mean_score": 96.0, "mean_raw": 4.0 }, "r2t2": { "mean_score": 97.0, "mean_raw": 3.0 }, "soniox": { "mean_score": 98.0, "mean_raw": 2.0 }, "stepfun": { "mean_score": 95.0, "mean_raw": 5.0 }, "vibevoice": { "mean_score": 96.0, "mean_raw": 4.0 }, "voxtral": { "mean_score": 95.0, "mean_raw": 5.0 }, "whisper": { "mean_score": 96.0, "mean_raw": 4.0 }, "whisper_v3_turbo": { "mean_score": 95.0, "mean_raw": 5.0 } } }, "75": { "duration_s": 74, "model_scores": { "apple_speechanalyzer": { "mean_score": 73.0, "mean_raw": 27.0 }, "doubao": { "mean_score": 87.0, "mean_raw": 13.0 }, "elevenlabs": { "mean_score": 95.0, "mean_raw": 5.0 }, "firered": { "mean_score": 74.0, "mean_raw": 26.0 }, "firered_aed": { "mean_score": 83.0, "mean_raw": 17.0 }, "flash": { "mean_score": 95.0, "mean_raw": 5.0 }, "funasr_nano": { "mean_score": 87.0, "mean_raw": 13.0 }, "gemini_3_5_flash_lite_stt": { "mean_score": 79.0, "mean_raw": 21.0 }, "gemini_3_5_flash_stt": { "mean_score": 91.0, "mean_raw": 9.0 }, "gemini_3_5_transcribe": { "mean_score": 90.0, "mean_raw": 10.0 }, "gemini_3_6_flash_stt": { "mean_score": 89.0, "mean_raw": 11.0 }, "gemini_3_7_flash_stt": { "mean_score": 87.0, "mean_raw": 13.0 }, "gemini_flash_stt": { "mean_score": 86.0, "mean_raw": 14.0 }, "gemini_pro_stt": { "mean_score": 94.0, "mean_raw": 6.0 }, "gemini_stt": { "mean_score": 69.0, "mean_raw": 31.0 }, "glm_asr_nano": { "mean_score": 91.0, "mean_raw": 9.0 }, "gpt_transcribe": { "mean_score": 93.0, "mean_raw": 7.0 }, "mai_transcribe_2": { "mean_score": 98.0, "mean_raw": 2.0 }, "mimo": { "mean_score": 96.0, "mean_raw": 4.0 }, "mimo_stt": { "mean_score": 67.0, "mean_raw": 33.0 }, "moss_audio": { "mean_score": 78.0, "mean_raw": 22.0 }, "qwen3_oss": { "mean_score": 95.0, "mean_raw": 5.0 }, "qwen_audio_3_1_asr": { "mean_score": 92.0, "mean_raw": 8.0 }, "r2t2": { "mean_score": 96.0, "mean_raw": 4.0 }, "soniox": { "mean_score": 89.0, "mean_raw": 11.0 }, "stepfun": { "mean_score": 95.0, "mean_raw": 5.0 }, "vibevoice": { "mean_score": 96.0, "mean_raw": 4.0 }, "voxtral": { "mean_score": 82.0, "mean_raw": 18.0 }, "whisper": { "mean_score": 95.0, "mean_raw": 5.0 } } }, "76": { "duration_s": 58, "model_scores": { "apple_speechanalyzer": { "mean_score": 78.0, "mean_raw": 22.0 }, "doubao": { "mean_score": 78.0, "mean_raw": 22.0 }, "elevenlabs": { "mean_score": 76.0, "mean_raw": 24.0 }, "firered": { "mean_score": 51.0, "mean_raw": 49.0 }, "firered_aed": { "mean_score": 64.0, "mean_raw": 36.0 }, "flash": { "mean_score": 79.0, "mean_raw": 21.0 }, "funasr_nano": { "mean_score": 72.0, "mean_raw": 28.0 }, "gemini_3_5_flash_lite_stt": { "mean_score": 66.0, "mean_raw": 34.0 }, "gemini_3_5_flash_stt": { "mean_score": 95.0, "mean_raw": 5.0 }, "gemini_3_5_transcribe": { "mean_score": 71.0, "mean_raw": 29.0 }, "gemini_3_6_flash_stt": { "mean_score": 77.0, "mean_raw": 23.0 }, "gemini_3_7_flash_stt": { "mean_score": 89.0, "mean_raw": 11.0 }, "gemini_3_8_flash_stt": { "mean_score": 97.0, "mean_raw": 3.0 }, "gemini_flash_stt": { "mean_score": 73.0, "mean_raw": 27.0 }, "gemini_pro_stt": { "mean_score": 94.0, "mean_raw": 6.0 }, "gemini_stt": { "mean_score": 64.0, "mean_raw": 36.0 }, "glm_asr_nano": { "mean_score": 71.0, "mean_raw": 29.0 }, "gpt_transcribe": { "mean_score": 68.0, "mean_raw": 32.0 }, "mai_transcribe_2": { "mean_score": 78.0, "mean_raw": 22.0 }, "mimo": { "mean_score": 94.0, "mean_raw": 6.0 }, "mimo_stt": { "mean_score": 77.0, "mean_raw": 23.0 }, "moss_audio": { "mean_score": 47.0, "mean_raw": 53.0 }, "qwen3_oss": { "mean_score": 68.0, "mean_raw": 32.0 }, "qwen_audio_3_1_asr": { "mean_score": 77.0, "mean_raw": 23.0 }, "r2t2": { "mean_score": 71.0, "mean_raw": 29.0 }, "soniox": { "mean_score": 94.0, "mean_raw": 6.0 }, "stepfun": { "mean_score": 76.0, "mean_raw": 24.0 }, "vibevoice": { "mean_score": 77.0, "mean_raw": 23.0 }, "voxtral": { "mean_score": 47.0, "mean_raw": 53.0 }, "whisper": { "mean_score": 69.0, "mean_raw": 31.0 }, "whisper_v3_turbo": { "mean_score": 47.0, "mean_raw": 53.0 } } }, "77": { "duration_s": 130, "model_scores": { "apple_speechanalyzer": { "mean_score": 75.0, "mean_raw": 25.0 }, "doubao": { "mean_score": 60.0, "mean_raw": 40.0 }, "elevenlabs": { "mean_score": 68.0, "mean_raw": 32.0 }, "firered": { "mean_score": 12.0, "mean_raw": 88.0 }, "firered_aed": { "mean_score": 72.0, "mean_raw": 28.0 }, "flash": { "mean_score": 60.0, "mean_raw": 40.0 }, "funasr_nano": { "mean_score": 62.0, "mean_raw": 38.0 }, "gemini_3_5_flash_lite_stt": { "mean_score": 49.0, "mean_raw": 51.0 }, "gemini_3_5_flash_stt": { "mean_score": 62.0, "mean_raw": 38.0 }, "gemini_3_5_transcribe": { "mean_score": 55.0, "mean_raw": 45.0 }, "gemini_3_6_flash_stt": { "mean_score": 49.0, "mean_raw": 51.0 }, "gemini_3_7_flash_stt": { "mean_score": 63.0, "mean_raw": 37.0 }, "gemini_3_8_flash_stt": { "mean_score": 85.0, "mean_raw": 15.0 }, "gemini_flash_stt": { "mean_score": 58.0, "mean_raw": 42.0 }, "gemini_pro_stt": { "mean_score": 60.0, "mean_raw": 40.0 }, "gemini_stt": { "mean_score": 62.0, "mean_raw": 38.0 }, "glm_asr_nano": { "mean_score": 83.0, "mean_raw": 17.0 }, "gpt_transcribe": { "mean_score": 65.0, "mean_raw": 35.0 }, "mai_transcribe_2": { "mean_score": 70.0, "mean_raw": 30.0 }, "mimo": { "mean_score": 65.0, "mean_raw": 35.0 }, "mimo_stt": { "mean_score": 83.0, "mean_raw": 17.0 }, "moss_audio": { "mean_score": 72.0, "mean_raw": 28.0 }, "qwen3_oss": { "mean_score": 52.0, "mean_raw": 48.0 }, "qwen_audio_3_1_asr": { "mean_score": 60.0, "mean_raw": 40.0 }, "r2t2": { "mean_score": 81.0, "mean_raw": 19.0 }, "soniox": { "mean_score": 89.0, "mean_raw": 11.0 }, "stepfun": { "mean_score": 59.0, "mean_raw": 41.0 }, "vibevoice": { "mean_score": 60.0, "mean_raw": 40.0 }, "voxtral": { "mean_score": 58.0, "mean_raw": 42.0 }, "whisper": { "mean_score": 64.0, "mean_raw": 36.0 }, "whisper_v3_turbo": { "mean_score": 65.0, "mean_raw": 35.0 } } }, "78": { "duration_s": 61, "model_scores": { "apple_speechanalyzer": { "mean_score": 98.0, "mean_raw": 2.0 }, "doubao": { "mean_score": 100.0, "mean_raw": 0.0 }, "elevenlabs": { "mean_score": 98.0, "mean_raw": 2.0 }, "firered": { "mean_score": 95.0, "mean_raw": 5.0 }, "firered_aed": { "mean_score": 89.0, "mean_raw": 11.0 }, "flash": { "mean_score": 97.0, "mean_raw": 3.0 }, "funasr_nano": { "mean_score": 90.0, "mean_raw": 10.0 }, "gemini_3_5_flash_lite_stt": { "mean_score": 90.0, "mean_raw": 10.0 }, "gemini_3_5_flash_stt": { "mean_score": 91.0, "mean_raw": 9.0 }, "gemini_3_5_transcribe": { "mean_score": 98.0, "mean_raw": 2.0 }, "gemini_3_6_flash_stt": { "mean_score": 88.0, "mean_raw": 12.0 }, "gemini_3_7_flash_stt": { "mean_score": 93.0, "mean_raw": 7.0 }, "gemini_3_8_flash_stt": { "mean_score": 76.0, "mean_raw": 24.0 }, "gemini_flash_stt": { "mean_score": 95.0, "mean_raw": 5.0 }, "gemini_pro_stt": { "mean_score": 96.0, "mean_raw": 4.0 }, "gemini_stt": { "mean_score": 90.0, "mean_raw": 10.0 }, "glm_asr_nano": { "mean_score": 97.0, "mean_raw": 3.0 }, "gpt_transcribe": { "mean_score": 96.0, "mean_raw": 4.0 }, "mai_transcribe_2": { "mean_score": 98.0, "mean_raw": 2.0 }, "mimo": { "mean_score": 96.0, "mean_raw": 4.0 }, "mimo_stt": { "mean_score": 96.0, "mean_raw": 4.0 }, "moss_audio": { "mean_score": 50.0, "mean_raw": 50.0 }, "qwen3_oss": { "mean_score": 96.0, "mean_raw": 4.0 }, "qwen_audio_3_1_asr": { "mean_score": 93.0, "mean_raw": 7.0 }, "r2t2": { "mean_score": 96.0, "mean_raw": 4.0 }, "soniox": { "mean_score": 96.0, "mean_raw": 4.0 }, "stepfun": { "mean_score": 94.0, "mean_raw": 6.0 }, "vibevoice": { "mean_score": 94.0, "mean_raw": 6.0 }, "voxtral": { "mean_score": 95.0, "mean_raw": 5.0 }, "whisper": { "mean_score": 95.0, "mean_raw": 5.0 }, "whisper_v3_turbo": { "mean_score": 95.0, "mean_raw": 5.0 } } }, "79": { "duration_s": 47, "model_scores": { "apple_speechanalyzer": { "mean_score": 84.0, "mean_raw": 16.0 }, "doubao": { "mean_score": 94.0, "mean_raw": 6.0 }, "elevenlabs": { "mean_score": 98.0, "mean_raw": 2.0 }, "firered": { "mean_score": 97.0, "mean_raw": 3.0 }, "firered_aed": { "mean_score": 94.0, "mean_raw": 6.0 }, "flash": { "mean_score": 96.0, "mean_raw": 4.0 }, "funasr_nano": { "mean_score": 95.0, "mean_raw": 5.0 }, "gemini_3_5_flash_lite_stt": { "mean_score": 91.0, "mean_raw": 9.0 }, "gemini_3_5_flash_stt": { "mean_score": 85.0, "mean_raw": 15.0 }, "gemini_3_5_transcribe": { "mean_score": 95.0, "mean_raw": 5.0 }, "gemini_3_6_flash_stt": { "mean_score": 88.0, "mean_raw": 12.0 }, "gemini_3_7_flash_stt": { "mean_score": 94.0, "mean_raw": 6.0 }, "gemini_3_8_flash_stt": { "mean_score": 92.0, "mean_raw": 8.0 }, "gemini_flash_stt": { "mean_score": 84.0, "mean_raw": 16.0 }, "gemini_pro_stt": { "mean_score": 94.0, "mean_raw": 6.0 }, "gemini_stt": { "mean_score": 83.0, "mean_raw": 17.0 }, "glm_asr_nano": { "mean_score": 95.0, "mean_raw": 5.0 }, "gpt_transcribe": { "mean_score": 97.0, "mean_raw": 3.0 }, "mai_transcribe_2": { "mean_score": 97.0, "mean_raw": 3.0 }, "mimo": { "mean_score": 97.0, "mean_raw": 3.0 }, "mimo_stt": { "mean_score": 91.0, "mean_raw": 9.0 }, "moss_audio": { "mean_score": 56.0, "mean_raw": 44.0 }, "qwen3_oss": { "mean_score": 94.0, "mean_raw": 6.0 }, "qwen_audio_3_1_asr": { "mean_score": 96.0, "mean_raw": 4.0 }, "r2t2": { "mean_score": 97.0, "mean_raw": 3.0 }, "soniox": { "mean_score": 99.0, "mean_raw": 1.0 }, "stepfun": { "mean_score": 95.0, "mean_raw": 5.0 }, "vibevoice": { "mean_score": 97.0, "mean_raw": 3.0 }, "voxtral": { "mean_score": 92.0, "mean_raw": 8.0 }, "whisper": { "mean_score": 91.0, "mean_raw": 9.0 }, "whisper_v3_turbo": { "mean_score": 95.0, "mean_raw": 5.0 } } }, "80": { "duration_s": 42, "model_scores": { "apple_speechanalyzer": { "mean_score": 95.0, "mean_raw": 5.0 }, "doubao": { "mean_score": 97.0, "mean_raw": 3.0 }, "elevenlabs": { "mean_score": 88.0, "mean_raw": 12.0 }, "firered": { "mean_score": 93.0, "mean_raw": 7.0 }, "firered_aed": { "mean_score": 98.0, "mean_raw": 2.0 }, "flash": { "mean_score": 97.0, "mean_raw": 3.0 }, "funasr_nano": { "mean_score": 98.0, "mean_raw": 2.0 }, "gemini_3_5_flash_lite_stt": { "mean_score": 72.0, "mean_raw": 28.0 }, "gemini_3_5_flash_stt": { "mean_score": 98.0, "mean_raw": 2.0 }, "gemini_3_5_transcribe": { "mean_score": 97.0, "mean_raw": 3.0 }, "gemini_3_6_flash_stt": { "mean_score": 90.0, "mean_raw": 10.0 }, "gemini_3_7_flash_stt": { "mean_score": 97.0, "mean_raw": 3.0 }, "gemini_3_8_flash_stt": { "mean_score": 95.0, "mean_raw": 5.0 }, "gemini_flash_stt": { "mean_score": 97.0, "mean_raw": 3.0 }, "gemini_pro_stt": { "mean_score": 89.0, "mean_raw": 11.0 }, "gemini_stt": { "mean_score": 98.0, "mean_raw": 2.0 }, "glm_asr_nano": { "mean_score": 98.0, "mean_raw": 2.0 }, "gpt_transcribe": { "mean_score": 98.0, "mean_raw": 2.0 }, "mai_transcribe_2": { "mean_score": 97.0, "mean_raw": 3.0 }, "mimo": { "mean_score": 97.0, "mean_raw": 3.0 }, "mimo_stt": { "mean_score": 97.0, "mean_raw": 3.0 }, "moss_audio": { "mean_score": 82.0, "mean_raw": 18.0 }, "qwen3_oss": { "mean_score": 97.0, "mean_raw": 3.0 }, "qwen_audio_3_1_asr": { "mean_score": 97.0, "mean_raw": 3.0 }, "r2t2": { "mean_score": 97.0, "mean_raw": 3.0 }, "soniox": { "mean_score": 97.0, "mean_raw": 3.0 }, "stepfun": { "mean_score": 97.0, "mean_raw": 3.0 }, "vibevoice": { "mean_score": 98.0, "mean_raw": 2.0 }, "voxtral": { "mean_score": 96.0, "mean_raw": 4.0 }, "whisper": { "mean_score": 98.0, "mean_raw": 2.0 }, "whisper_v3_turbo": { "mean_score": 85.0, "mean_raw": 15.0 } } }, "81": { "duration_s": 38, "model_scores": { "apple_speechanalyzer": { "mean_score": 80.0, "mean_raw": 20.0 }, "doubao": { "mean_score": 96.0, "mean_raw": 4.0 }, "elevenlabs": { "mean_score": 92.0, "mean_raw": 8.0 }, "firered": { "mean_score": 92.0, "mean_raw": 8.0 }, "firered_aed": { "mean_score": 89.0, "mean_raw": 11.0 }, "flash": { "mean_score": 90.0, "mean_raw": 10.0 }, "funasr_nano": { "mean_score": 94.0, "mean_raw": 6.0 }, "gemini_3_5_flash_lite_stt": { "mean_score": 80.0, "mean_raw": 20.0 }, "gemini_3_5_flash_stt": { "mean_score": 87.0, "mean_raw": 13.0 }, "gemini_3_5_transcribe": { "mean_score": 94.0, "mean_raw": 6.0 }, "gemini_3_6_flash_stt": { "mean_score": 92.0, "mean_raw": 8.0 }, "gemini_3_7_flash_stt": { "mean_score": 97.0, "mean_raw": 3.0 }, "gemini_3_8_flash_stt": { "mean_score": 94.0, "mean_raw": 6.0 }, "gemini_flash_stt": { "mean_score": 94.0, "mean_raw": 6.0 }, "gemini_pro_stt": { "mean_score": 92.0, "mean_raw": 8.0 }, "gemini_stt": { "mean_score": 91.0, "mean_raw": 9.0 }, "glm_asr_nano": { "mean_score": 76.0, "mean_raw": 24.0 }, "gpt_transcribe": { "mean_score": 91.0, "mean_raw": 9.0 }, "mai_transcribe_2": { "mean_score": 97.0, "mean_raw": 3.0 }, "mimo": { "mean_score": 97.0, "mean_raw": 3.0 }, "mimo_stt": { "mean_score": 93.0, "mean_raw": 7.0 }, "moss_audio": { "mean_score": 54.0, "mean_raw": 46.0 }, "qwen3_oss": { "mean_score": 92.0, "mean_raw": 8.0 }, "qwen_audio_3_1_asr": { "mean_score": 94.0, "mean_raw": 6.0 }, "r2t2": { "mean_score": 93.0, "mean_raw": 7.0 }, "soniox": { "mean_score": 92.0, "mean_raw": 8.0 }, "stepfun": { "mean_score": 87.0, "mean_raw": 13.0 }, "vibevoice": { "mean_score": 85.0, "mean_raw": 15.0 }, "voxtral": { "mean_score": 76.0, "mean_raw": 24.0 }, "whisper": { "mean_score": 96.0, "mean_raw": 4.0 }, "whisper_v3_turbo": { "mean_score": 75.0, "mean_raw": 25.0 } } }, "82": { "duration_s": 38, "model_scores": { "apple_speechanalyzer": { "mean_score": 83.0, "mean_raw": 17.0 }, "doubao": { "mean_score": 81.0, "mean_raw": 19.0 }, "elevenlabs": { "mean_score": 85.0, "mean_raw": 15.0 }, "firered": { "mean_score": 85.0, "mean_raw": 15.0 }, "firered_aed": { "mean_score": 94.0, "mean_raw": 6.0 }, "flash": { "mean_score": 97.0, "mean_raw": 3.0 }, "funasr_nano": { "mean_score": 81.0, "mean_raw": 19.0 }, "gemini_3_5_flash_lite_stt": { "mean_score": 95.0, "mean_raw": 5.0 }, "gemini_3_5_flash_stt": { "mean_score": 99.0, "mean_raw": 1.0 }, "gemini_3_5_transcribe": { "mean_score": 96.0, "mean_raw": 4.0 }, "gemini_3_6_flash_stt": { "mean_score": 90.0, "mean_raw": 10.0 }, "gemini_3_7_flash_stt": { "mean_score": 98.0, "mean_raw": 2.0 }, "gemini_3_8_flash_stt": { "mean_score": 94.0, "mean_raw": 6.0 }, "gemini_flash_stt": { "mean_score": 95.0, "mean_raw": 5.0 }, "gemini_pro_stt": { "mean_score": 99.0, "mean_raw": 1.0 }, "gemini_stt": { "mean_score": 92.0, "mean_raw": 8.0 }, "glm_asr_nano": { "mean_score": 75.0, "mean_raw": 25.0 }, "gpt_transcribe": { "mean_score": 85.0, "mean_raw": 15.0 }, "mai_transcribe_2": { "mean_score": 99.0, "mean_raw": 1.0 }, "mimo": { "mean_score": 90.0, "mean_raw": 10.0 }, "mimo_stt": { "mean_score": 91.0, "mean_raw": 9.0 }, "moss_audio": { "mean_score": 59.0, "mean_raw": 41.0 }, "qwen3_oss": { "mean_score": 97.0, "mean_raw": 3.0 }, "qwen_audio_3_1_asr": { "mean_score": 90.0, "mean_raw": 10.0 }, "r2t2": { "mean_score": 91.0, "mean_raw": 9.0 }, "soniox": { "mean_score": 90.0, "mean_raw": 10.0 }, "stepfun": { "mean_score": 93.0, "mean_raw": 7.0 }, "vibevoice": { "mean_score": 86.0, "mean_raw": 14.0 }, "voxtral": { "mean_score": 88.0, "mean_raw": 12.0 }, "whisper": { "mean_score": 85.0, "mean_raw": 15.0 }, "whisper_v3_turbo": { "mean_score": 83.0, "mean_raw": 17.0 } } }, "83": { "duration_s": 37, "model_scores": { "apple_speechanalyzer": { "mean_score": 90.0, "mean_raw": 10.0 }, "doubao": { "mean_score": 98.0, "mean_raw": 2.0 }, "elevenlabs": { "mean_score": 93.0, "mean_raw": 7.0 }, "firered": { "mean_score": 96.0, "mean_raw": 4.0 }, "firered_aed": { "mean_score": 96.0, "mean_raw": 4.0 }, "flash": { "mean_score": 97.0, "mean_raw": 3.0 }, "funasr_nano": { "mean_score": 93.0, "mean_raw": 7.0 }, "gemini_3_5_flash_lite_stt": { "mean_score": 90.0, "mean_raw": 10.0 }, "gemini_3_5_flash_stt": { "mean_score": 94.0, "mean_raw": 6.0 }, "gemini_3_5_transcribe": { "mean_score": 96.0, "mean_raw": 4.0 }, "gemini_3_6_flash_stt": { "mean_score": 93.0, "mean_raw": 7.0 }, "gemini_3_7_flash_stt": { "mean_score": 91.0, "mean_raw": 9.0 }, "gemini_3_8_flash_stt": { "mean_score": 100.0, "mean_raw": 0.0 }, "gemini_flash_stt": { "mean_score": 85.0, "mean_raw": 15.0 }, "gemini_pro_stt": { "mean_score": 90.0, "mean_raw": 10.0 }, "gemini_stt": { "mean_score": 95.0, "mean_raw": 5.0 }, "glm_asr_nano": { "mean_score": 92.0, "mean_raw": 8.0 }, "gpt_transcribe": { "mean_score": 93.0, "mean_raw": 7.0 }, "mai_transcribe_2": { "mean_score": 98.0, "mean_raw": 2.0 }, "mimo": { "mean_score": 97.0, "mean_raw": 3.0 }, "mimo_stt": { "mean_score": 93.0, "mean_raw": 7.0 }, "moss_audio": { "mean_score": 84.0, "mean_raw": 16.0 }, "qwen3_oss": { "mean_score": 94.0, "mean_raw": 6.0 }, "qwen_audio_3_1_asr": { "mean_score": 97.0, "mean_raw": 3.0 }, "r2t2": { "mean_score": 95.0, "mean_raw": 5.0 }, "soniox": { "mean_score": 96.0, "mean_raw": 4.0 }, "stepfun": { "mean_score": 96.0, "mean_raw": 4.0 }, "vibevoice": { "mean_score": 95.0, "mean_raw": 5.0 }, "voxtral": { "mean_score": 89.0, "mean_raw": 11.0 }, "whisper": { "mean_score": 97.0, "mean_raw": 3.0 }, "whisper_v3_turbo": { "mean_score": 95.0, "mean_raw": 5.0 } } }, "84": { "duration_s": 36, "model_scores": { "apple_speechanalyzer": { "mean_score": 60.0, "mean_raw": 40.0 }, "doubao": { "mean_score": 65.0, "mean_raw": 35.0 }, "elevenlabs": { "mean_score": 94.0, "mean_raw": 6.0 }, "firered": { "mean_score": 95.0, "mean_raw": 5.0 }, "firered_aed": { "mean_score": 64.0, "mean_raw": 36.0 }, "flash": { "mean_score": 62.0, "mean_raw": 38.0 }, "funasr_nano": { "mean_score": 62.0, "mean_raw": 38.0 }, "gemini_3_5_flash_lite_stt": { "mean_score": 96.0, "mean_raw": 4.0 }, "gemini_3_5_flash_stt": { "mean_score": 94.0, "mean_raw": 6.0 }, "gemini_3_5_transcribe": { "mean_score": 98.0, "mean_raw": 2.0 }, "gemini_3_6_flash_stt": { "mean_score": 95.0, "mean_raw": 5.0 }, "gemini_3_7_flash_stt": { "mean_score": 95.0, "mean_raw": 5.0 }, "gemini_3_8_flash_stt": { "mean_score": 97.0, "mean_raw": 3.0 }, "gemini_flash_stt": { "mean_score": 97.0, "mean_raw": 3.0 }, "gemini_pro_stt": { "mean_score": 95.0, "mean_raw": 5.0 }, "gemini_stt": { "mean_score": 95.0, "mean_raw": 5.0 }, "glm_asr_nano": { "mean_score": 63.0, "mean_raw": 37.0 }, "gpt_transcribe": { "mean_score": 95.0, "mean_raw": 5.0 }, "mai_transcribe_2": { "mean_score": 64.0, "mean_raw": 36.0 }, "mimo": { "mean_score": 97.0, "mean_raw": 3.0 }, "mimo_stt": { "mean_score": 63.0, "mean_raw": 37.0 }, "moss_audio": { "mean_score": 92.0, "mean_raw": 8.0 }, "qwen3_oss": { "mean_score": 96.0, "mean_raw": 4.0 }, "qwen_audio_3_1_asr": { "mean_score": 64.0, "mean_raw": 36.0 }, "r2t2": { "mean_score": 63.0, "mean_raw": 37.0 }, "soniox": { "mean_score": 97.0, "mean_raw": 3.0 }, "stepfun": { "mean_score": 96.0, "mean_raw": 4.0 }, "vibevoice": { "mean_score": 62.0, "mean_raw": 38.0 }, "voxtral": { "mean_score": 96.0, "mean_raw": 4.0 }, "whisper": { "mean_score": 95.0, "mean_raw": 5.0 }, "whisper_v3_turbo": { "mean_score": 95.0, "mean_raw": 5.0 } } }, "85": { "duration_s": 36, "model_scores": { "apple_speechanalyzer": { "mean_score": 82.0, "mean_raw": 18.0 }, "doubao": { "mean_score": 96.0, "mean_raw": 4.0 }, "elevenlabs": { "mean_score": 95.0, "mean_raw": 5.0 }, "firered": { "mean_score": 85.0, "mean_raw": 15.0 }, "firered_aed": { "mean_score": 97.0, "mean_raw": 3.0 }, "flash": { "mean_score": 95.0, "mean_raw": 5.0 }, "funasr_nano": { "mean_score": 91.0, "mean_raw": 9.0 }, "gemini_3_5_flash_lite_stt": { "mean_score": 89.0, "mean_raw": 11.0 }, "gemini_3_5_flash_stt": { "mean_score": 90.0, "mean_raw": 10.0 }, "gemini_3_5_transcribe": { "mean_score": 96.0, "mean_raw": 4.0 }, "gemini_3_6_flash_stt": { "mean_score": 90.0, "mean_raw": 10.0 }, "gemini_3_7_flash_stt": { "mean_score": 91.0, "mean_raw": 9.0 }, "gemini_3_8_flash_stt": { "mean_score": 91.0, "mean_raw": 9.0 }, "gemini_flash_stt": { "mean_score": 92.0, "mean_raw": 8.0 }, "gemini_pro_stt": { "mean_score": 97.0, "mean_raw": 3.0 }, "gemini_stt": { "mean_score": 89.0, "mean_raw": 11.0 }, "glm_asr_nano": { "mean_score": 84.0, "mean_raw": 16.0 }, "gpt_transcribe": { "mean_score": 89.0, "mean_raw": 11.0 }, "mai_transcribe_2": { "mean_score": 96.0, "mean_raw": 4.0 }, "mimo": { "mean_score": 96.0, "mean_raw": 4.0 }, "mimo_stt": { "mean_score": 91.0, "mean_raw": 9.0 }, "moss_audio": { "mean_score": 82.0, "mean_raw": 18.0 }, "qwen3_oss": { "mean_score": 97.0, "mean_raw": 3.0 }, "qwen_audio_3_1_asr": { "mean_score": 89.0, "mean_raw": 11.0 }, "r2t2": { "mean_score": 97.0, "mean_raw": 3.0 }, "soniox": { "mean_score": 93.0, "mean_raw": 7.0 }, "stepfun": { "mean_score": 92.0, "mean_raw": 8.0 }, "vibevoice": { "mean_score": 93.0, "mean_raw": 7.0 }, "voxtral": { "mean_score": 83.0, "mean_raw": 17.0 }, "whisper": { "mean_score": 90.0, "mean_raw": 10.0 }, "whisper_v3_turbo": { "mean_score": 80.0, "mean_raw": 20.0 } } }, "86": { "duration_s": 35, "model_scores": { "apple_speechanalyzer": { "mean_score": 73.0, "mean_raw": 27.0 }, "doubao": { "mean_score": 84.0, "mean_raw": 16.0 }, "elevenlabs": { "mean_score": 92.0, "mean_raw": 8.0 }, "firered": { "mean_score": 75.0, "mean_raw": 25.0 }, "firered_aed": { "mean_score": 84.0, "mean_raw": 16.0 }, "flash": { "mean_score": 91.0, "mean_raw": 9.0 }, "funasr_nano": { "mean_score": 91.0, "mean_raw": 9.0 }, "gemini_3_5_flash_lite_stt": { "mean_score": 70.0, "mean_raw": 30.0 }, "gemini_3_5_flash_stt": { "mean_score": 89.0, "mean_raw": 11.0 }, "gemini_3_5_transcribe": { "mean_score": 89.0, "mean_raw": 11.0 }, "gemini_3_6_flash_stt": { "mean_score": 88.0, "mean_raw": 12.0 }, "gemini_3_7_flash_stt": { "mean_score": 89.0, "mean_raw": 11.0 }, "gemini_3_8_flash_stt": { "mean_score": 92.0, "mean_raw": 8.0 }, "gemini_flash_stt": { "mean_score": 81.0, "mean_raw": 19.0 }, "gemini_pro_stt": { "mean_score": 98.0, "mean_raw": 2.0 }, "gemini_stt": { "mean_score": 97.0, "mean_raw": 3.0 }, "glm_asr_nano": { "mean_score": 82.0, "mean_raw": 18.0 }, "gpt_transcribe": { "mean_score": 84.0, "mean_raw": 16.0 }, "mai_transcribe_2": { "mean_score": 87.0, "mean_raw": 13.0 }, "mimo": { "mean_score": 92.0, "mean_raw": 8.0 }, "mimo_stt": { "mean_score": 88.0, "mean_raw": 12.0 }, "moss_audio": { "mean_score": 79.0, "mean_raw": 21.0 }, "qwen3_oss": { "mean_score": 81.0, "mean_raw": 19.0 }, "qwen_audio_3_1_asr": { "mean_score": 91.0, "mean_raw": 9.0 }, "r2t2": { "mean_score": 90.0, "mean_raw": 10.0 }, "soniox": { "mean_score": 88.0, "mean_raw": 12.0 }, "stepfun": { "mean_score": 82.0, "mean_raw": 18.0 }, "vibevoice": { "mean_score": 79.0, "mean_raw": 21.0 }, "voxtral": { "mean_score": 96.0, "mean_raw": 4.0 }, "whisper": { "mean_score": 79.0, "mean_raw": 21.0 }, "whisper_v3_turbo": { "mean_score": 81.0, "mean_raw": 19.0 } } }, "87": { "duration_s": 34, "model_scores": { "apple_speechanalyzer": { "mean_score": 80.0, "mean_raw": 20.0 }, "doubao": { "mean_score": 81.0, "mean_raw": 19.0 }, "elevenlabs": { "mean_score": 97.0, "mean_raw": 3.0 }, "firered": { "mean_score": 85.0, "mean_raw": 15.0 }, "firered_aed": { "mean_score": 81.0, "mean_raw": 19.0 }, "flash": { "mean_score": 81.0, "mean_raw": 19.0 }, "funasr_nano": { "mean_score": 91.0, "mean_raw": 9.0 }, "gemini_3_5_flash_lite_stt": { "mean_score": 85.0, "mean_raw": 15.0 }, "gemini_3_5_flash_stt": { "mean_score": 85.0, "mean_raw": 15.0 }, "gemini_3_5_transcribe": { "mean_score": 86.0, "mean_raw": 14.0 }, "gemini_3_6_flash_stt": { "mean_score": 81.0, "mean_raw": 19.0 }, "gemini_3_7_flash_stt": { "mean_score": 97.0, "mean_raw": 3.0 }, "gemini_3_8_flash_stt": { "mean_score": 94.0, "mean_raw": 6.0 }, "gemini_flash_stt": { "mean_score": 95.0, "mean_raw": 5.0 }, "gemini_pro_stt": { "mean_score": 95.0, "mean_raw": 5.0 }, "gemini_stt": { "mean_score": 77.0, "mean_raw": 23.0 }, "glm_asr_nano": { "mean_score": 82.0, "mean_raw": 18.0 }, "gpt_transcribe": { "mean_score": 95.0, "mean_raw": 5.0 }, "mai_transcribe_2": { "mean_score": 98.0, "mean_raw": 2.0 }, "mimo": { "mean_score": 94.0, "mean_raw": 6.0 }, "mimo_stt": { "mean_score": 96.0, "mean_raw": 4.0 }, "moss_audio": { "mean_score": 80.0, "mean_raw": 20.0 }, "qwen3_oss": { "mean_score": 91.0, "mean_raw": 9.0 }, "qwen_audio_3_1_asr": { "mean_score": 85.0, "mean_raw": 15.0 }, "r2t2": { "mean_score": 87.0, "mean_raw": 13.0 }, "soniox": { "mean_score": 71.0, "mean_raw": 29.0 }, "stepfun": { "mean_score": 82.0, "mean_raw": 18.0 }, "vibevoice": { "mean_score": 89.0, "mean_raw": 11.0 }, "voxtral": { "mean_score": 95.0, "mean_raw": 5.0 }, "whisper": { "mean_score": 83.0, "mean_raw": 17.0 }, "whisper_v3_turbo": { "mean_score": 85.0, "mean_raw": 15.0 } } }, "88": { "duration_s": 34, "model_scores": { "apple_speechanalyzer": { "mean_score": 92.0, "mean_raw": 8.0 }, "doubao": { "mean_score": 99.0, "mean_raw": 1.0 }, "elevenlabs": { "mean_score": 100.0, "mean_raw": 0.0 }, "firered": { "mean_score": 97.0, "mean_raw": 3.0 }, "firered_aed": { "mean_score": 98.0, "mean_raw": 2.0 }, "flash": { "mean_score": 100.0, "mean_raw": 0.0 }, "funasr_nano": { "mean_score": 100.0, "mean_raw": 0.0 }, "gemini_3_5_flash_lite_stt": { "mean_score": 97.0, "mean_raw": 3.0 }, "gemini_3_5_flash_stt": { "mean_score": 97.0, "mean_raw": 3.0 }, "gemini_3_5_transcribe": { "mean_score": 100.0, "mean_raw": 0.0 }, "gemini_3_6_flash_stt": { "mean_score": 83.0, "mean_raw": 17.0 }, "gemini_3_7_flash_stt": { "mean_score": 100.0, "mean_raw": 0.0 }, "gemini_3_8_flash_stt": { "mean_score": 99.0, "mean_raw": 1.0 }, "gemini_flash_stt": { "mean_score": 99.0, "mean_raw": 1.0 }, "gemini_pro_stt": { "mean_score": 99.0, "mean_raw": 1.0 }, "gemini_stt": { "mean_score": 94.0, "mean_raw": 6.0 }, "glm_asr_nano": { "mean_score": 96.0, "mean_raw": 4.0 }, "gpt_transcribe": { "mean_score": 100.0, "mean_raw": 0.0 }, "mai_transcribe_2": { "mean_score": 100.0, "mean_raw": 0.0 }, "mimo": { "mean_score": 98.0, "mean_raw": 2.0 }, "mimo_stt": { "mean_score": 98.0, "mean_raw": 2.0 }, "moss_audio": { "mean_score": 96.0, "mean_raw": 4.0 }, "qwen3_oss": { "mean_score": 100.0, "mean_raw": 0.0 }, "qwen_audio_3_1_asr": { "mean_score": 99.0, "mean_raw": 1.0 }, "r2t2": { "mean_score": 100.0, "mean_raw": 0.0 }, "soniox": { "mean_score": 95.0, "mean_raw": 5.0 }, "stepfun": { "mean_score": 99.0, "mean_raw": 1.0 }, "vibevoice": { "mean_score": 97.0, "mean_raw": 3.0 }, "voxtral": { "mean_score": 99.0, "mean_raw": 1.0 }, "whisper": { "mean_score": 99.0, "mean_raw": 1.0 }, "whisper_v3_turbo": { "mean_score": 99.0, "mean_raw": 1.0 } } }, "89": { "duration_s": 34, "model_scores": { "apple_speechanalyzer": { "mean_score": 95.0, "mean_raw": 5.0 }, "doubao": { "mean_score": 100.0, "mean_raw": 0.0 }, "elevenlabs": { "mean_score": 100.0, "mean_raw": 0.0 }, "firered": { "mean_score": 94.0, "mean_raw": 6.0 }, "firered_aed": { "mean_score": 100.0, "mean_raw": 0.0 }, "flash": { "mean_score": 100.0, "mean_raw": 0.0 }, "funasr_nano": { "mean_score": 100.0, "mean_raw": 0.0 }, "gemini_3_5_flash_lite_stt": { "mean_score": 88.0, "mean_raw": 12.0 }, "gemini_3_5_flash_stt": { "mean_score": 100.0, "mean_raw": 0.0 }, "gemini_3_5_transcribe": { "mean_score": 94.0, "mean_raw": 6.0 }, "gemini_3_6_flash_stt": { "mean_score": 99.0, "mean_raw": 1.0 }, "gemini_3_7_flash_stt": { "mean_score": 97.0, "mean_raw": 3.0 }, "gemini_3_8_flash_stt": { "mean_score": 96.0, "mean_raw": 4.0 }, "gemini_flash_stt": { "mean_score": 97.0, "mean_raw": 3.0 }, "gemini_pro_stt": { "mean_score": 100.0, "mean_raw": 0.0 }, "gemini_stt": { "mean_score": 92.0, "mean_raw": 8.0 }, "glm_asr_nano": { "mean_score": 97.0, "mean_raw": 3.0 }, "gpt_transcribe": { "mean_score": 100.0, "mean_raw": 0.0 }, "mai_transcribe_2": { "mean_score": 100.0, "mean_raw": 0.0 }, "mimo": { "mean_score": 100.0, "mean_raw": 0.0 }, "mimo_stt": { "mean_score": 99.0, "mean_raw": 1.0 }, "moss_audio": { "mean_score": 76.0, "mean_raw": 24.0 }, "qwen3_oss": { "mean_score": 100.0, "mean_raw": 0.0 }, "qwen_audio_3_1_asr": { "mean_score": 100.0, "mean_raw": 0.0 }, "r2t2": { "mean_score": 100.0, "mean_raw": 0.0 }, "soniox": { "mean_score": 100.0, "mean_raw": 0.0 }, "stepfun": { "mean_score": 100.0, "mean_raw": 0.0 }, "vibevoice": { "mean_score": 100.0, "mean_raw": 0.0 }, "voxtral": { "mean_score": 96.0, "mean_raw": 4.0 }, "whisper": { "mean_score": 100.0, "mean_raw": 0.0 }, "whisper_v3_turbo": { "mean_score": 97.0, "mean_raw": 3.0 } } }, "90": { "duration_s": 33, "model_scores": { "apple_speechanalyzer": { "mean_score": 93.0, "mean_raw": 7.0 }, "doubao": { "mean_score": 93.0, "mean_raw": 7.0 }, "elevenlabs": { "mean_score": 96.0, "mean_raw": 4.0 }, "firered": { "mean_score": 97.0, "mean_raw": 3.0 }, "firered_aed": { "mean_score": 95.0, "mean_raw": 5.0 }, "flash": { "mean_score": 100.0, "mean_raw": 0.0 }, "funasr_nano": { "mean_score": 98.0, "mean_raw": 2.0 }, "gemini_3_5_flash_lite_stt": { "mean_score": 77.0, "mean_raw": 23.0 }, "gemini_3_5_flash_stt": { "mean_score": 87.0, "mean_raw": 13.0 }, "gemini_3_5_transcribe": { "mean_score": 91.0, "mean_raw": 9.0 }, "gemini_3_6_flash_stt": { "mean_score": 95.0, "mean_raw": 5.0 }, "gemini_3_7_flash_stt": { "mean_score": 82.0, "mean_raw": 18.0 }, "gemini_3_8_flash_stt": { "mean_score": 93.0, "mean_raw": 7.0 }, "gemini_flash_stt": { "mean_score": 86.0, "mean_raw": 14.0 }, "gemini_pro_stt": { "mean_score": 97.0, "mean_raw": 3.0 }, "gemini_stt": { "mean_score": 84.0, "mean_raw": 16.0 }, "glm_asr_nano": { "mean_score": 90.0, "mean_raw": 10.0 }, "gpt_transcribe": { "mean_score": 98.0, "mean_raw": 2.0 }, "mai_transcribe_2": { "mean_score": 98.0, "mean_raw": 2.0 }, "mimo": { "mean_score": 97.0, "mean_raw": 3.0 }, "mimo_stt": { "mean_score": 90.0, "mean_raw": 10.0 }, "moss_audio": { "mean_score": 89.0, "mean_raw": 11.0 }, "qwen3_oss": { "mean_score": 93.0, "mean_raw": 7.0 }, "qwen_audio_3_1_asr": { "mean_score": 98.0, "mean_raw": 2.0 }, "r2t2": { "mean_score": 97.0, "mean_raw": 3.0 }, "soniox": { "mean_score": 91.0, "mean_raw": 9.0 }, "stepfun": { "mean_score": 90.0, "mean_raw": 10.0 }, "vibevoice": { "mean_score": 91.0, "mean_raw": 9.0 }, "voxtral": { "mean_score": 95.0, "mean_raw": 5.0 }, "whisper": { "mean_score": 99.0, "mean_raw": 1.0 }, "whisper_v3_turbo": { "mean_score": 95.0, "mean_raw": 5.0 } } }, "91": { "duration_s": 33, "model_scores": { "apple_speechanalyzer": { "mean_score": 83.0, "mean_raw": 17.0 }, "doubao": { "mean_score": 99.0, "mean_raw": 1.0 }, "elevenlabs": { "mean_score": 99.0, "mean_raw": 1.0 }, "firered": { "mean_score": 96.0, "mean_raw": 4.0 }, "firered_aed": { "mean_score": 99.0, "mean_raw": 1.0 }, "flash": { "mean_score": 98.0, "mean_raw": 2.0 }, "funasr_nano": { "mean_score": 99.0, "mean_raw": 1.0 }, "gemini_3_5_flash_lite_stt": { "mean_score": 80.0, "mean_raw": 20.0 }, "gemini_3_5_flash_stt": { "mean_score": 94.0, "mean_raw": 6.0 }, "gemini_3_5_transcribe": { "mean_score": 93.0, "mean_raw": 7.0 }, "gemini_3_6_flash_stt": { "mean_score": 91.0, "mean_raw": 9.0 }, "gemini_3_7_flash_stt": { "mean_score": 98.0, "mean_raw": 2.0 }, "gemini_3_8_flash_stt": { "mean_score": 99.0, "mean_raw": 1.0 }, "gemini_flash_stt": { "mean_score": 96.0, "mean_raw": 4.0 }, "gemini_pro_stt": { "mean_score": 97.0, "mean_raw": 3.0 }, "gemini_stt": { "mean_score": 93.0, "mean_raw": 7.0 }, "glm_asr_nano": { "mean_score": 63.0, "mean_raw": 37.0 }, "gpt_transcribe": { "mean_score": 97.0, "mean_raw": 3.0 }, "mai_transcribe_2": { "mean_score": 98.0, "mean_raw": 2.0 }, "mimo": { "mean_score": 99.0, "mean_raw": 1.0 }, "mimo_stt": { "mean_score": 97.0, "mean_raw": 3.0 }, "moss_audio": { "mean_score": 93.0, "mean_raw": 7.0 }, "qwen3_oss": { "mean_score": 99.0, "mean_raw": 1.0 }, "qwen_audio_3_1_asr": { "mean_score": 99.0, "mean_raw": 1.0 }, "r2t2": { "mean_score": 98.0, "mean_raw": 2.0 }, "soniox": { "mean_score": 98.0, "mean_raw": 2.0 }, "stepfun": { "mean_score": 63.0, "mean_raw": 37.0 }, "vibevoice": { "mean_score": 98.0, "mean_raw": 2.0 }, "voxtral": { "mean_score": 89.0, "mean_raw": 11.0 }, "whisper": { "mean_score": 96.0, "mean_raw": 4.0 }, "whisper_v3_turbo": { "mean_score": 96.0, "mean_raw": 4.0 } } }, "92": { "duration_s": 32, "model_scores": { "apple_speechanalyzer": { "mean_score": 100.0, "mean_raw": 0.0 }, "doubao": { "mean_score": 99.0, "mean_raw": 1.0 }, "elevenlabs": { "mean_score": 99.0, "mean_raw": 1.0 }, "firered": { "mean_score": 100.0, "mean_raw": 0.0 }, "firered_aed": { "mean_score": 98.0, "mean_raw": 2.0 }, "flash": { "mean_score": 100.0, "mean_raw": 0.0 }, "funasr_nano": { "mean_score": 100.0, "mean_raw": 0.0 }, "gemini_3_5_flash_lite_stt": { "mean_score": 94.0, "mean_raw": 6.0 }, "gemini_3_5_flash_stt": { "mean_score": 99.0, "mean_raw": 1.0 }, "gemini_3_5_transcribe": { "mean_score": 98.0, "mean_raw": 2.0 }, "gemini_3_6_flash_stt": { "mean_score": 93.0, "mean_raw": 7.0 }, "gemini_3_7_flash_stt": { "mean_score": 99.0, "mean_raw": 1.0 }, "gemini_3_8_flash_stt": { "mean_score": 98.0, "mean_raw": 2.0 }, "gemini_flash_stt": { "mean_score": 97.0, "mean_raw": 3.0 }, "gemini_pro_stt": { "mean_score": 99.0, "mean_raw": 1.0 }, "gemini_stt": { "mean_score": 97.0, "mean_raw": 3.0 }, "glm_asr_nano": { "mean_score": 100.0, "mean_raw": 0.0 }, "gpt_transcribe": { "mean_score": 100.0, "mean_raw": 0.0 }, "mai_transcribe_2": { "mean_score": 99.0, "mean_raw": 1.0 }, "mimo": { "mean_score": 100.0, "mean_raw": 0.0 }, "mimo_stt": { "mean_score": 100.0, "mean_raw": 0.0 }, "moss_audio": { "mean_score": 95.0, "mean_raw": 5.0 }, "qwen3_oss": { "mean_score": 100.0, "mean_raw": 0.0 }, "qwen_audio_3_1_asr": { "mean_score": 98.0, "mean_raw": 2.0 }, "r2t2": { "mean_score": 97.0, "mean_raw": 3.0 }, "soniox": { "mean_score": 100.0, "mean_raw": 0.0 }, "stepfun": { "mean_score": 100.0, "mean_raw": 0.0 }, "vibevoice": { "mean_score": 99.0, "mean_raw": 1.0 }, "voxtral": { "mean_score": 99.0, "mean_raw": 1.0 }, "whisper": { "mean_score": 99.0, "mean_raw": 1.0 }, "whisper_v3_turbo": { "mean_score": 99.0, "mean_raw": 1.0 } } }, "93": { "duration_s": 32, "model_scores": { "apple_speechanalyzer": { "mean_score": 77.0, "mean_raw": 23.0 }, "doubao": { "mean_score": 96.0, "mean_raw": 4.0 }, "elevenlabs": { "mean_score": 93.0, "mean_raw": 7.0 }, "firered": { "mean_score": 75.0, "mean_raw": 25.0 }, "firered_aed": { "mean_score": 90.0, "mean_raw": 10.0 }, "flash": { "mean_score": 98.0, "mean_raw": 2.0 }, "funasr_nano": { "mean_score": 97.0, "mean_raw": 3.0 }, "gemini_3_5_flash_lite_stt": { "mean_score": 96.0, "mean_raw": 4.0 }, "gemini_3_5_flash_stt": { "mean_score": 86.0, "mean_raw": 14.0 }, "gemini_3_5_transcribe": { "mean_score": 88.0, "mean_raw": 12.0 }, "gemini_3_6_flash_stt": { "mean_score": 80.0, "mean_raw": 20.0 }, "gemini_3_7_flash_stt": { "mean_score": 83.0, "mean_raw": 17.0 }, "gemini_3_8_flash_stt": { "mean_score": 90.0, "mean_raw": 10.0 }, "gemini_flash_stt": { "mean_score": 91.0, "mean_raw": 9.0 }, "gemini_pro_stt": { "mean_score": 93.0, "mean_raw": 7.0 }, "gemini_stt": { "mean_score": 91.0, "mean_raw": 9.0 }, "glm_asr_nano": { "mean_score": 84.0, "mean_raw": 16.0 }, "gpt_transcribe": { "mean_score": 95.0, "mean_raw": 5.0 }, "mai_transcribe_2": { "mean_score": 100.0, "mean_raw": 0.0 }, "mimo": { "mean_score": 93.0, "mean_raw": 7.0 }, "mimo_stt": { "mean_score": 93.0, "mean_raw": 7.0 }, "moss_audio": { "mean_score": 100.0, "mean_raw": 0.0 }, "qwen3_oss": { "mean_score": 96.0, "mean_raw": 4.0 }, "qwen_audio_3_1_asr": { "mean_score": 98.0, "mean_raw": 2.0 }, "r2t2": { "mean_score": 90.0, "mean_raw": 10.0 }, "soniox": { "mean_score": 92.0, "mean_raw": 8.0 }, "stepfun": { "mean_score": 68.0, "mean_raw": 32.0 }, "vibevoice": { "mean_score": 99.0, "mean_raw": 1.0 }, "voxtral": { "mean_score": 78.0, "mean_raw": 22.0 }, "whisper": { "mean_score": 98.0, "mean_raw": 2.0 }, "whisper_v3_turbo": { "mean_score": 77.0, "mean_raw": 23.0 } } }, "94": { "duration_s": 32, "model_scores": { "apple_speechanalyzer": { "mean_score": 93.0, "mean_raw": 7.0 }, "doubao": { "mean_score": 98.0, "mean_raw": 2.0 }, "elevenlabs": { "mean_score": 97.0, "mean_raw": 3.0 }, "firered": { "mean_score": 90.0, "mean_raw": 10.0 }, "firered_aed": { "mean_score": 99.0, "mean_raw": 1.0 }, "flash": { "mean_score": 98.0, "mean_raw": 2.0 }, "funasr_nano": { "mean_score": 92.0, "mean_raw": 8.0 }, "gemini_3_5_flash_lite_stt": { "mean_score": 91.0, "mean_raw": 9.0 }, "gemini_3_5_flash_stt": { "mean_score": 95.0, "mean_raw": 5.0 }, "gemini_3_5_transcribe": { "mean_score": 96.0, "mean_raw": 4.0 }, "gemini_3_6_flash_stt": { "mean_score": 78.0, "mean_raw": 22.0 }, "gemini_3_7_flash_stt": { "mean_score": 87.0, "mean_raw": 13.0 }, "gemini_3_8_flash_stt": { "mean_score": 90.0, "mean_raw": 10.0 }, "gemini_flash_stt": { "mean_score": 96.0, "mean_raw": 4.0 }, "gemini_pro_stt": { "mean_score": 99.0, "mean_raw": 1.0 }, "gemini_stt": { "mean_score": 90.0, "mean_raw": 10.0 }, "glm_asr_nano": { "mean_score": 96.0, "mean_raw": 4.0 }, "gpt_transcribe": { "mean_score": 97.0, "mean_raw": 3.0 }, "mai_transcribe_2": { "mean_score": 99.0, "mean_raw": 1.0 }, "mimo": { "mean_score": 93.0, "mean_raw": 7.0 }, "mimo_stt": { "mean_score": 98.0, "mean_raw": 2.0 }, "moss_audio": { "mean_score": 92.0, "mean_raw": 8.0 }, "qwen3_oss": { "mean_score": 92.0, "mean_raw": 8.0 }, "qwen_audio_3_1_asr": { "mean_score": 93.0, "mean_raw": 7.0 }, "r2t2": { "mean_score": 93.0, "mean_raw": 7.0 }, "soniox": { "mean_score": 97.0, "mean_raw": 3.0 }, "stepfun": { "mean_score": 96.0, "mean_raw": 4.0 }, "vibevoice": { "mean_score": 95.0, "mean_raw": 5.0 }, "voxtral": { "mean_score": 88.0, "mean_raw": 12.0 }, "whisper": { "mean_score": 98.0, "mean_raw": 2.0 }, "whisper_v3_turbo": { "mean_score": 77.0, "mean_raw": 23.0 } } }, "95": { "duration_s": 32, "model_scores": { "apple_speechanalyzer": { "mean_score": 96.0, "mean_raw": 4.0 }, "doubao": { "mean_score": 99.0, "mean_raw": 1.0 }, "elevenlabs": { "mean_score": 95.0, "mean_raw": 5.0 }, "firered": { "mean_score": 95.0, "mean_raw": 5.0 }, "firered_aed": { "mean_score": 99.0, "mean_raw": 1.0 }, "flash": { "mean_score": 96.0, "mean_raw": 4.0 }, "funasr_nano": { "mean_score": 95.0, "mean_raw": 5.0 }, "gemini_3_5_flash_lite_stt": { "mean_score": 59.0, "mean_raw": 41.0 }, "gemini_3_5_flash_stt": { "mean_score": 76.0, "mean_raw": 24.0 }, "gemini_3_5_transcribe": { "mean_score": 98.0, "mean_raw": 2.0 }, "gemini_3_6_flash_stt": { "mean_score": 94.0, "mean_raw": 6.0 }, "gemini_3_7_flash_stt": { "mean_score": 99.0, "mean_raw": 1.0 }, "gemini_3_8_flash_stt": { "mean_score": 98.0, "mean_raw": 2.0 }, "gemini_flash_stt": { "mean_score": 96.0, "mean_raw": 4.0 }, "gemini_pro_stt": { "mean_score": 97.0, "mean_raw": 3.0 }, "gemini_stt": { "mean_score": 93.0, "mean_raw": 7.0 }, "glm_asr_nano": { "mean_score": 97.0, "mean_raw": 3.0 }, "gpt_transcribe": { "mean_score": 95.0, "mean_raw": 5.0 }, "mai_transcribe_2": { "mean_score": 99.0, "mean_raw": 1.0 }, "mimo": { "mean_score": 97.0, "mean_raw": 3.0 }, "mimo_stt": { "mean_score": 99.0, "mean_raw": 1.0 }, "moss_audio": { "mean_score": 95.0, "mean_raw": 5.0 }, "qwen3_oss": { "mean_score": 93.0, "mean_raw": 7.0 }, "qwen_audio_3_1_asr": { "mean_score": 94.0, "mean_raw": 6.0 }, "r2t2": { "mean_score": 96.0, "mean_raw": 4.0 }, "soniox": { "mean_score": 100.0, "mean_raw": 0.0 }, "stepfun": { "mean_score": 77.0, "mean_raw": 23.0 }, "vibevoice": { "mean_score": 98.0, "mean_raw": 2.0 }, "voxtral": { "mean_score": 86.0, "mean_raw": 14.0 }, "whisper": { "mean_score": 97.0, "mean_raw": 3.0 }, "whisper_v3_turbo": { "mean_score": 95.0, "mean_raw": 5.0 } } }, "96": { "duration_s": 31, "model_scores": { "apple_speechanalyzer": { "mean_score": 100.0, "mean_raw": 0.0 }, "doubao": { "mean_score": 100.0, "mean_raw": 0.0 }, "elevenlabs": { "mean_score": 100.0, "mean_raw": 0.0 }, "firered": { "mean_score": 100.0, "mean_raw": 0.0 }, "firered_aed": { "mean_score": 99.0, "mean_raw": 1.0 }, "flash": { "mean_score": 100.0, "mean_raw": 0.0 }, "funasr_nano": { "mean_score": 100.0, "mean_raw": 0.0 }, "gemini_3_5_flash_lite_stt": { "mean_score": 99.0, "mean_raw": 1.0 }, "gemini_3_5_flash_stt": { "mean_score": 100.0, "mean_raw": 0.0 }, "gemini_3_5_transcribe": { "mean_score": 100.0, "mean_raw": 0.0 }, "gemini_3_6_flash_stt": { "mean_score": 98.0, "mean_raw": 2.0 }, "gemini_3_7_flash_stt": { "mean_score": 98.0, "mean_raw": 2.0 }, "gemini_3_8_flash_stt": { "mean_score": 100.0, "mean_raw": 0.0 }, "gemini_flash_stt": { "mean_score": 99.0, "mean_raw": 1.0 }, "gemini_pro_stt": { "mean_score": 99.0, "mean_raw": 1.0 }, "gemini_stt": { "mean_score": 99.0, "mean_raw": 1.0 }, "glm_asr_nano": { "mean_score": 100.0, "mean_raw": 0.0 }, "gpt_transcribe": { "mean_score": 99.0, "mean_raw": 1.0 }, "mai_transcribe_2": { "mean_score": 100.0, "mean_raw": 0.0 }, "mimo": { "mean_score": 99.0, "mean_raw": 1.0 }, "mimo_stt": { "mean_score": 100.0, "mean_raw": 0.0 }, "moss_audio": { "mean_score": 100.0, "mean_raw": 0.0 }, "qwen3_oss": { "mean_score": 100.0, "mean_raw": 0.0 }, "qwen_audio_3_1_asr": { "mean_score": 100.0, "mean_raw": 0.0 }, "r2t2": { "mean_score": 100.0, "mean_raw": 0.0 }, "soniox": { "mean_score": 99.0, "mean_raw": 1.0 }, "stepfun": { "mean_score": 85.0, "mean_raw": 15.0 }, "vibevoice": { "mean_score": 100.0, "mean_raw": 0.0 }, "voxtral": { "mean_score": 99.0, "mean_raw": 1.0 }, "whisper": { "mean_score": 100.0, "mean_raw": 0.0 }, "whisper_v3_turbo": { "mean_score": 100.0, "mean_raw": 0.0 } } }, "97": { "duration_s": 30, "model_scores": { "apple_speechanalyzer": { "mean_score": 96.0, "mean_raw": 4.0 }, "doubao": { "mean_score": 100.0, "mean_raw": 0.0 }, "elevenlabs": { "mean_score": 97.0, "mean_raw": 3.0 }, "firered": { "mean_score": 93.0, "mean_raw": 7.0 }, "firered_aed": { "mean_score": 96.0, "mean_raw": 4.0 }, "flash": { "mean_score": 96.0, "mean_raw": 4.0 }, "funasr_nano": { "mean_score": 96.0, "mean_raw": 4.0 }, "gemini_3_5_flash_lite_stt": { "mean_score": 97.0, "mean_raw": 3.0 }, "gemini_3_5_flash_stt": { "mean_score": 100.0, "mean_raw": 0.0 }, "gemini_3_5_transcribe": { "mean_score": 97.0, "mean_raw": 3.0 }, "gemini_3_6_flash_stt": { "mean_score": 100.0, "mean_raw": 0.0 }, "gemini_3_7_flash_stt": { "mean_score": 98.0, "mean_raw": 2.0 }, "gemini_3_8_flash_stt": { "mean_score": 98.0, "mean_raw": 2.0 }, "gemini_flash_stt": { "mean_score": 98.0, "mean_raw": 2.0 }, "gemini_pro_stt": { "mean_score": 99.0, "mean_raw": 1.0 }, "gemini_stt": { "mean_score": 95.0, "mean_raw": 5.0 }, "glm_asr_nano": { "mean_score": 98.0, "mean_raw": 2.0 }, "gpt_transcribe": { "mean_score": 97.0, "mean_raw": 3.0 }, "mai_transcribe_2": { "mean_score": 96.0, "mean_raw": 4.0 }, "mimo": { "mean_score": 98.0, "mean_raw": 2.0 }, "mimo_stt": { "mean_score": 95.0, "mean_raw": 5.0 }, "moss_audio": { "mean_score": 95.0, "mean_raw": 5.0 }, "qwen3_oss": { "mean_score": 97.0, "mean_raw": 3.0 }, "qwen_audio_3_1_asr": { "mean_score": 97.0, "mean_raw": 3.0 }, "r2t2": { "mean_score": 93.0, "mean_raw": 7.0 }, "soniox": { "mean_score": 98.0, "mean_raw": 2.0 }, "stepfun": { "mean_score": 82.0, "mean_raw": 18.0 }, "vibevoice": { "mean_score": 99.0, "mean_raw": 1.0 }, "voxtral": { "mean_score": 95.0, "mean_raw": 5.0 }, "whisper": { "mean_score": 97.0, "mean_raw": 3.0 }, "whisper_v3_turbo": { "mean_score": 93.0, "mean_raw": 7.0 } } }, "98": { "duration_s": 30, "model_scores": { "apple_speechanalyzer": { "mean_score": 94.0, "mean_raw": 6.0 }, "doubao": { "mean_score": 99.0, "mean_raw": 1.0 }, "elevenlabs": { "mean_score": 97.0, "mean_raw": 3.0 }, "firered": { "mean_score": 97.0, "mean_raw": 3.0 }, "firered_aed": { "mean_score": 99.0, "mean_raw": 1.0 }, "flash": { "mean_score": 99.0, "mean_raw": 1.0 }, "funasr_nano": { "mean_score": 99.0, "mean_raw": 1.0 }, "gemini_3_5_flash_lite_stt": { "mean_score": 97.0, "mean_raw": 3.0 }, "gemini_3_5_flash_stt": { "mean_score": 97.0, "mean_raw": 3.0 }, "gemini_3_5_transcribe": { "mean_score": 98.0, "mean_raw": 2.0 }, "gemini_3_6_flash_stt": { "mean_score": 98.0, "mean_raw": 2.0 }, "gemini_3_7_flash_stt": { "mean_score": 98.0, "mean_raw": 2.0 }, "gemini_3_8_flash_stt": { "mean_score": 98.0, "mean_raw": 2.0 }, "gemini_flash_stt": { "mean_score": 96.0, "mean_raw": 4.0 }, "gemini_pro_stt": { "mean_score": 100.0, "mean_raw": 0.0 }, "gemini_stt": { "mean_score": 97.0, "mean_raw": 3.0 }, "glm_asr_nano": { "mean_score": 99.0, "mean_raw": 1.0 }, "gpt_transcribe": { "mean_score": 99.0, "mean_raw": 1.0 }, "mai_transcribe_2": { "mean_score": 99.0, "mean_raw": 1.0 }, "mimo": { "mean_score": 99.0, "mean_raw": 1.0 }, "mimo_stt": { "mean_score": 99.0, "mean_raw": 1.0 }, "moss_audio": { "mean_score": 99.0, "mean_raw": 1.0 }, "qwen3_oss": { "mean_score": 98.0, "mean_raw": 2.0 }, "qwen_audio_3_1_asr": { "mean_score": 99.0, "mean_raw": 1.0 }, "r2t2": { "mean_score": 98.0, "mean_raw": 2.0 }, "soniox": { "mean_score": 98.0, "mean_raw": 2.0 }, "stepfun": { "mean_score": 79.0, "mean_raw": 21.0 }, "vibevoice": { "mean_score": 99.0, "mean_raw": 1.0 }, "voxtral": { "mean_score": 97.0, "mean_raw": 3.0 }, "whisper": { "mean_score": 97.0, "mean_raw": 3.0 }, "whisper_v3_turbo": { "mean_score": 95.0, "mean_raw": 5.0 } } }, "99": { "duration_s": 30, "model_scores": { "apple_speechanalyzer": { "mean_score": 100.0, "mean_raw": 0.0 }, "doubao": { "mean_score": 100.0, "mean_raw": 0.0 }, "elevenlabs": { "mean_score": 100.0, "mean_raw": 0.0 }, "firered": { "mean_score": 98.0, "mean_raw": 2.0 }, "firered_aed": { "mean_score": 100.0, "mean_raw": 0.0 }, "flash": { "mean_score": 100.0, "mean_raw": 0.0 }, "funasr_nano": { "mean_score": 100.0, "mean_raw": 0.0 }, "gemini_3_5_flash_lite_stt": { "mean_score": 99.0, "mean_raw": 1.0 }, "gemini_3_5_flash_stt": { "mean_score": 96.0, "mean_raw": 4.0 }, "gemini_3_5_transcribe": { "mean_score": 97.0, "mean_raw": 3.0 }, "gemini_3_6_flash_stt": { "mean_score": 90.0, "mean_raw": 10.0 }, "gemini_3_7_flash_stt": { "mean_score": 99.0, "mean_raw": 1.0 }, "gemini_3_8_flash_stt": { "mean_score": 95.0, "mean_raw": 5.0 }, "gemini_flash_stt": { "mean_score": 97.0, "mean_raw": 3.0 }, "gemini_pro_stt": { "mean_score": 99.0, "mean_raw": 1.0 }, "gemini_stt": { "mean_score": 96.0, "mean_raw": 4.0 }, "glm_asr_nano": { "mean_score": 100.0, "mean_raw": 0.0 }, "gpt_transcribe": { "mean_score": 100.0, "mean_raw": 0.0 }, "mai_transcribe_2": { "mean_score": 100.0, "mean_raw": 0.0 }, "mimo": { "mean_score": 100.0, "mean_raw": 0.0 }, "mimo_stt": { "mean_score": 98.0, "mean_raw": 2.0 }, "moss_audio": { "mean_score": 100.0, "mean_raw": 0.0 }, "qwen3_oss": { "mean_score": 100.0, "mean_raw": 0.0 }, "qwen_audio_3_1_asr": { "mean_score": 100.0, "mean_raw": 0.0 }, "r2t2": { "mean_score": 99.0, "mean_raw": 1.0 }, "soniox": { "mean_score": 99.0, "mean_raw": 1.0 }, "stepfun": { "mean_score": 85.0, "mean_raw": 15.0 }, "vibevoice": { "mean_score": 99.0, "mean_raw": 1.0 }, "voxtral": { "mean_score": 98.0, "mean_raw": 2.0 }, "whisper": { "mean_score": 100.0, "mean_raw": 0.0 }, "whisper_v3_turbo": { "mean_score": 98.0, "mean_raw": 2.0 } } }, "100": { "duration_s": 30, "model_scores": { "apple_speechanalyzer": { "mean_score": 93.0, "mean_raw": 7.0 }, "doubao": { "mean_score": 95.0, "mean_raw": 5.0 }, "elevenlabs": { "mean_score": 97.0, "mean_raw": 3.0 }, "firered": { "mean_score": 84.0, "mean_raw": 16.0 }, "firered_aed": { "mean_score": 78.0, "mean_raw": 22.0 }, "flash": { "mean_score": 92.0, "mean_raw": 8.0 }, "funasr_nano": { "mean_score": 89.0, "mean_raw": 11.0 }, "gemini_3_5_flash_lite_stt": { "mean_score": 92.0, "mean_raw": 8.0 }, "gemini_3_5_flash_stt": { "mean_score": 97.0, "mean_raw": 3.0 }, "gemini_3_5_transcribe": { "mean_score": 98.0, "mean_raw": 2.0 }, "gemini_3_6_flash_stt": { "mean_score": 98.0, "mean_raw": 2.0 }, "gemini_3_7_flash_stt": { "mean_score": 97.0, "mean_raw": 3.0 }, "gemini_3_8_flash_stt": { "mean_score": 97.0, "mean_raw": 3.0 }, "gemini_flash_stt": { "mean_score": 98.0, "mean_raw": 2.0 }, "gemini_pro_stt": { "mean_score": 97.0, "mean_raw": 3.0 }, "gemini_stt": { "mean_score": 97.0, "mean_raw": 3.0 }, "glm_asr_nano": { "mean_score": 95.0, "mean_raw": 5.0 }, "gpt_transcribe": { "mean_score": 97.0, "mean_raw": 3.0 }, "mai_transcribe_2": { "mean_score": 98.0, "mean_raw": 2.0 }, "mimo": { "mean_score": 98.0, "mean_raw": 2.0 }, "mimo_stt": { "mean_score": 94.0, "mean_raw": 6.0 }, "moss_audio": { "mean_score": 94.0, "mean_raw": 6.0 }, "qwen3_oss": { "mean_score": 95.0, "mean_raw": 5.0 }, "qwen_audio_3_1_asr": { "mean_score": 98.0, "mean_raw": 2.0 }, "r2t2": { "mean_score": 96.0, "mean_raw": 4.0 }, "soniox": { "mean_score": 96.0, "mean_raw": 4.0 }, "stepfun": { "mean_score": 97.0, "mean_raw": 3.0 }, "vibevoice": { "mean_score": 96.0, "mean_raw": 4.0 }, "voxtral": { "mean_score": 86.0, "mean_raw": 14.0 }, "whisper": { "mean_score": 100.0, "mean_raw": 0.0 }, "whisper_v3_turbo": { "mean_score": 96.0, "mean_raw": 4.0 } } } }, "judge_agreement": {}, "generated_date": "2026-09-23", "fidelity_floor": 20.0, "group_floors": { "meaning-flip": 4.171, "entity": 5.64, "cultural": 1.654, "fidelity": 8.535 }, "group_loss_share": { "meaning-flip": 0.2085, "entity": 0.282, "cultural": 0.0827, "fidelity": 0.4268 }, "metrics_comparison": { "total_tokens": 15440, "total_minutes": 78.3, "n_samples": 100, "by_model": { "apple_speechanalyzer": { "total_ded": 2328.0, "avg_per_sample": 23.28, "ded_per_min": 29.73, "ded_per_1k_tok": 150.78, "fidelity_1k": 0, "clean_rate": 7.0, "accept_at_5": 19.0, "accept_at_10": 39.0, "accept_at_20": 60.0 }, "doubao": { "total_ded": 812.0, "avg_per_sample": 8.12, "ded_per_min": 10.37, "ded_per_1k_tok": 52.59, "fidelity_1k": 47.4, "clean_rate": 21.0, "accept_at_5": 57.0, "accept_at_10": 72.0, "accept_at_20": 90.0 }, "elevenlabs": { "total_ded": 727.0, "avg_per_sample": 7.27, "ded_per_min": 9.28, "ded_per_1k_tok": 47.09, "fidelity_1k": 52.9, "clean_rate": 17.0, "accept_at_5": 59.0, "accept_at_10": 78.0, "accept_at_20": 89.0 }, "firered": { "total_ded": 2752.0, "avg_per_sample": 27.52, "ded_per_min": 35.15, "ded_per_1k_tok": 178.24, "fidelity_1k": 0, "clean_rate": 4.0, "accept_at_5": 18.0, "accept_at_10": 34.0, "accept_at_20": 50.0 }, "firered_aed": { "total_ded": 1390.0, "avg_per_sample": 13.9, "ded_per_min": 17.75, "ded_per_1k_tok": 90.03, "fidelity_1k": 10.0, "clean_rate": 12.0, "accept_at_5": 41.0, "accept_at_10": 57.0, "accept_at_20": 73.0 }, "flash": { "total_ded": 700.0, "avg_per_sample": 7.0, "ded_per_min": 8.94, "ded_per_1k_tok": 45.34, "fidelity_1k": 54.7, "clean_rate": 23.0, "accept_at_5": 67.0, "accept_at_10": 79.0, "accept_at_20": 89.0 }, "funasr_nano": { "total_ded": 1624.0, "avg_per_sample": 16.24, "ded_per_min": 20.74, "ded_per_1k_tok": 105.18, "fidelity_1k": 0, "clean_rate": 17.0, "accept_at_5": 42.0, "accept_at_10": 60.0, "accept_at_20": 73.0 }, "gemini_3_5_flash_lite_stt": { "total_ded": 2360.0, "avg_per_sample": 23.6, "ded_per_min": 30.14, "ded_per_1k_tok": 152.85, "fidelity_1k": 0, "clean_rate": 3.0, "accept_at_5": 20.0, "accept_at_10": 37.0, "accept_at_20": 59.0 }, "gemini_3_5_flash_stt": { "total_ded": 1069.0, "avg_per_sample": 10.69, "ded_per_min": 13.65, "ded_per_1k_tok": 69.24, "fidelity_1k": 30.8, "clean_rate": 9.0, "accept_at_5": 40.0, "accept_at_10": 60.0, "accept_at_20": 87.0 }, "gemini_3_5_transcribe": { "total_ded": 1162.0, "avg_per_sample": 11.62, "ded_per_min": 14.84, "ded_per_1k_tok": 75.26, "fidelity_1k": 24.7, "clean_rate": 8.0, "accept_at_5": 43.0, "accept_at_10": 58.0, "accept_at_20": 84.0 }, "gemini_3_6_flash_stt": { "total_ded": 1362.0, "avg_per_sample": 13.62, "ded_per_min": 17.39, "ded_per_1k_tok": 88.21, "fidelity_1k": 11.8, "clean_rate": 6.0, "accept_at_5": 31.0, "accept_at_10": 50.0, "accept_at_20": 75.0 }, "gemini_3_7_flash_stt": { "total_ded": 939.0, "avg_per_sample": 9.39, "ded_per_min": 11.99, "ded_per_1k_tok": 60.82, "fidelity_1k": 39.2, "clean_rate": 8.0, "accept_at_5": 47.0, "accept_at_10": 63.0, "accept_at_20": 86.0 }, "gemini_3_8_flash_stt": { "total_ded": 701.0, "avg_per_sample": 7.01, "ded_per_min": 8.95, "ded_per_1k_tok": 45.4, "fidelity_1k": 54.6, "clean_rate": 5.0, "accept_at_5": 57.0, "accept_at_10": 78.0, "accept_at_20": 92.0 }, "gemini_flash_stt": { "total_ded": 1061.0, "avg_per_sample": 10.61, "ded_per_min": 13.55, "ded_per_1k_tok": 68.72, "fidelity_1k": 31.3, "clean_rate": 3.0, "accept_at_5": 47.0, "accept_at_10": 60.0, "accept_at_20": 88.0 }, "gemini_pro_stt": { "total_ded": 772.0, "avg_per_sample": 7.72, "ded_per_min": 9.86, "ded_per_1k_tok": 50.0, "fidelity_1k": 50.0, "clean_rate": 14.0, "accept_at_5": 59.0, "accept_at_10": 76.0, "accept_at_20": 90.0 }, "gemini_stt": { "total_ded": 1636.0, "avg_per_sample": 16.36, "ded_per_min": 20.89, "ded_per_1k_tok": 105.96, "fidelity_1k": 0, "clean_rate": 3.0, "accept_at_5": 30.0, "accept_at_10": 46.0, "accept_at_20": 71.0 }, "glm_asr_nano": { "total_ded": 1218.0, "avg_per_sample": 12.18, "ded_per_min": 15.56, "ded_per_1k_tok": 78.89, "fidelity_1k": 21.1, "clean_rate": 16.0, "accept_at_5": 47.0, "accept_at_10": 63.0, "accept_at_20": 80.0 }, "gpt_transcribe": { "total_ded": 863.0, "avg_per_sample": 8.63, "ded_per_min": 11.02, "ded_per_1k_tok": 55.89, "fidelity_1k": 44.1, "clean_rate": 13.0, "accept_at_5": 56.0, "accept_at_10": 64.0, "accept_at_20": 92.0 }, "mai_transcribe_2": { "total_ded": 434.0, "avg_per_sample": 4.34, "ded_per_min": 5.54, "ded_per_1k_tok": 28.11, "fidelity_1k": 71.9, "clean_rate": 33.0, "accept_at_5": 78.0, "accept_at_10": 88.0, "accept_at_20": 95.0 }, "mimo": { "total_ded": 564.0, "avg_per_sample": 5.64, "ded_per_min": 7.2, "ded_per_1k_tok": 36.53, "fidelity_1k": 63.5, "clean_rate": 22.0, "accept_at_5": 69.0, "accept_at_10": 84.0, "accept_at_20": 94.0 }, "mimo_stt": { "total_ded": 925.0, "avg_per_sample": 9.25, "ded_per_min": 11.81, "ded_per_1k_tok": 59.91, "fidelity_1k": 40.1, "clean_rate": 6.0, "accept_at_5": 48.0, "accept_at_10": 72.0, "accept_at_20": 89.0 }, "moss_audio": { "total_ded": 2317.0, "avg_per_sample": 23.17, "ded_per_min": 29.59, "ded_per_1k_tok": 150.06, "fidelity_1k": 0, "clean_rate": 12.0, "accept_at_5": 27.0, "accept_at_10": 39.0, "accept_at_20": 56.0 }, "qwen3_oss": { "total_ded": 1080.0, "avg_per_sample": 10.8, "ded_per_min": 13.79, "ded_per_1k_tok": 69.95, "fidelity_1k": 30.1, "clean_rate": 18.0, "accept_at_5": 48.0, "accept_at_10": 65.0, "accept_at_20": 82.0 }, "qwen_audio_3_1_asr": { "total_ded": 708.0, "avg_per_sample": 7.08, "ded_per_min": 9.04, "ded_per_1k_tok": 45.85, "fidelity_1k": 54.1, "clean_rate": 20.0, "accept_at_5": 61.0, "accept_at_10": 81.0, "accept_at_20": 91.0 }, "r2t2": { "total_ded": 819.0, "avg_per_sample": 8.19, "ded_per_min": 10.46, "ded_per_1k_tok": 53.04, "fidelity_1k": 47.0, "clean_rate": 15.0, "accept_at_5": 57.0, "accept_at_10": 75.0, "accept_at_20": 89.0 }, "soniox": { "total_ded": 860.0, "avg_per_sample": 8.6, "ded_per_min": 10.98, "ded_per_1k_tok": 55.7, "fidelity_1k": 44.3, "clean_rate": 10.0, "accept_at_5": 54.0, "accept_at_10": 70.0, "accept_at_20": 87.0 }, "stepfun": { "total_ded": 1297.0, "avg_per_sample": 12.97, "ded_per_min": 16.56, "ded_per_1k_tok": 84.0, "fidelity_1k": 16.0, "clean_rate": 11.0, "accept_at_5": 39.0, "accept_at_10": 59.0, "accept_at_20": 79.0 }, "vibevoice": { "total_ded": 887.0, "avg_per_sample": 8.87, "ded_per_min": 11.33, "ded_per_1k_tok": 57.45, "fidelity_1k": 42.6, "clean_rate": 15.0, "accept_at_5": 55.0, "accept_at_10": 72.0, "accept_at_20": 86.0 }, "voxtral": { "total_ded": 2003.0, "avg_per_sample": 20.03, "ded_per_min": 25.58, "ded_per_1k_tok": 129.73, "fidelity_1k": 0, "clean_rate": 5.0, "accept_at_5": 30.0, "accept_at_10": 40.0, "accept_at_20": 57.0 }, "whisper": { "total_ded": 1061.0, "avg_per_sample": 10.61, "ded_per_min": 13.55, "ded_per_1k_tok": 68.72, "fidelity_1k": 31.3, "clean_rate": 10.0, "accept_at_5": 51.0, "accept_at_10": 64.0, "accept_at_20": 83.0 }, "whisper_v3_turbo": { "total_ded": 1841.0, "avg_per_sample": 18.41, "ded_per_min": 23.51, "ded_per_1k_tok": 119.24, "fidelity_1k": 0, "clean_rate": 5.0, "accept_at_5": 37.0, "accept_at_10": 46.0, "accept_at_20": 65.0 } } }, "pricing": { "apple_speechanalyzer": { "usd_per_min": null, "verified": false, "basis": "no-marginal-cost", "source_url": null, "checked": null, "deployment": "on-device", "note": "Bundled with macOS 26; fully on-device after locale asset download" }, "doubao": { "usd_per_min": null, "verified": false, "basis": "unpriced", "source_url": "https://www.volcengine.com/docs/6561/1359370", "checked": "2026-08-01", "deployment": "hosted-api", "note": "UNRESOLVED, still deliberately null, but the earlier reasoning was wrong and is corrected here 2026-08-27. This benchmark calls resource id volc.seedasr.auc at /api/v3/auc/bigmodel/submit. That resource id IS 豆包录音文件识别模型2.0 -- 大模型录音文件识别 is the service category, not a different, pricier product. The earlier note claimed the two were distinct and priced the row out of the chart on that basis. The price is still null for a different and real reason: Volcengine bills this by purchased volume tier AND by variant (标准版 / 极速版 / 闲时版), and the resource id does not say which variant or which package the account holds. Across the plausible range that is roughly $0.003-$0.011 per minute, a 3.7x spread, which is too wide to plot. Fill in only from the Volcengine console." }, "elevenlabs": { "usd_per_min": 0.003667, "verified": true, "basis": "vendor-list", "source_url": "https://elevenlabs.io/pricing/api", "checked": "2026-08-01", "deployment": "hosted-api", "note": "CORRECTED 2026-08-01. Was 0.0067, which is 1.8x too high and looks like a stale ~$0.40/hour figure. The API pricing page states Scribe v2 at $0.22/hour on every plan tier (Free through Business), and Scribe v2 Realtime at $0.39/hour. $0.22/hour = $0.003667/minute." }, "firered": { "usd_per_min": null, "verified": false, "basis": "no-marginal-cost", "source_url": null, "checked": null, "deployment": "local-cpu", "note": "Local CPU only (no GPU kernel for int8 in ONNX Runtime CUDA EP). Measured RTF 0.23 at 16 threads on a 24-core desktop." }, "firered_aed": { "usd_per_min": null, "verified": false, "basis": "no-marginal-cost", "source_url": null, "checked": null, "deployment": "local-gpu", "note": "Local RTX 4090 marginal cost ~ $0. Cloud-equivalent = measured RTF 0.10 x ~$1.10/hr (L40S class) / 60 = ~$0.002/min. fp32; 8.9 GB VRAM." }, "flash": { "usd_per_min": 0.0021, "verified": true, "basis": "vendor-list", "source_url": "https://openrouter.ai/qwen/qwen3-asr-flash-2026-02-10/pricing", "checked": "2026-08-01", "deployment": "hosted-api", "note": "CORRECTED 2026-08-01. Was 0.0008, which is 2.6x too low. Qwen3-ASR-Flash bills $0.000035 per audio second = $0.0021/minute; text tokens are not charged. Alibaba's own Model Studio pricing page could not be fetched (no pricing on the models page, and the console requires login), so the per-second figure is taken from OpenRouter's published effective rate. Treat as good to one significant figure and re-check against the Alibaba console before publishing a cost claim." }, "funasr_nano": { "usd_per_min": null, "verified": false, "basis": "self-hosted-gpu", "source_url": null, "checked": null, "deployment": "modal-pattern-a", "note": "Modal L4 · install funasr from GitHub main + tiktoken · idx 6 tokenizer self-bug UNVERIFIED as of 2026-08-01: no vendor source URL on file. For modal-pattern-a models this is our own amortised L4 GPU rate, identical across them, and not a vendor price." }, "gemini_3_5_flash_lite_stt": { "usd_per_min": null, "verified": false, "basis": "unpriced", "source_url": null, "checked": null, "deployment": "hosted-api", "note": "gemini-3.5-flash-lite (stable ID, no -preview) · $0.30/1M in · $2.50/1M out · 官方主打 350 tok/s 速度与性价比 · thinkingLevel=minimal" }, "gemini_3_5_flash_stt": { "usd_per_min": null, "verified": false, "basis": "unpriced", "source_url": null, "checked": null, "deployment": "hosted-api", "note": "gemini-3.5-flash (stable ID, no -preview) · $1.50/1M in · $9.00/1M out (incl thinking) · thinkingLevel=minimal → 0 thinking tokens" }, "gemini_3_5_transcribe": { "usd_per_min": 0.005, "verified": true, "basis": "vendor-list", "source_url": "https://ai.google.dev/gemini-api/docs/pricing", "checked": "2026-08-26", "deployment": "hosted-api", "note": "gemini-3.5-transcribe, checked 2026-08-26 on ai.google.dev/gemini-api/docs/pricing. Unlike every other Gemini row here it carries a per-minute list price alongside the token one: input $2.00/1M tokens or $0.003/min of audio, output $12.00/1M tokens or $0.002/min of text. The per-minute figures are used, summed to $0.005/min, which is the same all-in-per-minute-of-audio basis the dedicated-ASR vendors on this board are quoted on. The token basis is not usable for this model anyway: the Interactions API reports total_output_tokens 0 on every call despite returning a transcript, so an output cost cannot be computed from usage." }, "gemini_3_6_flash_stt": { "usd_per_min": null, "verified": false, "basis": "unpriced", "source_url": null, "checked": null, "deployment": "hosted-api", "note": "gemini-3.6-flash (stable ID, no -preview). List price $1.50/1M in, $7.50/1M out. An introductory $0.75/$3.75 applies through 2026-12-31, so a run today costs half the list figure; the gemini_3_7_flash_stt row records the same promo and this row was silently omitting it. thinkingLevel=minimal gives 0 thinking tokens." }, "gemini_3_7_flash_stt": { "usd_per_min": null, "verified": false, "basis": "unpriced", "source_url": null, "checked": null, "deployment": "hosted-api", "note": "gemini-3.7-flash (stable ID, no -preview) - $1.50/1M in, $7.50/1M out list price; an introductory $0.75/$3.75 applies through 2026-12-31 (same promo as 3.6, checked 2026-08-18 on ai.google.dev/gemini-api/docs/pricing). thinkingLevel=low: MINIMAL is rejected with HTTP 400 on this model, so unlike 3.5 and 3.6 its thinking CANNOT be switched off and every call bills thinking tokens at the output rate. Measured on this benchmark: 289k thinking tokens per 100 clips against 12k of actual transcript, which is 24x the output volume and makes a 100-clip sweep cost roughly 7.7x what 3.6 costs for the same audio." }, "gemini_3_8_flash_stt": { "usd_per_min": null, "verified": false, "basis": "unpriced", "source_url": null, "checked": null, "deployment": "hosted-api", "note": "gemini-3.8-flash (stable ID, no -preview): $1.50/1M in, $7.50/1M out list, with the same introductory $0.75/$3.75 running through 2026-12-31 that 3.6 and 3.7 get (checked 2026-09-03 on ai.google.dev/gemini-api/docs/pricing). Billed per token rather than per audio-minute, so it stays off the price chart like the other LLM-as-ASR rows. thinkingLevel=low is the floor: MINIMAL is rejected with HTTP 400 exactly as on 3.7. The difference from 3.7 is what low actually costs. Measured over this run: 0 thinking tokens across all 97 calls, against 285,823 for 3.7 on the same audio (23.9x its own transcript volume). At promo rates the full sweep is $0.157 here and $1.233 on 3.7, so 3.8 does the same job for one-eighth the money despite an identical list price and an identical thinking label." }, "gemini_flash_stt": { "usd_per_min": null, "verified": false, "basis": "unpriced", "source_url": null, "checked": null, "deployment": "hosted-api", "note": "thinkingBudget=0 (works on 3.0 Flash) · 100 idx wall ~30s · 比 3.1 Flash Lite 强很多" }, "gemini_pro_stt": { "usd_per_min": null, "verified": false, "basis": "unpriced", "source_url": null, "checked": null, "deployment": "hosted-api", "note": "thinkingBudget=0 returns HTTP 400 · 必须 thinkingLevel='low' 起步 · 最贵但成绩最好的 LLM-as-ASR · 100 idx wall ~4-5 min" }, "gemini_stt": { "usd_per_min": null, "verified": false, "basis": "unpriced", "source_url": null, "checked": null, "deployment": "hosted-api", "note": "thinkingBudget=0 · skill 注释: cheapest 3.1 + improved ASR/audio quality" }, "glm_asr_nano": { "usd_per_min": null, "verified": false, "basis": "self-hosted-gpu", "source_url": null, "checked": null, "deployment": "modal-pattern-a", "note": "Modal L4 · idx 48 catastrophic decoder loop documented UNVERIFIED as of 2026-08-01: no vendor source URL on file. For modal-pattern-a models this is our own amortised L4 GPU rate, identical across them, and not a vendor price." }, "gpt_transcribe": { "usd_per_min": 0.0045, "verified": true, "basis": "vendor-list", "source_url": "https://developers.openai.com/api/pricing/", "checked": "2026-08-01", "deployment": "hosted-api", "note": "OpenAI transcription models table, checked 2026-08-01: gpt-transcribe $0.0045/minute, between gpt-4o-mini-transcribe ($0.003) and gpt-4o-transcribe ($0.006). Billed by audio duration, which is why the input/output token columns read '-'." }, "mai_transcribe_2": { "usd_per_min": 0.001667, "verified": true, "basis": "vendor-list", "source_url": "https://openrouter.ai/microsoft/mai-transcribe-2", "checked": "2026-09-03", "deployment": "hosted-api", "note": "$0.001667/minute, i.e. $0.10/audio-hour, the rate OpenRouter lists for microsoft/mai-transcribe-2 and the rate the run was actually billed at: 100 clips / 79.8 minutes came to $0.1331, which is $0.100/hour to three decimals. The price is the gateway's, not Microsoft's: Azure's own Speech pricing page lists MAI-Transcribe under the Fast Transcription SKU but renders no figure, so there is no first-party number to check against. Same gateway prices MAI-Transcribe-1.5 at $0.36/hour, confirmed by billing one clip through both ids." }, "mimo": { "usd_per_min": 0.001233, "verified": true, "basis": "vendor-list", "source_url": "https://mimo.mi.com/docs/zh-CN/price/pay-as-you-go", "checked": "2026-08-02", "deployment": "modal-pattern-a", "note": "Xiaomi official hosted API list price, checked 2026-08-02: ASR is billed by input audio duration at ¥0.5/hour domestic, $0.074/hour international. $0.074/60 = $0.001233 per minute; the USD figure is the vendor's own, not a conversion of the CNY one. NOTE the benchmark ran self-hosted 8B weights on a Modal L4, not this endpoint; the price is what a buyer would actually pay Xiaomi for the same model, which is the comparable quantity against the other hosted APIs on the chart." }, "mimo_stt": { "usd_per_min": null, "verified": false, "basis": "unpriced", "source_url": null, "checked": null, "deployment": "hosted-api", "note": "mimo-v2.5 multimodal (Standard, NOT Pro — Pro 拒绝音频 multimodal). 约 5/100 高敏感样本被安全过滤返空" }, "moss_audio": { "usd_per_min": null, "verified": false, "basis": "self-hosted-gpu", "source_url": null, "checked": null, "deployment": "modal-pattern-a", "note": "Modal L4 same as mimo · audio-understanding multimodal LLM (MOSS-Audio-Encoder + Qwen3-8B + DeepStack adapter) · ASR 是其多任务能力之一 UNVERIFIED as of 2026-08-01: no vendor source URL on file. For modal-pattern-a models this is our own amortised L4 GPU rate, identical across them, and not a vendor price." }, "qwen3_oss": { "usd_per_min": null, "verified": false, "basis": "self-hosted-gpu", "source_url": null, "checked": null, "deployment": "modal-pattern-a", "note": "Modal L4 UNVERIFIED as of 2026-08-01: no vendor source URL on file. For modal-pattern-a models this is our own amortised L4 GPU rate, identical across them, and not a vendor price." }, "qwen_audio_3_1_asr": { "usd_per_min": null, "verified": false, "basis": "unpriced", "source_url": "https://www.qwencloud.com/models/qwen-audio-3.1-asr-flash", "checked": "2026-09-23", "deployment": "hosted-api", "note": "Billed per token, not per audio minute: $0.15/1M input, $0.47/1M output (qwencloud.com model page; help.aliyun.com lists the same as CNY 1.094 / 3.427 for Singapore), checked 2026-09-23. Measured over this run: 64,966 input tokens (about 835 per audio minute) and 12,096 output tokens for 77.8 minutes, $0.0154 in total, i.e. $0.000198/min. That figure is ours, not a vendor per-minute quote, so it stays off the price chart under the price_basis rules." }, "r2t2": { "usd_per_min": null, "verified": false, "basis": "self-hosted-gpu", "source_url": null, "checked": null, "deployment": "modal-pattern-a", "note": "Same amortised Modal L4 rate as the other modal-pattern-a rows, not a vendor price. Measured on this run: 211.7 s of server-side inference for 4738 s of audio on one L4 (vLLM backend)." }, "soniox": { "usd_per_min": 0.001667, "verified": true, "basis": "vendor-equivalent", "source_url": "https://soniox.com/pricing/", "checked": "2026-08-01", "deployment": "hosted-api", "note": "$0.001667/minute, i.e. the $0.10/hour Soniox quotes for async file transcription (stt-async-v5), checked 2026-08-01. Basis is 'vendor-equivalent' rather than 'vendor-list' because Soniox does not actually bill by audio duration: it bills tokens ($1.50/1M input audio tokens, $3.50/1M output text tokens) and publishes the hourly figure as an approximate equivalent at roughly 30k audio tokens per hour. Comparable enough to plot against the duration-billed rows, but a bill will not match it exactly. Real-time stt-rt-v5 is quoted at $0.12/hour." }, "stepfun": { "usd_per_min": null, "verified": false, "basis": "vendor-announced", "source_url": "https://x.com/StepFun_ai/status/2047690693762539830", "checked": "2026-08-27", "deployment": "hosted-api", "note": "$0.022 per hour of audio = $0.000367/minute, from StepFun's own launch post for StepAudio 2.5 ASR, checked 2026-08-27. Cross-checks against the ¥0.15/hour figure quoted for the domestic platform (¥0.15 / 6.7513 = $0.0222). Basis is 'vendor-announced' rather than 'vendor-list' because platform.stepfun.com's pricing-detail page 404s and the number comes from the vendor's announcement instead of a maintained price table. At this rate it is the cheapest priced model on the board by a factor of three, so treat it as the vendor's claim until the pricing page is reachable. Free tier is rate-limited to RPM=10." }, "vibevoice": { "usd_per_min": null, "verified": false, "basis": "self-hosted-gpu", "source_url": null, "checked": null, "deployment": "modal-pattern-a", "note": "Modal L4 same as mimo UNVERIFIED as of 2026-08-01: no vendor source URL on file. For modal-pattern-a models this is our own amortised L4 GPU rate, identical across them, and not a vendor price." }, "voxtral": { "usd_per_min": null, "verified": false, "basis": "self-hosted-gpu", "source_url": null, "checked": null, "deployment": "modal-pattern-a", "note": "Modal L4 same as mimo · 13 langs incl. Chinese · realtime-capable but file-mode used here UNVERIFIED as of 2026-08-01: no vendor source URL on file. For modal-pattern-a models this is our own amortised L4 GPU rate, identical across them, and not a vendor price." }, "whisper": { "usd_per_min": 0.003, "verified": true, "basis": "vendor-list", "source_url": "https://developers.openai.com/api/pricing/", "checked": "2026-08-01", "deployment": "hosted-api", "note": "OpenAI transcription models table, checked 2026-08-01: gpt-4o-mini-transcribe $0.003/minute. Billed by audio duration." }, "whisper_v3_turbo": { "usd_per_min": null, "verified": false, "basis": "no-marginal-cost", "source_url": null, "checked": null, "deployment": "local-gpu", "note": "Local RTX 4090 marginal cost ~ $0; cloud-equivalent ≈ $0.005/min" } }, "audio_minutes": 78.3, "model_meta": { "apple_legacy": { "dur": null, "durb": "none-published", "win": null, "mb": null, "im": "both", "so": true, "hot": true, "hotm": "contextual_strings", "hotn": null, "bias": "none", "wt": "segment", "dia": [], "label": "Apple Legacy SFSpeechRecognizer (service-permitted)", "vendor": "Apple · SFSpeechRecognizer · service-permitted zh-CN", "color": "#68717e", "abbrev": "ALS", "cat": "api", "params": null, "llm": false }, "apple_legacy_ondevice": { "dur": null, "durb": "none-published", "win": null, "mb": null, "im": "both", "so": true, "hot": true, "hotm": "contextual_strings", "hotn": null, "bias": "none", "wt": "segment", "dia": [], "label": "Apple SFSpeechRecognizer (旧版 · pre-macOS 26)", "vendor": "Apple · SFSpeechRecognizer · 旧版端侧 zh-CN", "color": "#486581", "abbrev": "ALO", "cat": "local-system", "params": null, "llm": false }, "apple_speechanalyzer": { "dur": null, "durb": "none-published", "win": null, "mb": null, "im": "both", "so": true, "hot": false, "hotm": null, "hotn": null, "bias": "none", "wt": "segment", "dia": [], "label": "Apple SpeechAnalyzer (macOS 26)", "vendor": "Apple · SpeechAnalyzer / SpeechTranscriber · on-device zh-CN", "color": "#8b5e3c", "abbrev": "ASA", "cat": "local-system", "params": null, "llm": false }, "doubao": { "dur": 18000, "durb": "vendor-cap", "win": null, "mb": 512, "im": "file", "so": false, "hot": true, "hotm": "freeform_context", "hotn": null, "bias": "vocabulary", "wt": "word", "dia": [ "粤", "川", "沪" ], "label": "Doubao 2.0", "vendor": "ByteDance · Seed-ASR", "color": "#a82316", "abbrev": "Dou", "cat": "api", "params": null, "llm": false }, "elevenlabs": { "dur": 36000, "durb": "vendor-cap", "win": null, "mb": 3072, "im": "file", "so": false, "hot": true, "hotm": "keyterm_prompting", "hotn": 1000, "bias": "none", "wt": "word", "dia": [ "粤" ], "label": "Scribe v2", "vendor": "ElevenLabs · Scribe v2", "color": "#d5a75d", "abbrev": "Scr", "cat": "api", "params": null, "llm": false }, "firered": { "dur": 39, "durb": "measured-cap", "win": null, "mb": null, "im": "file", "so": false, "hot": false, "hotm": null, "hotn": null, "bias": "none", "wt": "none", "dia": [ "粤", "川" ], "label": "FireRedASR2-CTC", "vendor": "Xiaohongshu OSS · FireRedASR2-CTC (int8 · 本地 CPU)", "color": "#b3423a", "abbrev": "FR2C", "cat": "oss-small", "params": 0.78, "llm": false }, "firered_aed": { "dur": null, "durb": "native-window", "win": 60, "mb": null, "im": "file", "so": false, "hot": false, "hotm": null, "hotn": null, "bias": "none", "wt": "word", "dia": [ "粤", "吴", "闽南", "客", "赣", "湘", "晋", "川" ], "label": "FireRedASR2-AED", "vendor": "Xiaohongshu OSS · FireRedASR2-AED (1.15B · 本地 GPU)", "color": "#c0492b", "abbrev": "FR2A", "cat": "oss-small", "params": 1.15, "llm": false }, "flash": { "dur": 300, "durb": "vendor-cap", "win": null, "mb": 10, "im": "file", "so": true, "hot": true, "hotm": "freeform_context", "hotn": null, "bias": "none", "wt": "word", "dia": [ "粤", "川" ], "label": "Qwen3-ASR-Flash", "vendor": "Alibaba · qwen3-asr-flash-2026-02-10", "color": "#2b6b4f", "abbrev": "Qwn", "cat": "api", "params": null, "llm": false }, "funasr_nano": { "dur": null, "durb": "native-window", "win": 30, "mb": null, "im": "both", "so": false, "hot": true, "hotm": "hotwords", "hotn": null, "bias": "none", "wt": "none", "dia": [ "粤", "吴", "闽南", "客", "赣", "湘", "晋", "川" ], "label": "Fun-ASR-Nano", "vendor": "Alibaba/DAMO OSS · Fun-ASR-Nano", "color": "#8c6d4f", "abbrev": "Fun", "cat": "oss-small", "params": 0.6, "llm": false }, "gemini_3_5_flash_lite_stt": { "dur": 34200, "durb": "vendor-cap", "win": null, "mb": 2048, "im": "file", "so": true, "hot": false, "hotm": "freeform_context", "hotn": null, "bias": "formatting", "wt": "segment", "dia": [], "label": "Gemini 3.5 Flash-Lite (LLM-as-ASR)", "vendor": "Google · gemini-3.5-flash-lite · multimodal 直收音频", "color": "#6f86d8", "abbrev": "G35L", "cat": "api", "params": null, "llm": true }, "gemini_3_5_flash_stt": { "dur": 34200, "durb": "vendor-cap", "win": null, "mb": 2048, "im": "file", "so": true, "hot": false, "hotm": "freeform_context", "hotn": null, "bias": "formatting", "wt": "segment", "dia": [], "label": "Gemini 3.5 Flash (LLM-as-ASR)", "vendor": "Google · gemini-3.5-flash · multimodal 直收音频", "color": "#3d74c4", "abbrev": "G35", "cat": "api", "params": null, "llm": true }, "gemini_3_5_transcribe": { "dur": 3600, "durb": "vendor-cap", "win": null, "mb": null, "im": "file", "so": false, "hot": true, "hotm": "custom_vocabulary", "hotn": 1000, "bias": "formatting", "wt": "word", "dia": [], "label": "Gemini 3.5 Transcribe", "vendor": "Google · gemini-3.5-transcribe · 专用语音模型 (不是 LLM-as-ASR)", "color": "#0e7c9e", "abbrev": "G35T", "cat": "api", "params": null, "llm": false }, "gemini_3_6_flash_stt": { "dur": 34200, "durb": "vendor-cap", "win": null, "mb": 2048, "im": "file", "so": true, "hot": false, "hotm": "freeform_context", "hotn": null, "bias": "formatting", "wt": "segment", "dia": [], "label": "Gemini 3.6 Flash (LLM-as-ASR)", "vendor": "Google · gemini-3.6-flash · multimodal 直收音频", "color": "#3c62d9", "abbrev": "G36", "cat": "api", "params": null, "llm": true }, "gemini_3_7_flash_stt": { "dur": 34200, "durb": "vendor-cap", "win": null, "mb": 2048, "im": "file", "so": true, "hot": false, "hotm": "freeform_context", "hotn": null, "bias": "formatting", "wt": "segment", "dia": [], "label": "Gemini 3.7 Flash (LLM-as-ASR · thinkingLevel=low)", "vendor": "Google · gemini-3.7-flash · multimodal 直收音频", "color": "#2b4bb8", "abbrev": "G37", "cat": "api", "params": null, "llm": true }, "gemini_3_8_flash_stt": { "dur": 34200, "durb": "vendor-cap", "win": null, "mb": 2048, "im": "file", "so": true, "hot": false, "hotm": "freeform_context", "hotn": null, "bias": "formatting", "wt": "segment", "dia": [], "label": "Gemini 3.8 Flash (LLM-as-ASR · thinkingLevel=low)", "vendor": "Google · gemini-3.8-flash · multimodal 直收音频", "color": "#1e4fd0", "abbrev": "G38", "cat": "api", "params": null, "llm": true }, "gemini_flash_stt": { "dur": 34200, "durb": "vendor-cap", "win": null, "mb": 2048, "im": "file", "so": true, "hot": false, "hotm": "freeform_context", "hotn": null, "bias": "formatting", "wt": "segment", "dia": [], "label": "Gemini 3 Flash (LLM-as-ASR)", "vendor": "Google · gemini-3-flash-preview · multimodal 直收音频", "color": "#5b9bd5", "abbrev": "G3F", "cat": "api", "params": null, "llm": true }, "gemini_pro_stt": { "dur": 34200, "durb": "vendor-cap", "win": null, "mb": 2048, "im": "file", "so": true, "hot": false, "hotm": "freeform_context", "hotn": null, "bias": "formatting", "wt": "segment", "dia": [], "label": "Gemini 3 Pro (LLM-as-ASR · thinkingLevel=low)", "vendor": "Google · gemini-3-pro-preview · thinkingLevel=low · multimodal", "color": "#1e5fa0", "abbrev": "G3P", "cat": "api", "params": null, "llm": true }, "gemini_stt": { "dur": 34200, "durb": "vendor-cap", "win": null, "mb": 2048, "im": "file", "so": true, "hot": false, "hotm": "freeform_context", "hotn": null, "bias": "formatting", "wt": "segment", "dia": [], "label": "Gemini 3.1 Flash Lite (LLM-as-ASR)", "vendor": "Google · gemini-3.1-flash-lite-preview · multimodal 直收音频", "color": "#4587e8", "abbrev": "Gem", "cat": "api", "params": null, "llm": true }, "gemma4": { "dur": null, "durb": "native-window", "win": 30, "mb": null, "im": "file", "so": false, "hot": false, "hotm": null, "hotn": null, "bias": "unknown", "wt": "none", "dia": [], "label": "Gemma 4 12B-it", "vendor": "Google OSS · Gemma 4 12B-it (LLM-as-ASR)", "color": "#8a6a4a", "abbrev": "G4", "cat": "oss-large", "params": 12.0, "llm": true }, "glm_asr_nano": { "dur": null, "durb": "none-published", "win": null, "mb": null, "im": "file", "so": false, "hot": false, "hotm": null, "hotn": null, "bias": "none", "wt": "none", "dia": [ "粤" ], "label": "GLM-ASR-Nano", "vendor": "Zhipu OSS · GLM-ASR-Nano (1.5B)", "color": "#a64ca6", "abbrev": "GLM", "cat": "oss-small", "params": 1.5, "llm": false }, "gpt_transcribe": { "dur": null, "durb": "untested", "win": null, "mb": 25, "im": "file", "so": true, "hot": true, "hotm": "keyword_hints", "hotn": null, "bias": "vocabulary", "wt": "none", "dia": [], "label": "GPT Transcribe", "vendor": "OpenAI · gpt-transcribe (带词表提示)", "color": "#b5399b", "abbrev": "gptT", "cat": "api", "params": null, "llm": false }, "gpt_transcribe_noprompt": { "dur": null, "durb": "untested", "win": null, "mb": 25, "im": "file", "so": true, "hot": true, "hotm": "keyword_hints", "hotn": null, "bias": "none", "wt": "none", "dia": [], "label": "GPT Transcribe (no hint)", "vendor": "OpenAI · gpt-transcribe (无词表提示)", "color": "#d4759e", "abbrev": "gptT-nh", "cat": "api", "params": null, "llm": false }, "mai_transcribe_2": { "dur": null, "durb": "vendor-cap", "win": null, "mb": 300, "im": "file", "so": false, "hot": true, "hotm": "phrase_list", "hotn": null, "bias": "none", "wt": "none", "dia": [ "yue" ], "label": "MAI-Transcribe-2", "vendor": "Microsoft · MAI-Transcribe-2 (via OpenRouter, 无 keyword 偏置)", "color": "#3b7a3b", "abbrev": "MAI2", "cat": "api", "params": null, "llm": false }, "mimo": { "dur": null, "durb": "none-published", "win": null, "mb": null, "im": "file", "so": false, "hot": false, "hotm": null, "hotn": null, "bias": "none", "wt": "none", "dia": [ "粤", "吴", "闽南", "川" ], "label": "MiMo-V2.5-ASR", "vendor": "Xiaomi · MiMo-V2.5-ASR (8B)", "color": "#7c4eaa", "abbrev": "MiMo", "cat": "oss-large", "params": 8.0, "llm": false }, "mimo_stt": { "dur": null, "durb": "none-published", "win": null, "mb": null, "im": "file", "so": false, "hot": false, "hotm": "freeform_context", "hotn": null, "bias": "formatting", "wt": "none", "dia": [], "label": "MiMo V2.5 (LLM-as-ASR)", "vendor": "Xiaomi · mimo-v2.5 · multimodal", "color": "#e0a530", "abbrev": "MiM", "cat": "api", "params": null, "llm": true }, "moss_audio": { "dur": null, "durb": "none-published", "win": null, "mb": null, "im": "file", "so": false, "hot": false, "hotm": null, "hotn": null, "bias": "formatting", "wt": "word", "dia": [ "粤" ], "label": "MOSS-Audio-8B-Instruct (LLM-as-ASR)", "vendor": "OpenMOSS / MOSI.AI OSS · MOSS-Audio-8B-Instruct (Apache 2.0 · audio-understanding LLM)", "color": "#6e8e3a", "abbrev": "MOSS", "cat": "oss-large", "params": 8.6, "llm": true }, "qwen3_oss": { "dur": null, "durb": "none-published", "win": null, "mb": null, "im": "both", "so": false, "hot": false, "hotm": null, "hotn": null, "bias": "none", "wt": "word", "dia": [ "粤", "吴", "闽南", "川", "东北" ], "label": "Qwen3-ASR-1.7B", "vendor": "Alibaba OSS · Qwen3-ASR-1.7B", "color": "#a0522d", "abbrev": "Q3O", "cat": "oss-small", "params": 1.7, "llm": false }, "qwen_audio_3_1_asr": { "dur": null, "durb": "untested", "win": null, "mb": null, "im": "file", "so": false, "hot": false, "hotm": "freeform_context", "hotn": null, "bias": "none", "wt": "word", "dia": [], "label": "Qwen-Audio-3.1-ASR", "vendor": "Alibaba · qwen-audio-3.1-asr-flash", "color": "#3f9e74", "abbrev": "QA31", "cat": "api", "params": null, "llm": false }, "r2t2": { "dur": null, "durb": "none-published", "win": null, "mb": null, "im": "both", "so": false, "hot": false, "hotm": "freeform_context", "hotn": null, "bias": "none", "wt": "none", "dia": [], "label": "Confucius4-R2T2", "vendor": "NetEase Youdao · Confucius4-R2T2 (Qwen3-ASR-1.7B 微调)", "color": "#9c6b1f", "abbrev": "R2T2", "cat": "oss-small", "params": 1.7, "llm": false }, "soniox": { "dur": 18000, "durb": "vendor-cap", "win": null, "mb": null, "im": "file", "so": false, "hot": true, "hotm": "freeform_context", "hotn": null, "bias": "none", "wt": "token", "dia": [], "label": "Soniox stt-async-v5", "vendor": "Soniox · stt-async-v5 (无 hotword 偏置)", "color": "#00a6a6", "abbrev": "Snx", "cat": "api", "params": null, "llm": false }, "stepfun": { "dur": 1800, "durb": "vendor-cap", "win": null, "mb": null, "im": "file", "so": true, "hot": true, "hotm": "hotwords", "hotn": null, "bias": "none", "wt": "none", "dia": [ "川" ], "label": "Step-Audio 2.5 ASR", "vendor": "StepFun · stepaudio-2.5-asr (4B · 0.6B enc + 4B Qwen + MTP-5)", "color": "#2a6f7a", "abbrev": "Step", "cat": "api", "params": 4.0, "llm": false }, "vibevoice": { "dur": 3600, "durb": "vendor-cap", "win": null, "mb": null, "im": "file", "so": false, "hot": true, "hotm": "hotwords", "hotn": null, "bias": "none", "wt": "segment", "dia": [], "label": "VibeVoice-ASR", "vendor": "Microsoft · VibeVoice-ASR (9B)", "color": "#e07b39", "abbrev": "Vibe", "cat": "oss-large", "params": 9.0, "llm": false }, "voxtral": { "dur": null, "durb": "context-derived", "win": null, "mb": null, "im": "both", "so": false, "hot": false, "hotm": null, "hotn": null, "bias": "none", "wt": "none", "dia": [], "label": "Voxtral-Mini-4B-Realtime", "vendor": "Mistral OSS · Voxtral-Mini-4B-Realtime", "color": "#d97c5e", "abbrev": "Vox", "cat": "oss-mid", "params": 4.0, "llm": true }, "whisper": { "dur": 1500, "durb": "vendor-cap", "win": null, "mb": 25, "im": "file", "so": true, "hot": false, "hotm": "freeform_context", "hotn": null, "bias": "vocabulary", "wt": "none", "dia": [], "label": "GPT-4o-mini-transcribe", "vendor": "OpenAI · gpt-4o-mini-transcribe", "color": "#6e7c9a", "abbrev": "4oT", "cat": "api", "params": null, "llm": false }, "whisper_v3_turbo": { "dur": null, "durb": "native-window", "win": 30, "mb": null, "im": "file", "so": false, "hot": false, "hotm": "freeform_context", "hotn": null, "bias": "formatting", "wt": "word", "dia": [], "label": "Whisper-Large-v3-turbo", "vendor": "OpenAI OSS · whisper-large-v3-turbo (810M)", "color": "#4a5568", "abbrev": "v3T", "cat": "oss-small", "params": 0.81, "llm": false } }, "_judges": [ "deepseek_flash" ], "_rubric_versions": [ "v6.8", "v6.9" ] }