{
  "tasks": 152,
  "task_ids": [
    0,
    1,
    2,
    3,
    4,
    5,
    6,
    7,
    8,
    9,
    10,
    11,
    13,
    14,
    15,
    16,
    17,
    18,
    19,
    20,
    21,
    23,
    24,
    25,
    26,
    27,
    28,
    29,
    30,
    31,
    33,
    34,
    35,
    36,
    37,
    39,
    40,
    41,
    42,
    43,
    44,
    45,
    46,
    47,
    48,
    49,
    51,
    52,
    53,
    54,
    55,
    56,
    57,
    58,
    59,
    60,
    61,
    62,
    63,
    64,
    65,
    66,
    67,
    68,
    69,
    70,
    71,
    72,
    73,
    74,
    75,
    76,
    77,
    78,
    79,
    80,
    81,
    82,
    83,
    84,
    85,
    86,
    87,
    88,
    89,
    91,
    92,
    93,
    94,
    95,
    96,
    97,
    98,
    99,
    100,
    101,
    102,
    104,
    105,
    106,
    107,
    108,
    109,
    110,
    111,
    112,
    113,
    114,
    115,
    116,
    117,
    118,
    119,
    120,
    121,
    122,
    123,
    124,
    126,
    127,
    129,
    130,
    131,
    132,
    133,
    134,
    135,
    136,
    138,
    139,
    140,
    141,
    142,
    143,
    144,
    145,
    146,
    147,
    149,
    150,
    151,
    152,
    153,
    154,
    155,
    156,
    157,
    158,
    159,
    160,
    161,
    163
  ],
  "model": "qwen2.5-coder:1.5b",
  "arms": {
    "python": {
      "first_pass": 89,
      "first_pass_rate": 0.5855,
      "first_pass_ci95": [
        0.506,
        0.6608
      ],
      "first_gate_reject": 3,
      "final_pass": 90,
      "final_pass_rate": 0.5921,
      "final_pass_ci95": [
        0.5126,
        0.667
      ],
      "gained_by_feedback": 1,
      "lost_by_feedback": 0,
      "mcnemar_p_first_vs_final": 1.0,
      "first_wrong": 63,
      "first_wrong_caught_by_gate": 3,
      "catch_rate": 0.0476,
      "catch_rate_ci95": [
        0.0163,
        0.1309
      ],
      "gate_rejects_that_would_pass": 0,
      "gate_runnable_when_rejected": true,
      "final_gate_ok": 151,
      "final_silent_wrong": 61,
      "final_still_rejected": 1,
      "pass_given_gate_ok": 0.596,
      "repair_rounds": 4,
      "extract_failures": 0,
      "output_tokens": 9039,
      "tokens_per_final_pass": 100.4,
      "first_rejection_kinds": {
        "SyntaxError": 3
      },
      "first_rejection_classes": {
        "syntax": 3
      },
      "first_wrong_caught_semantic": 0,
      "semantic_catch_rate": 0.0,
      "semantic_catch_ci95": [
        0.0,
        0.0575
      ],
      "first_attempt_tokens": 7863,
      "shipped_correct_no_feedback": 89,
      "shipped_correct_no_feedback_ci95": [
        0.506,
        0.6608
      ],
      "shipped_correct": 90,
      "shipped_correct_rate": 0.5921,
      "shipped_correct_ci95": [
        0.5126,
        0.667
      ],
      "shipped_correct_first_try": 89,
      "feedback_gains_by_first_rejection_class": {
        "syntax": 1
      },
      "tokens_per_shipped_correct": 100.4,
      "shipped_correct_after_repair": 1,
      "rejected_but_tests_pass_final": 0
    },
    "python+mypy": {
      "first_pass": 89,
      "first_pass_rate": 0.5855,
      "first_pass_ci95": [
        0.506,
        0.6608
      ],
      "first_gate_reject": 13,
      "final_pass": 95,
      "final_pass_rate": 0.625,
      "final_pass_ci95": [
        0.5458,
        0.698
      ],
      "gained_by_feedback": 6,
      "lost_by_feedback": 0,
      "mcnemar_p_first_vs_final": 0.03125,
      "first_wrong": 63,
      "first_wrong_caught_by_gate": 10,
      "catch_rate": 0.1587,
      "catch_rate_ci95": [
        0.0886,
        0.2681
      ],
      "gate_rejects_that_would_pass": 3,
      "gate_runnable_when_rejected": true,
      "final_gate_ok": 148,
      "final_silent_wrong": 54,
      "final_still_rejected": 4,
      "pass_given_gate_ok": 0.6419,
      "repair_rounds": 17,
      "extract_failures": 0,
      "output_tokens": 10758,
      "tokens_per_final_pass": 113.2,
      "first_rejection_kinds": {
        "mypy:name-defined": 5,
        "SyntaxError": 3,
        "mypy:return-value": 1,
        "mypy:assignment": 1,
        "mypy:return": 1,
        "mypy:arg-type": 1,
        "mypy:var-annotated": 1
      },
      "first_rejection_classes": {
        "syntax": 3,
        "semantic": 7,
        "import": 3
      },
      "first_wrong_caught_semantic": 4,
      "semantic_catch_rate": 0.0635,
      "semantic_catch_ci95": [
        0.025,
        0.1522
      ],
      "first_attempt_tokens": 7863,
      "shipped_correct_no_feedback": 86,
      "shipped_correct_no_feedback_ci95": [
        0.4863,
        0.642
      ],
      "shipped_correct": 94,
      "shipped_correct_rate": 0.6184,
      "shipped_correct_ci95": [
        0.5392,
        0.6918
      ],
      "shipped_correct_first_try": 86,
      "feedback_gains_by_first_rejection_class": {
        "semantic": 4,
        "import": 3,
        "syntax": 1
      },
      "tokens_per_shipped_correct": 114.4,
      "shipped_correct_after_repair": 8,
      "rejected_but_tests_pass_final": 1
    },
    "javascript": {
      "first_pass": 86,
      "first_pass_rate": 0.5658,
      "first_pass_ci95": [
        0.4863,
        0.642
      ],
      "first_gate_reject": 3,
      "final_pass": 87,
      "final_pass_rate": 0.5724,
      "final_pass_ci95": [
        0.4929,
        0.6483
      ],
      "gained_by_feedback": 1,
      "lost_by_feedback": 0,
      "mcnemar_p_first_vs_final": 1.0,
      "first_wrong": 66,
      "first_wrong_caught_by_gate": 3,
      "catch_rate": 0.0455,
      "catch_rate_ci95": [
        0.0156,
        0.1253
      ],
      "gate_rejects_that_would_pass": 0,
      "gate_runnable_when_rejected": true,
      "final_gate_ok": 151,
      "final_silent_wrong": 64,
      "final_still_rejected": 1,
      "pass_given_gate_ok": 0.5762,
      "repair_rounds": 4,
      "extract_failures": 0,
      "output_tokens": 10771,
      "tokens_per_final_pass": 123.8,
      "first_rejection_kinds": {
        "other": 3
      },
      "first_rejection_classes": {
        "syntax": 3
      },
      "first_wrong_caught_semantic": 0,
      "semantic_catch_rate": 0.0,
      "semantic_catch_ci95": [
        0.0,
        0.055
      ],
      "first_attempt_tokens": 10417,
      "shipped_correct_no_feedback": 86,
      "shipped_correct_no_feedback_ci95": [
        0.4863,
        0.642
      ],
      "shipped_correct": 87,
      "shipped_correct_rate": 0.5724,
      "shipped_correct_ci95": [
        0.4929,
        0.6483
      ],
      "shipped_correct_first_try": 86,
      "feedback_gains_by_first_rejection_class": {
        "syntax": 1
      },
      "tokens_per_shipped_correct": 123.8,
      "shipped_correct_after_repair": 1,
      "rejected_but_tests_pass_final": 0
    },
    "typescript": {
      "first_pass": 56,
      "first_pass_rate": 0.3684,
      "first_pass_ci95": [
        0.2959,
        0.4475
      ],
      "first_gate_reject": 63,
      "final_pass": 96,
      "final_pass_rate": 0.6316,
      "final_pass_ci95": [
        0.5525,
        0.7041
      ],
      "gained_by_feedback": 40,
      "lost_by_feedback": 0,
      "mcnemar_p_first_vs_final": 0.0,
      "first_wrong": 96,
      "first_wrong_caught_by_gate": 60,
      "catch_rate": 0.625,
      "catch_rate_ci95": [
        0.5251,
        0.7153
      ],
      "gate_rejects_that_would_pass": 3,
      "gate_runnable_when_rejected": true,
      "final_gate_ok": 145,
      "final_silent_wrong": 52,
      "final_still_rejected": 7,
      "pass_given_gate_ok": 0.6621,
      "repair_rounds": 70,
      "extract_failures": 0,
      "output_tokens": 31130,
      "tokens_per_final_pass": 324.3,
      "first_rejection_kinds": {
        "TS1160": 31,
        "TS1005": 8,
        "TS2304": 7,
        "TS1351": 5,
        "TS1002": 5,
        "TS2345": 1,
        "TS7053": 1,
        "TS2322": 1,
        "TS1443": 1,
        "TS2532": 1,
        "TS2769": 1,
        "TS2552": 1
      },
      "first_rejection_classes": {
        "syntax": 50,
        "semantic": 13
      },
      "first_wrong_caught_semantic": 10,
      "semantic_catch_rate": 0.1042,
      "semantic_catch_ci95": [
        0.0576,
        0.1812
      ],
      "first_attempt_tokens": 24257,
      "shipped_correct_no_feedback": 53,
      "shipped_correct_no_feedback_ci95": [
        0.2775,
        0.4273
      ],
      "shipped_correct": 93,
      "shipped_correct_rate": 0.6118,
      "shipped_correct_ci95": [
        0.5325,
        0.6856
      ],
      "shipped_correct_first_try": 53,
      "feedback_gains_by_first_rejection_class": {
        "syntax": 34,
        "semantic": 6
      },
      "tokens_per_shipped_correct": 334.7,
      "shipped_correct_after_repair": 40,
      "rejected_but_tests_pass_final": 3
    },
    "java": {
      "first_pass": 75,
      "first_pass_rate": 0.4934,
      "first_pass_ci95": [
        0.4151,
        0.5721
      ],
      "first_gate_reject": 36,
      "final_pass": 86,
      "final_pass_rate": 0.5658,
      "final_pass_ci95": [
        0.4863,
        0.642
      ],
      "gained_by_feedback": 11,
      "lost_by_feedback": 0,
      "mcnemar_p_first_vs_final": 0.000977,
      "first_wrong": 77,
      "first_wrong_caught_by_gate": 36,
      "catch_rate": 0.4675,
      "catch_rate_ci95": [
        0.3603,
        0.5778
      ],
      "gate_rejects_that_would_pass": 0,
      "gate_runnable_when_rejected": false,
      "final_gate_ok": 138,
      "final_silent_wrong": 52,
      "final_still_rejected": 14,
      "pass_given_gate_ok": 0.6232,
      "repair_rounds": 51,
      "extract_failures": 1,
      "output_tokens": 28332,
      "tokens_per_final_pass": 329.4,
      "first_rejection_kinds": {
        "cannot find symbol": 17,
        "incompatible types": 10,
        "missing return": 7,
        "other": 2
      },
      "first_rejection_classes": {
        "semantic": 35,
        "syntax": 1
      },
      "first_wrong_caught_semantic": 35,
      "semantic_catch_rate": 0.4545,
      "semantic_catch_ci95": [
        0.3481,
        0.5653
      ],
      "first_attempt_tokens": 10668,
      "shipped_correct_no_feedback": 75,
      "shipped_correct_no_feedback_ci95": [
        0.4151,
        0.5721
      ],
      "shipped_correct": 86,
      "shipped_correct_rate": 0.5658,
      "shipped_correct_ci95": [
        0.4863,
        0.642
      ],
      "shipped_correct_first_try": 75,
      "feedback_gains_by_first_rejection_class": {
        "semantic": 11
      },
      "tokens_per_shipped_correct": 329.4,
      "shipped_correct_after_repair": 11,
      "rejected_but_tests_pass_final": 0
    },
    "go": {
      "first_pass": 49,
      "first_pass_rate": 0.3224,
      "first_pass_ci95": [
        0.2532,
        0.4003
      ],
      "first_gate_reject": 68,
      "final_pass": 75,
      "final_pass_rate": 0.4934,
      "final_pass_ci95": [
        0.4151,
        0.5721
      ],
      "gained_by_feedback": 26,
      "lost_by_feedback": 0,
      "mcnemar_p_first_vs_final": 0.0,
      "first_wrong": 103,
      "first_wrong_caught_by_gate": 68,
      "catch_rate": 0.6602,
      "catch_rate_ci95": [
        0.5644,
        0.7444
      ],
      "gate_rejects_that_would_pass": 0,
      "gate_runnable_when_rejected": false,
      "final_gate_ok": 129,
      "final_silent_wrong": 54,
      "final_still_rejected": 23,
      "pass_given_gate_ok": 0.5814,
      "repair_rounds": 98,
      "extract_failures": 2,
      "output_tokens": 42072,
      "tokens_per_final_pass": 561.0,
      "first_rejection_kinds": {
        "undefined": 56,
        "other": 5,
        "expected": 4,
        "mismatched types": 2,
        "cannot use": 1
      },
      "first_rejection_classes": {
        "import": 37,
        "semantic": 27,
        "syntax": 4
      },
      "first_wrong_caught_semantic": 27,
      "semantic_catch_rate": 0.2621,
      "semantic_catch_ci95": [
        0.1869,
        0.3545
      ],
      "first_attempt_tokens": 13230,
      "shipped_correct_no_feedback": 49,
      "shipped_correct_no_feedback_ci95": [
        0.2532,
        0.4003
      ],
      "shipped_correct": 75,
      "shipped_correct_rate": 0.4934,
      "shipped_correct_ci95": [
        0.4151,
        0.5721
      ],
      "shipped_correct_first_try": 49,
      "feedback_gains_by_first_rejection_class": {
        "import": 14,
        "semantic": 11,
        "syntax": 1
      },
      "tokens_per_shipped_correct": 561.0,
      "shipped_correct_after_repair": 26,
      "rejected_but_tests_pass_final": 0
    },
    "rust": {
      "first_pass": 57,
      "first_pass_rate": 0.375,
      "first_pass_ci95": [
        0.302,
        0.4542
      ],
      "first_gate_reject": 53,
      "final_pass": 60,
      "final_pass_rate": 0.3947,
      "final_pass_ci95": [
        0.3205,
        0.4741
      ],
      "gained_by_feedback": 3,
      "lost_by_feedback": 0,
      "mcnemar_p_first_vs_final": 0.25,
      "first_wrong": 95,
      "first_wrong_caught_by_gate": 53,
      "catch_rate": 0.5579,
      "catch_rate_ci95": [
        0.4577,
        0.6536
      ],
      "gate_rejects_that_would_pass": 0,
      "gate_runnable_when_rejected": false,
      "final_gate_ok": 109,
      "final_silent_wrong": 49,
      "final_still_rejected": 43,
      "pass_given_gate_ok": 0.5505,
      "repair_rounds": 96,
      "extract_failures": 0,
      "output_tokens": 24615,
      "tokens_per_final_pass": 410.2,
      "first_rejection_kinds": {
        "E0308": 16,
        "E0277": 9,
        "E0599": 9,
        "E0425": 7,
        "E0382": 4,
        "E0412": 2,
        "E0384": 2,
        "E0596": 2,
        "E0502": 1,
        "E0433": 1
      },
      "first_rejection_classes": {
        "semantic": 53
      },
      "first_wrong_caught_semantic": 53,
      "semantic_catch_rate": 0.5579,
      "semantic_catch_ci95": [
        0.4577,
        0.6536
      ],
      "first_attempt_tokens": 10079,
      "shipped_correct_no_feedback": 57,
      "shipped_correct_no_feedback_ci95": [
        0.302,
        0.4542
      ],
      "shipped_correct": 60,
      "shipped_correct_rate": 0.3947,
      "shipped_correct_ci95": [
        0.3205,
        0.4741
      ],
      "shipped_correct_first_try": 57,
      "feedback_gains_by_first_rejection_class": {
        "semantic": 3
      },
      "tokens_per_shipped_correct": 410.2,
      "shipped_correct_after_repair": 3,
      "rejected_but_tests_pass_final": 0
    }
  },
  "paired_final": {
    "python vs python+mypy": {
      "only_first_passes": 1,
      "only_second_passes": 5,
      "mcnemar_p": 0.21875
    },
    "javascript vs typescript": {
      "only_first_passes": 11,
      "only_second_passes": 17,
      "mcnemar_p": 0.344928
    },
    "python vs rust": {
      "only_first_passes": 43,
      "only_second_passes": 13,
      "mcnemar_p": 7.3e-05
    },
    "python vs typescript": {
      "only_first_passes": 20,
      "only_second_passes": 23,
      "mcnemar_p": 0.760792
    },
    "python vs go": {
      "only_first_passes": 28,
      "only_second_passes": 13,
      "mcnemar_p": 0.027533
    },
    "python vs java": {
      "only_first_passes": 22,
      "only_second_passes": 18,
      "mcnemar_p": 0.635828
    },
    "typescript vs rust": {
      "only_first_passes": 40,
      "only_second_passes": 7,
      "mcnemar_p": 1e-06
    }
  },
  "all_pairs_holm": [
    {
      "a": "python",
      "b": "python+mypy",
      "only_a": 1,
      "only_b": 5,
      "p": 0.21875,
      "p_holm": 1.0
    },
    {
      "a": "python",
      "b": "javascript",
      "only_a": 22,
      "only_b": 19,
      "p": 0.755229,
      "p_holm": 1.0
    },
    {
      "a": "python",
      "b": "typescript",
      "only_a": 20,
      "only_b": 23,
      "p": 0.760792,
      "p_holm": 1.0
    },
    {
      "a": "python",
      "b": "java",
      "only_a": 22,
      "only_b": 18,
      "p": 0.635828,
      "p_holm": 1.0
    },
    {
      "a": "python",
      "b": "go",
      "only_a": 28,
      "only_b": 13,
      "p": 0.027533,
      "p_holm": 0.385462
    },
    {
      "a": "python",
      "b": "rust",
      "only_a": 43,
      "only_b": 13,
      "p": 7.3e-05,
      "p_holm": 0.001314
    },
    {
      "a": "python+mypy",
      "b": "javascript",
      "only_a": 25,
      "only_b": 18,
      "p": 0.360378,
      "p_holm": 1.0
    },
    {
      "a": "python+mypy",
      "b": "typescript",
      "only_a": 20,
      "only_b": 19,
      "p": 1.0,
      "p_holm": 1.0
    },
    {
      "a": "python+mypy",
      "b": "java",
      "only_a": 24,
      "only_b": 16,
      "p": 0.268187,
      "p_holm": 1.0
    },
    {
      "a": "python+mypy",
      "b": "go",
      "only_a": 31,
      "only_b": 12,
      "p": 0.005402,
      "p_holm": 0.086432
    },
    {
      "a": "python+mypy",
      "b": "rust",
      "only_a": 45,
      "only_b": 11,
      "p": 5e-06,
      "p_holm": 0.0001
    },
    {
      "a": "javascript",
      "b": "typescript",
      "only_a": 11,
      "only_b": 17,
      "p": 0.344928,
      "p_holm": 1.0
    },
    {
      "a": "javascript",
      "b": "java",
      "only_a": 21,
      "only_b": 20,
      "p": 1.0,
      "p_holm": 1.0
    },
    {
      "a": "javascript",
      "b": "go",
      "only_a": 26,
      "only_b": 14,
      "p": 0.08069,
      "p_holm": 0.96828
    },
    {
      "a": "javascript",
      "b": "rust",
      "only_a": 34,
      "only_b": 7,
      "p": 2.5e-05,
      "p_holm": 0.000475
    },
    {
      "a": "typescript",
      "b": "java",
      "only_a": 25,
      "only_b": 18,
      "p": 0.360378,
      "p_holm": 1.0
    },
    {
      "a": "typescript",
      "b": "go",
      "only_a": 32,
      "only_b": 14,
      "p": 0.011352,
      "p_holm": 0.17028
    },
    {
      "a": "typescript",
      "b": "rust",
      "only_a": 40,
      "only_b": 7,
      "p": 1e-06,
      "p_holm": 2.1e-05
    },
    {
      "a": "java",
      "b": "go",
      "only_a": 23,
      "only_b": 12,
      "p": 0.089531,
      "p_holm": 0.984841
    },
    {
      "a": "java",
      "b": "rust",
      "only_a": 40,
      "only_b": 14,
      "p": 0.000535,
      "p_holm": 0.009095
    },
    {
      "a": "go",
      "b": "rust",
      "only_a": 31,
      "only_b": 16,
      "p": 0.039986,
      "p_holm": 0.519818
    }
  ],
  "sensitivity_ts_jsstops": {
    "arm": "typescript-jsstops",
    "stats": {
      "first_pass": 89,
      "first_pass_rate": 0.5855,
      "first_pass_ci95": [
        0.506,
        0.6608
      ],
      "first_gate_reject": 14,
      "final_pass": 96,
      "final_pass_rate": 0.6316,
      "final_pass_ci95": [
        0.5525,
        0.7041
      ],
      "gained_by_feedback": 7,
      "lost_by_feedback": 0,
      "mcnemar_p_first_vs_final": 0.015625,
      "first_wrong": 63,
      "first_wrong_caught_by_gate": 11,
      "catch_rate": 0.1746,
      "catch_rate_ci95": [
        0.1004,
        0.2862
      ],
      "gate_rejects_that_would_pass": 3,
      "gate_runnable_when_rejected": true,
      "final_gate_ok": 146,
      "final_silent_wrong": 53,
      "final_still_rejected": 6,
      "pass_given_gate_ok": 0.6575,
      "repair_rounds": 20,
      "extract_failures": 0,
      "output_tokens": 13046,
      "tokens_per_final_pass": 135.9,
      "first_rejection_kinds": {
        "TS2304": 7,
        "TS2345": 1,
        "TS7053": 1,
        "TS2322": 1,
        "TS2365": 1,
        "TS2532": 1,
        "TS2769": 1,
        "TS2552": 1
      },
      "first_rejection_classes": {
        "semantic": 14
      },
      "first_wrong_caught_semantic": 11,
      "semantic_catch_rate": 0.1746,
      "semantic_catch_ci95": [
        0.1004,
        0.2862
      ],
      "first_attempt_tokens": 10954,
      "shipped_correct_no_feedback": 86,
      "shipped_correct_no_feedback_ci95": [
        0.4863,
        0.642
      ],
      "shipped_correct": 93,
      "shipped_correct_rate": 0.6118,
      "shipped_correct_ci95": [
        0.5325,
        0.6856
      ],
      "shipped_correct_first_try": 86,
      "feedback_gains_by_first_rejection_class": {
        "semantic": 7
      },
      "tokens_per_shipped_correct": 140.3,
      "shipped_correct_after_repair": 7,
      "rejected_but_tests_pass_final": 3
    },
    "first_completions_with_console_log": 0,
    "original_ts_first_completions_with_console_log": 110,
    "paired": {
      "javascript vs typescript-jsstops (no_feedback)": {
        "only_first_passes": 12,
        "only_second_passes": 12,
        "mcnemar_p": 1.0
      },
      "javascript vs typescript-jsstops (with_feedback)": {
        "only_first_passes": 12,
        "only_second_passes": 18,
        "mcnemar_p": 0.361595
      },
      "typescript vs typescript-jsstops (no_feedback)": {
        "only_first_passes": 0,
        "only_second_passes": 33,
        "mcnemar_p": 0.0
      },
      "typescript vs typescript-jsstops (with_feedback)": {
        "only_first_passes": 1,
        "only_second_passes": 1,
        "mcnemar_p": 1.0
      }
    }
  }
}