{
  "version": "groundlm-hard-v1",
  "collection_complete": true,
  "new_slots": 4800,
  "planned_slots": 4800,
  "attempts": 4871,
  "spent_usd": 17.44514990461,
  "cap_usd": 27,
  "remaining_cap_usd": 9.554850095390002,
  "uncertain_cost_upper_bound_usd": 0.306151615,
  "bootstrap_samples": 2000,
  "bootstrap_unit": "graph case, stratified by task and size",
  "models": {
    "deepseek/deepseek-v4.1-flash-1017": {
      "endpoint": "sail-research/fp4",
      "quantization": "fp4",
      "collected_slots": 960,
      "planned_slots": 960,
      "correct": 633,
      "completed_numeric_wrong": 19,
      "invalid_completed": 0,
      "cutoffs": 308,
      "provider_or_protocol_errors": 0,
      "cost_usd": 1.54677297461,
      "metrics": {
        "accuracy_original": {
          "value": 0.7333333333333333,
          "n_graphs": 120,
          "stratified_bootstrap_ci95": [
            0.6833333333333333,
            0.7916666666666666
          ],
          "ci95": [
            0.6478756061385266,
            0.8043154495327459
          ],
          "ci_method": "Wilson; graph unit, unconditional within the available denominator"
        },
        "accuracy_repeat": {
          "value": 0.6916666666666667,
          "n_graphs": 120,
          "stratified_bootstrap_ci95": [
            0.6416666666666667,
            0.7416666666666667
          ],
          "ci95": [
            0.6041700227619957,
            0.7672726301108352
          ],
          "ci_method": "Wilson; graph unit, unconditional within the available denominator"
        },
        "agreement_repeat": {
          "value": 0.9397590361445783,
          "n_graphs": 83,
          "stratified_bootstrap_ci95": [
            0.8888888888888888,
            0.9871794871794872
          ],
          "ci95": [
            0.8666157190676185,
            0.9739965989581174
          ],
          "ci_method": "Wilson; graph unit, unconditional within the available denominator"
        },
        "survival_repeat": {
          "value": 0.875,
          "n_graphs": 88,
          "stratified_bootstrap_ci95": [
            0.8131868131868132,
            0.9348505434782606
          ],
          "ci95": [
            0.789882441523256,
            0.9287472584780764
          ],
          "ci_method": "Wilson; graph unit, unconditional within the available denominator"
        },
        "returned_pair_coverage_repeat": {
          "value": 1.0,
          "n_graphs": 120,
          "stratified_bootstrap_ci95": [
            1.0,
            1.0
          ],
          "ci95": [
            0.9689808335812982,
            1
          ],
          "ci_method": "Wilson; graph unit, unconditional within the available denominator"
        },
        "parseable_pair_coverage_repeat": {
          "value": 0.6916666666666667,
          "n_graphs": 120,
          "stratified_bootstrap_ci95": [
            0.6416666666666667,
            0.7418749999999988
          ],
          "ci95": [
            0.6041700227619957,
            0.7672726301108352
          ],
          "ci_method": "Wilson; graph unit, unconditional within the available denominator"
        },
        "accuracy_neutral": {
          "value": 0.6166666666666667,
          "n_graphs": 120,
          "stratified_bootstrap_ci95": [
            0.5747916666666666,
            0.6583333333333333
          ],
          "ci95": [
            0.5273408053842072,
            0.698754722451429
          ],
          "ci_method": "Wilson; graph unit, unconditional within the available denominator"
        },
        "agreement_neutral": {
          "value": 0.9358974358974359,
          "n_graphs": 78,
          "stratified_bootstrap_ci95": [
            0.881578947368421,
            0.9868421052631579
          ],
          "ci95": [
            0.858564045368291,
            0.9723106846262084
          ],
          "ci_method": "Wilson; graph unit, unconditional within the available denominator"
        },
        "survival_neutral": {
          "value": 0.8181818181818182,
          "n_graphs": 88,
          "stratified_bootstrap_ci95": [
            0.75,
            0.8888888888888888
          ],
          "ci95": [
            0.7248774097917652,
            0.8848690023305776
          ],
          "ci_method": "Wilson; graph unit, unconditional within the available denominator"
        },
        "returned_pair_coverage_neutral": {
          "value": 1.0,
          "n_graphs": 120,
          "stratified_bootstrap_ci95": [
            1.0,
            1.0
          ],
          "ci95": [
            0.9689808335812982,
            1
          ],
          "ci_method": "Wilson; graph unit, unconditional within the available denominator"
        },
        "parseable_pair_coverage_neutral": {
          "value": 0.65,
          "n_graphs": 120,
          "stratified_bootstrap_ci95": [
            0.6083333333333333,
            0.6916666666666667
          ],
          "ci95": [
            0.5612132459772454,
            0.729481004097144
          ],
          "ci_method": "Wilson; graph unit, unconditional within the available denominator"
        },
        "accuracy_degree_aligned": {
          "value": 0.5833333333333334,
          "n_graphs": 120,
          "stratified_bootstrap_ci95": [
            0.5333333333333333,
            0.6333333333333333
          ],
          "ci95": [
            0.4938803270016448,
            0.6676164785952383
          ],
          "ci_method": "Wilson; graph unit, unconditional within the available denominator"
        },
        "agreement_degree_aligned": {
          "value": 1.0,
          "n_graphs": 70,
          "stratified_bootstrap_ci95": [
            1.0,
            1.0
          ],
          "ci95": [
            0.9479769375897358,
            1
          ],
          "ci_method": "Wilson; graph unit, unconditional within the available denominator"
        },
        "survival_degree_aligned": {
          "value": 0.7954545454545454,
          "n_graphs": 88,
          "stratified_bootstrap_ci95": [
            0.7252747252747253,
            0.8690476190476191
          ],
          "ci95": [
            0.6996803607394857,
            0.866512736231261
          ],
          "ci_method": "Wilson; graph unit, unconditional within the available denominator"
        },
        "returned_pair_coverage_degree_aligned": {
          "value": 1.0,
          "n_graphs": 120,
          "stratified_bootstrap_ci95": [
            1.0,
            1.0
          ],
          "ci95": [
            0.9689808335812982,
            1
          ],
          "ci_method": "Wilson; graph unit, unconditional within the available denominator"
        },
        "parseable_pair_coverage_degree_aligned": {
          "value": 0.5833333333333334,
          "n_graphs": 120,
          "stratified_bootstrap_ci95": [
            0.5333333333333333,
            0.6333333333333333
          ],
          "ci95": [
            0.4938803270016448,
            0.6676164785952383
          ],
          "ci_method": "Wilson; graph unit, unconditional within the available denominator"
        },
        "accuracy_degree_reversed": {
          "value": 0.5666666666666667,
          "n_graphs": 120,
          "stratified_bootstrap_ci95": [
            0.5166666666666667,
            0.6166666666666667
          ],
          "ci95": [
            0.4772992088553929,
            0.6518982356221135
          ],
          "ci_method": "Wilson; graph unit, unconditional within the available denominator"
        },
        "agreement_degree_reversed": {
          "value": 0.9848484848484849,
          "n_graphs": 66,
          "stratified_bootstrap_ci95": [
            0.9538461538461539,
            1.0
          ],
          "ci95": [
            0.9190407973793389,
            0.9973203594560178
          ],
          "ci_method": "Wilson; graph unit, unconditional within the available denominator"
        },
        "survival_degree_reversed": {
          "value": 0.7386363636363636,
          "n_graphs": 88,
          "stratified_bootstrap_ci95": [
            0.6666666666666666,
            0.8160919540229885
          ],
          "ci95": [
            0.6382422002911167,
            0.8190676088006404
          ],
          "ci_method": "Wilson; graph unit, unconditional within the available denominator"
        },
        "returned_pair_coverage_degree_reversed": {
          "value": 1.0,
          "n_graphs": 120,
          "stratified_bootstrap_ci95": [
            1.0,
            1.0
          ],
          "ci95": [
            0.9689808335812982,
            1
          ],
          "ci_method": "Wilson; graph unit, unconditional within the available denominator"
        },
        "parseable_pair_coverage_degree_reversed": {
          "value": 0.55,
          "n_graphs": 120,
          "stratified_bootstrap_ci95": [
            0.5,
            0.6083333333333333
          ],
          "ci95": [
            0.46081543419687815,
            0.6360826491612517
          ],
          "ci_method": "Wilson; graph unit, unconditional within the available denominator"
        },
        "accuracy_format_json": {
          "value": 0.725,
          "n_graphs": 120,
          "stratified_bootstrap_ci95": [
            0.6666666666666666,
            0.7833333333333333
          ],
          "ci95": [
            0.6390703417242344,
            0.7969710333873498
          ],
          "ci_method": "Wilson; graph unit, unconditional within the available denominator"
        },
        "agreement_format_json": {
          "value": 0.9411764705882353,
          "n_graphs": 85,
          "stratified_bootstrap_ci95": [
            0.8888576779026217,
            0.9876543209876543
          ],
          "ci95": [
            0.869585615491605,
            0.9746148399665869
          ],
          "ci_method": "Wilson; graph unit, unconditional within the available denominator"
        },
        "survival_format_json": {
          "value": 0.9090909090909091,
          "n_graphs": 88,
          "stratified_bootstrap_ci95": [
            0.8526226076555024,
            0.9647161422708618
          ],
          "ci95": [
            0.8307460942048943,
            0.953213578523832
          ],
          "ci_method": "Wilson; graph unit, unconditional within the available denominator"
        },
        "returned_pair_coverage_format_json": {
          "value": 1.0,
          "n_graphs": 120,
          "stratified_bootstrap_ci95": [
            1.0,
            1.0
          ],
          "ci95": [
            0.9689808335812982,
            1
          ],
          "ci_method": "Wilson; graph unit, unconditional within the available denominator"
        },
        "parseable_pair_coverage_format_json": {
          "value": 0.7083333333333334,
          "n_graphs": 120,
          "stratified_bootstrap_ci95": [
            0.6583333333333333,
            0.7666666666666667
          ],
          "ci95": [
            0.6215576502025948,
            0.7821843637896129
          ],
          "ci_method": "Wilson; graph unit, unconditional within the available denominator"
        },
        "accuracy_order_shuffled": {
          "value": 0.6416666666666667,
          "n_graphs": 120,
          "stratified_bootstrap_ci95": [
            0.5916666666666667,
            0.6833333333333333
          ],
          "ci95": [
            0.5527053600179038,
            0.7218392094967975
          ],
          "ci_method": "Wilson; graph unit, unconditional within the available denominator"
        },
        "agreement_order_shuffled": {
          "value": 0.9615384615384616,
          "n_graphs": 78,
          "stratified_bootstrap_ci95": [
            0.9135802469135802,
            1.0
          ],
          "ci95": [
            0.8929155395249844,
            0.9868341745868385
          ],
          "ci_method": "Wilson; graph unit, unconditional within the available denominator"
        },
        "survival_order_shuffled": {
          "value": 0.8522727272727273,
          "n_graphs": 88,
          "stratified_bootstrap_ci95": [
            0.7888725490196078,
            0.9166666666666666
          ],
          "ci95": [
            0.7634888952813103,
            0.9115874895684263
          ],
          "ci_method": "Wilson; graph unit, unconditional within the available denominator"
        },
        "returned_pair_coverage_order_shuffled": {
          "value": 1.0,
          "n_graphs": 120,
          "stratified_bootstrap_ci95": [
            1.0,
            1.0
          ],
          "ci95": [
            0.9689808335812982,
            1
          ],
          "ci_method": "Wilson; graph unit, unconditional within the available denominator"
        },
        "parseable_pair_coverage_order_shuffled": {
          "value": 0.65,
          "n_graphs": 120,
          "stratified_bootstrap_ci95": [
            0.6,
            0.6916666666666667
          ],
          "ci95": [
            0.5612132459772454,
            0.729481004097144
          ],
          "ci_method": "Wilson; graph unit, unconditional within the available denominator"
        },
        "accuracy_distractor": {
          "value": 0.7166666666666667,
          "n_graphs": 120,
          "stratified_bootstrap_ci95": [
            0.6583333333333333,
            0.775
          ],
          "ci95": [
            0.6302979754694622,
            0.7895937190824338
          ],
          "ci_method": "Wilson; graph unit, unconditional within the available denominator"
        },
        "agreement_distractor": {
          "value": 0.9523809523809523,
          "n_graphs": 84,
          "stratified_bootstrap_ci95": [
            0.9080459770114943,
            0.9882352941176471
          ],
          "ci95": [
            0.8838668496724192,
            0.9813282531905093
          ],
          "ci_method": "Wilson; graph unit, unconditional within the available denominator"
        },
        "survival_distractor": {
          "value": 0.9090909090909091,
          "n_graphs": 88,
          "stratified_bootstrap_ci95": [
            0.8522727272727273,
            0.963855421686747
          ],
          "ci95": [
            0.8307460942048943,
            0.953213578523832
          ],
          "ci_method": "Wilson; graph unit, unconditional within the available denominator"
        },
        "returned_pair_coverage_distractor": {
          "value": 1.0,
          "n_graphs": 120,
          "stratified_bootstrap_ci95": [
            1.0,
            1.0
          ],
          "ci95": [
            0.9689808335812982,
            1
          ],
          "ci_method": "Wilson; graph unit, unconditional within the available denominator"
        },
        "parseable_pair_coverage_distractor": {
          "value": 0.7,
          "n_graphs": 120,
          "stratified_bootstrap_ci95": [
            0.65,
            0.7502083333333321
          ],
          "ci95": [
            0.6128485756752667,
            0.7747437577572526
          ],
          "ci_method": "Wilson; graph unit, unconditional within the available denominator"
        },
        "matched_semantic_gap": {
          "value": -0.030303030303030304,
          "n_graphs": 66,
          "stratified_bootstrap_ci95": [
            -0.07352941176470588,
            0.0
          ],
          "ci95": [
            -0.07352941176470588,
            0.0
          ],
          "ci_method": "Task/size-stratified graph bootstrap"
        }
      },
      "by_task": {
        "weighted_shortest_path": {
          "correct": 156,
          "collected": 160,
          "planned": 160
        },
        "shortest_path_count": {
          "correct": 88,
          "collected": 160,
          "planned": 160
        },
        "exact_walk_count": {
          "correct": 27,
          "collected": 160,
          "planned": 160
        },
        "triangle_count": {
          "correct": 65,
          "collected": 160,
          "planned": 160
        },
        "minimum_cut": {
          "correct": 138,
          "collected": 160,
          "planned": 160
        },
        "constrained_reachability": {
          "correct": 159,
          "collected": 160,
          "planned": 160
        }
      }
    },
    "z-ai/glm-5.3": {
      "endpoint": "baidu/fp8",
      "quantization": "fp8",
      "collected_slots": 960,
      "planned_slots": 960,
      "correct": 568,
      "completed_numeric_wrong": 20,
      "invalid_completed": 0,
      "cutoffs": 372,
      "provider_or_protocol_errors": 0,
      "cost_usd": 9.038421079999999,
      "metrics": {
        "accuracy_original": {
          "value": 0.6333333333333333,
          "n_graphs": 120,
          "stratified_bootstrap_ci95": [
            0.575,
            0.6916666666666667
          ],
          "ci95": [
            0.5442242408550724,
            0.7141706480999404
          ],
          "ci_method": "Wilson; graph unit, unconditional within the available denominator"
        },
        "accuracy_repeat": {
          "value": 0.6416666666666667,
          "n_graphs": 120,
          "stratified_bootstrap_ci95": [
            0.5916666666666667,
            0.6916666666666667
          ],
          "ci95": [
            0.5527053600179038,
            0.7218392094967975
          ],
          "ci_method": "Wilson; graph unit, unconditional within the available denominator"
        },
        "agreement_repeat": {
          "value": 0.958904109589041,
          "n_graphs": 73,
          "stratified_bootstrap_ci95": [
            0.9102272727272727,
            1.0
          ],
          "ci95": [
            0.8859990837370193,
            0.9859260596460788
          ],
          "ci_method": "Wilson; graph unit, unconditional within the available denominator"
        },
        "survival_repeat": {
          "value": 0.9210526315789473,
          "n_graphs": 76,
          "stratified_bootstrap_ci95": [
            0.8625,
            0.9736842105263158
          ],
          "ci95": [
            0.8382725651786221,
            0.9633159947665274
          ],
          "ci_method": "Wilson; graph unit, unconditional within the available denominator"
        },
        "returned_pair_coverage_repeat": {
          "value": 1.0,
          "n_graphs": 120,
          "stratified_bootstrap_ci95": [
            1.0,
            1.0
          ],
          "ci95": [
            0.9689808335812982,
            1
          ],
          "ci_method": "Wilson; graph unit, unconditional within the available denominator"
        },
        "parseable_pair_coverage_repeat": {
          "value": 0.6083333333333333,
          "n_graphs": 120,
          "stratified_bootstrap_ci95": [
            0.55,
            0.6666666666666666
          ],
          "ci95": [
            0.5189378169992755,
            0.6910080302766723
          ],
          "ci_method": "Wilson; graph unit, unconditional within the available denominator"
        },
        "accuracy_neutral": {
          "value": 0.6,
          "n_graphs": 120,
          "stratified_bootstrap_ci95": [
            0.5416666666666666,
            0.6583333333333333
          ],
          "ci95": [
            0.5105602479246331,
            0.6832359187916265
          ],
          "ci_method": "Wilson; graph unit, unconditional within the available denominator"
        },
        "agreement_neutral": {
          "value": 0.9420289855072463,
          "n_graphs": 69,
          "stratified_bootstrap_ci95": [
            0.8873239436619719,
            0.9855124223602485
          ],
          "ci95": [
            0.8602075664233865,
            0.9772276111196282
          ],
          "ci_method": "Wilson; graph unit, unconditional within the available denominator"
        },
        "survival_neutral": {
          "value": 0.8552631578947368,
          "n_graphs": 76,
          "stratified_bootstrap_ci95": [
            0.7875,
            0.918918918918919
          ],
          "ci95": [
            0.7591255101452979,
            0.917214837308422
          ],
          "ci_method": "Wilson; graph unit, unconditional within the available denominator"
        },
        "returned_pair_coverage_neutral": {
          "value": 1.0,
          "n_graphs": 120,
          "stratified_bootstrap_ci95": [
            1.0,
            1.0
          ],
          "ci95": [
            0.9689808335812982,
            1
          ],
          "ci_method": "Wilson; graph unit, unconditional within the available denominator"
        },
        "parseable_pair_coverage_neutral": {
          "value": 0.575,
          "n_graphs": 120,
          "stratified_bootstrap_ci95": [
            0.525,
            0.625
          ],
          "ci95": [
            0.48557751722993264,
            0.6597696078072621
          ],
          "ci_method": "Wilson; graph unit, unconditional within the available denominator"
        },
        "accuracy_degree_aligned": {
          "value": 0.5416666666666666,
          "n_graphs": 120,
          "stratified_bootstrap_ci95": [
            0.49166666666666664,
            0.5916666666666667
          ],
          "ci95": [
            0.45260969062370227,
            0.6281387121747393
          ],
          "ci_method": "Wilson; graph unit, unconditional within the available denominator"
        },
        "agreement_degree_aligned": {
          "value": 0.96875,
          "n_graphs": 64,
          "stratified_bootstrap_ci95": [
            0.921843998015873,
            1.0
          ],
          "ci95": [
            0.8930270822904198,
            0.9913878616538279
          ],
          "ci_method": "Wilson; graph unit, unconditional within the available denominator"
        },
        "survival_degree_aligned": {
          "value": 0.8157894736842105,
          "n_graphs": 76,
          "stratified_bootstrap_ci95": [
            0.7407407407407407,
            0.8888888888888888
          ],
          "ci95": [
            0.7142173789108939,
            0.8869740410479683
          ],
          "ci_method": "Wilson; graph unit, unconditional within the available denominator"
        },
        "returned_pair_coverage_degree_aligned": {
          "value": 1.0,
          "n_graphs": 120,
          "stratified_bootstrap_ci95": [
            1.0,
            1.0
          ],
          "ci95": [
            0.9689808335812982,
            1
          ],
          "ci_method": "Wilson; graph unit, unconditional within the available denominator"
        },
        "parseable_pair_coverage_degree_aligned": {
          "value": 0.5333333333333333,
          "n_graphs": 120,
          "stratified_bootstrap_ci95": [
            0.48333333333333334,
            0.5752083333333321
          ],
          "ci95": [
            0.44442789269674543,
            0.6201708295420078
          ],
          "ci_method": "Wilson; graph unit, unconditional within the available denominator"
        },
        "accuracy_degree_reversed": {
          "value": 0.525,
          "n_graphs": 120,
          "stratified_bootstrap_ci95": [
            0.475,
            0.575
          ],
          "ci95": [
            0.4362699530582508,
            0.6121790886208143
          ],
          "ci_method": "Wilson; graph unit, unconditional within the available denominator"
        },
        "agreement_degree_reversed": {
          "value": 0.9384615384615385,
          "n_graphs": 65,
          "stratified_bootstrap_ci95": [
            0.8769230769230769,
            0.9850746268656716
          ],
          "ci95": [
            0.8521784813447711,
            0.9758109450336491
          ],
          "ci_method": "Wilson; graph unit, unconditional within the available denominator"
        },
        "survival_degree_reversed": {
          "value": 0.8026315789473685,
          "n_graphs": 76,
          "stratified_bootstrap_ci95": [
            0.7272727272727273,
            0.8767123287671232
          ],
          "ci95": [
            0.6995616502581209,
            0.8765801272024555
          ],
          "ci_method": "Wilson; graph unit, unconditional within the available denominator"
        },
        "returned_pair_coverage_degree_reversed": {
          "value": 1.0,
          "n_graphs": 120,
          "stratified_bootstrap_ci95": [
            1.0,
            1.0
          ],
          "ci95": [
            0.9689808335812982,
            1
          ],
          "ci_method": "Wilson; graph unit, unconditional within the available denominator"
        },
        "parseable_pair_coverage_degree_reversed": {
          "value": 0.5416666666666666,
          "n_graphs": 120,
          "stratified_bootstrap_ci95": [
            0.49166666666666664,
            0.5916666666666667
          ],
          "ci95": [
            0.45260969062370227,
            0.6281387121747393
          ],
          "ci_method": "Wilson; graph unit, unconditional within the available denominator"
        },
        "accuracy_format_json": {
          "value": 0.6333333333333333,
          "n_graphs": 120,
          "stratified_bootstrap_ci95": [
            0.5833333333333334,
            0.6916666666666667
          ],
          "ci95": [
            0.5442242408550724,
            0.7141706480999404
          ],
          "ci_method": "Wilson; graph unit, unconditional within the available denominator"
        },
        "agreement_format_json": {
          "value": 0.9452054794520548,
          "n_graphs": 73,
          "stratified_bootstrap_ci95": [
            0.8904109589041096,
            0.9866666666666667
          ],
          "ci95": [
            0.8674100245784053,
            0.9784875025843021
          ],
          "ci_method": "Wilson; graph unit, unconditional within the available denominator"
        },
        "survival_format_json": {
          "value": 0.9078947368421053,
          "n_graphs": 76,
          "stratified_bootstrap_ci95": [
            0.8513513513513513,
            0.961038961038961
          ],
          "ci95": [
            0.8218729215106407,
            0.954665995936223
          ],
          "ci_method": "Wilson; graph unit, unconditional within the available denominator"
        },
        "returned_pair_coverage_format_json": {
          "value": 1.0,
          "n_graphs": 120,
          "stratified_bootstrap_ci95": [
            1.0,
            1.0
          ],
          "ci95": [
            0.9689808335812982,
            1
          ],
          "ci_method": "Wilson; graph unit, unconditional within the available denominator"
        },
        "parseable_pair_coverage_format_json": {
          "value": 0.6083333333333333,
          "n_graphs": 120,
          "stratified_bootstrap_ci95": [
            0.5583333333333333,
            0.6583333333333333
          ],
          "ci95": [
            0.5189378169992755,
            0.6910080302766723
          ],
          "ci_method": "Wilson; graph unit, unconditional within the available denominator"
        },
        "accuracy_order_shuffled": {
          "value": 0.6083333333333333,
          "n_graphs": 120,
          "stratified_bootstrap_ci95": [
            0.55,
            0.6583333333333333
          ],
          "ci95": [
            0.5189378169992755,
            0.6910080302766723
          ],
          "ci_method": "Wilson; graph unit, unconditional within the available denominator"
        },
        "agreement_order_shuffled": {
          "value": 0.9710144927536232,
          "n_graphs": 69,
          "stratified_bootstrap_ci95": [
            0.9285714285714286,
            1.0
          ],
          "ci95": [
            0.9003341623693473,
            0.9920147973076356
          ],
          "ci_method": "Wilson; graph unit, unconditional within the available denominator"
        },
        "survival_order_shuffled": {
          "value": 0.881578947368421,
          "n_graphs": 76,
          "stratified_bootstrap_ci95": [
            0.8181818181818182,
            0.9436619718309859
          ],
          "ci95": [
            0.7900078224920836,
            0.9364318099582082
          ],
          "ci_method": "Wilson; graph unit, unconditional within the available denominator"
        },
        "returned_pair_coverage_order_shuffled": {
          "value": 1.0,
          "n_graphs": 120,
          "stratified_bootstrap_ci95": [
            1.0,
            1.0
          ],
          "ci95": [
            0.9689808335812982,
            1
          ],
          "ci_method": "Wilson; graph unit, unconditional within the available denominator"
        },
        "parseable_pair_coverage_order_shuffled": {
          "value": 0.575,
          "n_graphs": 120,
          "stratified_bootstrap_ci95": [
            0.525,
            0.625
          ],
          "ci95": [
            0.48557751722993264,
            0.6597696078072621
          ],
          "ci_method": "Wilson; graph unit, unconditional within the available denominator"
        },
        "accuracy_distractor": {
          "value": 0.55,
          "n_graphs": 120,
          "stratified_bootstrap_ci95": [
            0.5,
            0.6083333333333333
          ],
          "ci95": [
            0.46081543419687815,
            0.6360826491612517
          ],
          "ci_method": "Wilson; graph unit, unconditional within the available denominator"
        },
        "agreement_distractor": {
          "value": 0.9384615384615385,
          "n_graphs": 65,
          "stratified_bootstrap_ci95": [
            0.8732394366197183,
            0.9852941176470589
          ],
          "ci95": [
            0.8521784813447711,
            0.9758109450336491
          ],
          "ci_method": "Wilson; graph unit, unconditional within the available denominator"
        },
        "survival_distractor": {
          "value": 0.8026315789473685,
          "n_graphs": 76,
          "stratified_bootstrap_ci95": [
            0.727241594022416,
            0.8805970149253731
          ],
          "ci95": [
            0.6995616502581209,
            0.8765801272024555
          ],
          "ci_method": "Wilson; graph unit, unconditional within the available denominator"
        },
        "returned_pair_coverage_distractor": {
          "value": 1.0,
          "n_graphs": 120,
          "stratified_bootstrap_ci95": [
            1.0,
            1.0
          ],
          "ci95": [
            0.9689808335812982,
            1
          ],
          "ci_method": "Wilson; graph unit, unconditional within the available denominator"
        },
        "parseable_pair_coverage_distractor": {
          "value": 0.5416666666666666,
          "n_graphs": 120,
          "stratified_bootstrap_ci95": [
            0.49166666666666664,
            0.5916666666666667
          ],
          "ci95": [
            0.45260969062370227,
            0.6281387121747393
          ],
          "ci_method": "Wilson; graph unit, unconditional within the available denominator"
        },
        "matched_semantic_gap": {
          "value": 0.0,
          "n_graphs": 63,
          "stratified_bootstrap_ci95": [
            -0.06153846153846154,
            0.06451612903225806
          ],
          "ci95": [
            -0.06153846153846154,
            0.06451612903225806
          ],
          "ci_method": "Task/size-stratified graph bootstrap"
        }
      },
      "by_task": {
        "weighted_shortest_path": {
          "correct": 150,
          "collected": 160,
          "planned": 160
        },
        "shortest_path_count": {
          "correct": 47,
          "collected": 160,
          "planned": 160
        },
        "exact_walk_count": {
          "correct": 16,
          "collected": 160,
          "planned": 160
        },
        "triangle_count": {
          "correct": 56,
          "collected": 160,
          "planned": 160
        },
        "minimum_cut": {
          "correct": 141,
          "collected": 160,
          "planned": 160
        },
        "constrained_reachability": {
          "correct": 158,
          "collected": 160,
          "planned": 160
        }
      }
    },
    "minimax/minimax-m3.1-flash": {
      "endpoint": "tenstorrent/fp8",
      "quantization": "fp8",
      "collected_slots": 960,
      "planned_slots": 960,
      "correct": 630,
      "completed_numeric_wrong": 121,
      "invalid_completed": 3,
      "cutoffs": 206,
      "provider_or_protocol_errors": 0,
      "cost_usd": 1.51740294,
      "metrics": {
        "accuracy_original": {
          "value": 0.675,
          "n_graphs": 120,
          "stratified_bootstrap_ci95": [
            0.625,
            0.725
          ],
          "ci95": [
            0.5869018408594758,
            0.7522414508939786
          ],
          "ci_method": "Wilson; graph unit, unconditional within the available denominator"
        },
        "accuracy_repeat": {
          "value": 0.7583333333333333,
          "n_graphs": 120,
          "stratified_bootstrap_ci95": [
            0.7166666666666667,
            0.8083333333333333
          ],
          "ci95": [
            0.6744985095256915,
            0.8261415878246459
          ],
          "ci_method": "Wilson; graph unit, unconditional within the available denominator"
        },
        "agreement_repeat": {
          "value": 0.8152173913043478,
          "n_graphs": 92,
          "stratified_bootstrap_ci95": [
            0.7553137212331742,
            0.875
          ],
          "ci95": [
            0.7238594289322455,
            0.8813066517355159
          ],
          "ci_method": "Wilson; graph unit, unconditional within the available denominator"
        },
        "survival_repeat": {
          "value": 0.9135802469135802,
          "n_graphs": 81,
          "stratified_bootstrap_ci95": [
            0.862489224137931,
            0.9625115740740741
          ],
          "ci95": [
            0.8322008336612976,
            0.9575074166600399
          ],
          "ci_method": "Wilson; graph unit, unconditional within the available denominator"
        },
        "returned_pair_coverage_repeat": {
          "value": 1.0,
          "n_graphs": 120,
          "stratified_bootstrap_ci95": [
            1.0,
            1.0
          ],
          "ci95": [
            0.9689808335812982,
            1
          ],
          "ci_method": "Wilson; graph unit, unconditional within the available denominator"
        },
        "parseable_pair_coverage_repeat": {
          "value": 0.7666666666666667,
          "n_graphs": 120,
          "stratified_bootstrap_ci95": [
            0.7166666666666667,
            0.8083333333333333
          ],
          "ci95": [
            0.6834455292418565,
            0.8333442486681694
          ],
          "ci_method": "Wilson; graph unit, unconditional within the available denominator"
        },
        "accuracy_neutral": {
          "value": 0.5916666666666667,
          "n_graphs": 120,
          "stratified_bootstrap_ci95": [
            0.5416666666666666,
            0.65
          ],
          "ci95": [
            0.5022078316314181,
            0.6754386545251533
          ],
          "ci_method": "Wilson; graph unit, unconditional within the available denominator"
        },
        "agreement_neutral": {
          "value": 0.7586206896551724,
          "n_graphs": 87,
          "stratified_bootstrap_ci95": [
            0.6966292134831461,
            0.8181818181818182
          ],
          "ci95": [
            0.6590101624669209,
            0.8363583684240523
          ],
          "ci_method": "Wilson; graph unit, unconditional within the available denominator"
        },
        "survival_neutral": {
          "value": 0.8024691358024691,
          "n_graphs": 81,
          "stratified_bootstrap_ci95": [
            0.733313953488372,
            0.8765432098765432
          ],
          "ci95": [
            0.7029560930488865,
            0.8745917318129872
          ],
          "ci_method": "Wilson; graph unit, unconditional within the available denominator"
        },
        "returned_pair_coverage_neutral": {
          "value": 1.0,
          "n_graphs": 120,
          "stratified_bootstrap_ci95": [
            1.0,
            1.0
          ],
          "ci95": [
            0.9689808335812982,
            1
          ],
          "ci_method": "Wilson; graph unit, unconditional within the available denominator"
        },
        "parseable_pair_coverage_neutral": {
          "value": 0.725,
          "n_graphs": 120,
          "stratified_bootstrap_ci95": [
            0.675,
            0.775
          ],
          "ci95": [
            0.6390703417242344,
            0.7969710333873498
          ],
          "ci_method": "Wilson; graph unit, unconditional within the available denominator"
        },
        "accuracy_degree_aligned": {
          "value": 0.6083333333333333,
          "n_graphs": 120,
          "stratified_bootstrap_ci95": [
            0.558125,
            0.6666666666666666
          ],
          "ci95": [
            0.5189378169992755,
            0.6910080302766723
          ],
          "ci_method": "Wilson; graph unit, unconditional within the available denominator"
        },
        "agreement_degree_aligned": {
          "value": 0.8048780487804879,
          "n_graphs": 82,
          "stratified_bootstrap_ci95": [
            0.7441860465116279,
            0.8674698795180723
          ],
          "ci95": [
            0.7062981066280997,
            0.8761710252681821
          ],
          "ci_method": "Wilson; graph unit, unconditional within the available denominator"
        },
        "survival_degree_aligned": {
          "value": 0.8024691358024691,
          "n_graphs": 81,
          "stratified_bootstrap_ci95": [
            0.7368118572292801,
            0.8701298701298701
          ],
          "ci95": [
            0.7029560930488865,
            0.8745917318129872
          ],
          "ci_method": "Wilson; graph unit, unconditional within the available denominator"
        },
        "returned_pair_coverage_degree_aligned": {
          "value": 1.0,
          "n_graphs": 120,
          "stratified_bootstrap_ci95": [
            1.0,
            1.0
          ],
          "ci95": [
            0.9689808335812982,
            1
          ],
          "ci_method": "Wilson; graph unit, unconditional within the available denominator"
        },
        "parseable_pair_coverage_degree_aligned": {
          "value": 0.6833333333333333,
          "n_graphs": 120,
          "stratified_bootstrap_ci95": [
            0.625,
            0.7333333333333333
          ],
          "ci95": [
            0.5955213183551339,
            0.7597716539580089
          ],
          "ci_method": "Wilson; graph unit, unconditional within the available denominator"
        },
        "accuracy_degree_reversed": {
          "value": 0.6,
          "n_graphs": 120,
          "stratified_bootstrap_ci95": [
            0.55,
            0.65
          ],
          "ci95": [
            0.5105602479246331,
            0.6832359187916265
          ],
          "ci_method": "Wilson; graph unit, unconditional within the available denominator"
        },
        "agreement_degree_reversed": {
          "value": 0.7674418604651163,
          "n_graphs": 86,
          "stratified_bootstrap_ci95": [
            0.7,
            0.8372093023255814
          ],
          "ci95": [
            0.6679041466259255,
            0.8441089106398416
          ],
          "ci_method": "Wilson; graph unit, unconditional within the available denominator"
        },
        "survival_degree_reversed": {
          "value": 0.8148148148148148,
          "n_graphs": 81,
          "stratified_bootstrap_ci95": [
            0.7469879518072289,
            0.8846153846153846
          ],
          "ci95": [
            0.7166841055275491,
            0.8844370999409316
          ],
          "ci_method": "Wilson; graph unit, unconditional within the available denominator"
        },
        "returned_pair_coverage_degree_reversed": {
          "value": 1.0,
          "n_graphs": 120,
          "stratified_bootstrap_ci95": [
            1.0,
            1.0
          ],
          "ci95": [
            0.9689808335812982,
            1
          ],
          "ci_method": "Wilson; graph unit, unconditional within the available denominator"
        },
        "parseable_pair_coverage_degree_reversed": {
          "value": 0.7166666666666667,
          "n_graphs": 120,
          "stratified_bootstrap_ci95": [
            0.6666666666666666,
            0.7666666666666667
          ],
          "ci95": [
            0.6302979754694622,
            0.7895937190824338
          ],
          "ci_method": "Wilson; graph unit, unconditional within the available denominator"
        },
        "accuracy_format_json": {
          "value": 0.7,
          "n_graphs": 120,
          "stratified_bootstrap_ci95": [
            0.65,
            0.75
          ],
          "ci95": [
            0.6128485756752667,
            0.7747437577572526
          ],
          "ci_method": "Wilson; graph unit, unconditional within the available denominator"
        },
        "agreement_format_json": {
          "value": 0.7978723404255319,
          "n_graphs": 94,
          "stratified_bootstrap_ci95": [
            0.7391304347826086,
            0.8571789321789319
          ],
          "ci95": [
            0.7057493276333983,
            0.8666051801078104
          ],
          "ci_method": "Wilson; graph unit, unconditional within the available denominator"
        },
        "survival_format_json": {
          "value": 0.9012345679012346,
          "n_graphs": 81,
          "stratified_bootstrap_ci95": [
            0.8444176706827309,
            0.957172619047619
          ],
          "ci95": [
            0.8170349859835296,
            0.9490998837312007
          ],
          "ci_method": "Wilson; graph unit, unconditional within the available denominator"
        },
        "returned_pair_coverage_format_json": {
          "value": 1.0,
          "n_graphs": 120,
          "stratified_bootstrap_ci95": [
            1.0,
            1.0
          ],
          "ci95": [
            0.9689808335812982,
            1
          ],
          "ci_method": "Wilson; graph unit, unconditional within the available denominator"
        },
        "parseable_pair_coverage_format_json": {
          "value": 0.7833333333333333,
          "n_graphs": 120,
          "stratified_bootstrap_ci95": [
            0.7333333333333333,
            0.825
          ],
          "ci95": [
            0.7014563746709188,
            0.8476327643584836
          ],
          "ci_method": "Wilson; graph unit, unconditional within the available denominator"
        },
        "accuracy_order_shuffled": {
          "value": 0.6583333333333333,
          "n_graphs": 120,
          "stratified_bootstrap_ci95": [
            0.6083333333333333,
            0.7083333333333334
          ],
          "ci95": [
            0.5697483143197248,
            0.7370956163143529
          ],
          "ci_method": "Wilson; graph unit, unconditional within the available denominator"
        },
        "agreement_order_shuffled": {
          "value": 0.8117647058823529,
          "n_graphs": 85,
          "stratified_bootstrap_ci95": [
            0.7529411764705882,
            0.8658536585365854
          ],
          "ci95": [
            0.7158897563059731,
            0.8806785655511492
          ],
          "ci_method": "Wilson; graph unit, unconditional within the available denominator"
        },
        "survival_order_shuffled": {
          "value": 0.8518518518518519,
          "n_graphs": 81,
          "stratified_bootstrap_ci95": [
            0.7951807228915663,
            0.9113924050632911
          ],
          "ci95": [
            0.7586684293879009,
            0.9131729179004011
          ],
          "ci_method": "Wilson; graph unit, unconditional within the available denominator"
        },
        "returned_pair_coverage_order_shuffled": {
          "value": 1.0,
          "n_graphs": 120,
          "stratified_bootstrap_ci95": [
            1.0,
            1.0
          ],
          "ci95": [
            0.9689808335812982,
            1
          ],
          "ci_method": "Wilson; graph unit, unconditional within the available denominator"
        },
        "parseable_pair_coverage_order_shuffled": {
          "value": 0.7083333333333334,
          "n_graphs": 120,
          "stratified_bootstrap_ci95": [
            0.6583333333333333,
            0.7583333333333333
          ],
          "ci95": [
            0.6215576502025948,
            0.7821843637896129
          ],
          "ci_method": "Wilson; graph unit, unconditional within the available denominator"
        },
        "accuracy_distractor": {
          "value": 0.6583333333333333,
          "n_graphs": 120,
          "stratified_bootstrap_ci95": [
            0.5916666666666667,
            0.7166666666666667
          ],
          "ci95": [
            0.5697483143197248,
            0.7370956163143529
          ],
          "ci_method": "Wilson; graph unit, unconditional within the available denominator"
        },
        "agreement_distractor": {
          "value": 0.7282608695652174,
          "n_graphs": 92,
          "stratified_bootstrap_ci95": [
            0.6630434782608695,
            0.7916666666666666
          ],
          "ci95": [
            0.629581470079455,
            0.8086422435075448
          ],
          "ci_method": "Wilson; graph unit, unconditional within the available denominator"
        },
        "survival_distractor": {
          "value": 0.8148148148148148,
          "n_graphs": 81,
          "stratified_bootstrap_ci95": [
            0.7439024390243902,
            0.8860759493670886
          ],
          "ci95": [
            0.7166841055275491,
            0.8844370999409316
          ],
          "ci_method": "Wilson; graph unit, unconditional within the available denominator"
        },
        "returned_pair_coverage_distractor": {
          "value": 1.0,
          "n_graphs": 120,
          "stratified_bootstrap_ci95": [
            1.0,
            1.0
          ],
          "ci95": [
            0.9689808335812982,
            1
          ],
          "ci_method": "Wilson; graph unit, unconditional within the available denominator"
        },
        "parseable_pair_coverage_distractor": {
          "value": 0.7666666666666667,
          "n_graphs": 120,
          "stratified_bootstrap_ci95": [
            0.7166666666666667,
            0.8166666666666667
          ],
          "ci95": [
            0.6834455292418565,
            0.8333442486681694
          ],
          "ci_method": "Wilson; graph unit, unconditional within the available denominator"
        },
        "matched_semantic_gap": {
          "value": -0.012658227848101266,
          "n_graphs": 79,
          "stratified_bootstrap_ci95": [
            -0.0759493670886076,
            0.051298701298701205
          ],
          "ci95": [
            -0.0759493670886076,
            0.051298701298701205
          ],
          "ci_method": "Task/size-stratified graph bootstrap"
        }
      },
      "by_task": {
        "weighted_shortest_path": {
          "correct": 157,
          "collected": 160,
          "planned": 160
        },
        "shortest_path_count": {
          "correct": 71,
          "collected": 160,
          "planned": 160
        },
        "exact_walk_count": {
          "correct": 20,
          "collected": 160,
          "planned": 160
        },
        "triangle_count": {
          "correct": 75,
          "collected": 160,
          "planned": 160
        },
        "minimum_cut": {
          "correct": 149,
          "collected": 160,
          "planned": 160
        },
        "constrained_reachability": {
          "correct": 158,
          "collected": 160,
          "planned": 160
        }
      }
    },
    "inclusionai/ling-3.1-flash": {
      "endpoint": "novita",
      "quantization": "unknown",
      "collected_slots": 960,
      "planned_slots": 960,
      "correct": 356,
      "completed_numeric_wrong": 7,
      "invalid_completed": 0,
      "cutoffs": 597,
      "provider_or_protocol_errors": 0,
      "cost_usd": 2.95084098,
      "metrics": {
        "accuracy_original": {
          "value": 0.39166666666666666,
          "n_graphs": 120,
          "stratified_bootstrap_ci95": [
            0.3416666666666667,
            0.45
          ],
          "ci95": [
            0.3089919697233277,
            0.4810621830007244
          ],
          "ci_method": "Wilson; graph unit, unconditional within the available denominator"
        },
        "accuracy_repeat": {
          "value": 0.43333333333333335,
          "n_graphs": 120,
          "stratified_bootstrap_ci95": [
            0.375,
            0.49166666666666664
          ],
          "ci95": [
            0.3481017643778866,
            0.5227007911446071
          ],
          "ci_method": "Wilson; graph unit, unconditional within the available denominator"
        },
        "agreement_repeat": {
          "value": 1.0,
          "n_graphs": 42,
          "stratified_bootstrap_ci95": [
            1.0,
            1.0
          ],
          "ci95": [
            0.9162012091343005,
            1
          ],
          "ci_method": "Wilson; graph unit, unconditional within the available denominator"
        },
        "survival_repeat": {
          "value": 0.8936170212765957,
          "n_graphs": 47,
          "stratified_bootstrap_ci95": [
            0.795906771799629,
            0.9761904761904762
          ],
          "ci95": [
            0.7740573065626205,
            0.9536952178189174
          ],
          "ci_method": "Wilson; graph unit, unconditional within the available denominator"
        },
        "returned_pair_coverage_repeat": {
          "value": 1.0,
          "n_graphs": 120,
          "stratified_bootstrap_ci95": [
            1.0,
            1.0
          ],
          "ci95": [
            0.9689808335812982,
            1
          ],
          "ci_method": "Wilson; graph unit, unconditional within the available denominator"
        },
        "parseable_pair_coverage_repeat": {
          "value": 0.35,
          "n_graphs": 120,
          "stratified_bootstrap_ci95": [
            0.2916666666666667,
            0.4083333333333333
          ],
          "ci95": [
            0.27051899590285605,
            0.4387867540227545
          ],
          "ci_method": "Wilson; graph unit, unconditional within the available denominator"
        },
        "accuracy_neutral": {
          "value": 0.35833333333333334,
          "n_graphs": 120,
          "stratified_bootstrap_ci95": [
            0.3,
            0.4166666666666667
          ],
          "ci95": [
            0.27816079050320264,
            0.4472946399820963
          ],
          "ci_method": "Wilson; graph unit, unconditional within the available denominator"
        },
        "agreement_neutral": {
          "value": 1.0,
          "n_graphs": 37,
          "stratified_bootstrap_ci95": [
            1.0,
            1.0
          ],
          "ci95": [
            0.9059421741628021,
            0.9999999999999999
          ],
          "ci_method": "Wilson; graph unit, unconditional within the available denominator"
        },
        "survival_neutral": {
          "value": 0.7872340425531915,
          "n_graphs": 47,
          "stratified_bootstrap_ci95": [
            0.6799811320754717,
            0.8888888888888888
          ],
          "ci95": [
            0.6509597329080201,
            0.8801029200190482
          ],
          "ci_method": "Wilson; graph unit, unconditional within the available denominator"
        },
        "returned_pair_coverage_neutral": {
          "value": 1.0,
          "n_graphs": 120,
          "stratified_bootstrap_ci95": [
            1.0,
            1.0
          ],
          "ci95": [
            0.9689808335812982,
            1
          ],
          "ci_method": "Wilson; graph unit, unconditional within the available denominator"
        },
        "parseable_pair_coverage_neutral": {
          "value": 0.30833333333333335,
          "n_graphs": 120,
          "stratified_bootstrap_ci95": [
            0.25833333333333336,
            0.36666666666666664
          ],
          "ci95": [
            0.23272736988916487,
            0.3958299772380043
          ],
          "ci_method": "Wilson; graph unit, unconditional within the available denominator"
        },
        "accuracy_degree_aligned": {
          "value": 0.2916666666666667,
          "n_graphs": 120,
          "stratified_bootstrap_ci95": [
            0.23333333333333334,
            0.35
          ],
          "ci95": [
            0.21781563621038724,
            0.3784423497974053
          ],
          "ci_method": "Wilson; graph unit, unconditional within the available denominator"
        },
        "agreement_degree_aligned": {
          "value": 0.9705882352941176,
          "n_graphs": 34,
          "stratified_bootstrap_ci95": [
            0.9090909090909091,
            1.0
          ],
          "ci95": [
            0.8508442670731458,
            0.9947890937237309
          ],
          "ci_method": "Wilson; graph unit, unconditional within the available denominator"
        },
        "survival_degree_aligned": {
          "value": 0.7021276595744681,
          "n_graphs": 47,
          "stratified_bootstrap_ci95": [
            0.5833333333333334,
            0.8163265306122449
          ],
          "ci95": [
            0.5602232608095433,
            0.8134874949539491
          ],
          "ci_method": "Wilson; graph unit, unconditional within the available denominator"
        },
        "returned_pair_coverage_degree_aligned": {
          "value": 1.0,
          "n_graphs": 120,
          "stratified_bootstrap_ci95": [
            1.0,
            1.0
          ],
          "ci95": [
            0.9689808335812982,
            1
          ],
          "ci_method": "Wilson; graph unit, unconditional within the available denominator"
        },
        "parseable_pair_coverage_degree_aligned": {
          "value": 0.2833333333333333,
          "n_graphs": 120,
          "stratified_bootstrap_ci95": [
            0.225,
            0.3416666666666667
          ],
          "ci95": [
            0.2104062809175663,
            0.3697020245305379
          ],
          "ci_method": "Wilson; graph unit, unconditional within the available denominator"
        },
        "accuracy_degree_reversed": {
          "value": 0.31666666666666665,
          "n_graphs": 120,
          "stratified_bootstrap_ci95": [
            0.25833333333333336,
            0.375
          ],
          "ci95": [
            0.24022834604199117,
            0.4044786816448662
          ],
          "ci_method": "Wilson; graph unit, unconditional within the available denominator"
        },
        "agreement_degree_reversed": {
          "value": 1.0,
          "n_graphs": 36,
          "stratified_bootstrap_ci95": [
            1.0,
            1.0
          ],
          "ci95": [
            0.9035813714055411,
            0.9999999999999999
          ],
          "ci_method": "Wilson; graph unit, unconditional within the available denominator"
        },
        "survival_degree_reversed": {
          "value": 0.7659574468085106,
          "n_graphs": 47,
          "stratified_bootstrap_ci95": [
            0.6511337209302326,
            0.8695652173913043
          ],
          "ci95": [
            0.6277574244984482,
            0.863967254137726
          ],
          "ci_method": "Wilson; graph unit, unconditional within the available denominator"
        },
        "returned_pair_coverage_degree_reversed": {
          "value": 1.0,
          "n_graphs": 120,
          "stratified_bootstrap_ci95": [
            1.0,
            1.0
          ],
          "ci95": [
            0.9689808335812982,
            1
          ],
          "ci_method": "Wilson; graph unit, unconditional within the available denominator"
        },
        "parseable_pair_coverage_degree_reversed": {
          "value": 0.3,
          "n_graphs": 120,
          "stratified_bootstrap_ci95": [
            0.24166666666666667,
            0.3502083333333322
          ],
          "ci95": [
            0.22525624224274754,
            0.38715142432473326
          ],
          "ci_method": "Wilson; graph unit, unconditional within the available denominator"
        },
        "accuracy_format_json": {
          "value": 0.425,
          "n_graphs": 120,
          "stratified_bootstrap_ci95": [
            0.375,
            0.475
          ],
          "ci95": [
            0.340230392192738,
            0.5144224827700674
          ],
          "ci_method": "Wilson; graph unit, unconditional within the available denominator"
        },
        "agreement_format_json": {
          "value": 1.0,
          "n_graphs": 43,
          "stratified_bootstrap_ci95": [
            1.0,
            1.0
          ],
          "ci95": [
            0.917990196774213,
            1
          ],
          "ci_method": "Wilson; graph unit, unconditional within the available denominator"
        },
        "survival_format_json": {
          "value": 0.9148936170212766,
          "n_graphs": 47,
          "stratified_bootstrap_ci95": [
            0.8367346938775511,
            0.9787234042553191
          ],
          "ci95": [
            0.8006846221303582,
            0.9664058765420737
          ],
          "ci_method": "Wilson; graph unit, unconditional within the available denominator"
        },
        "returned_pair_coverage_format_json": {
          "value": 1.0,
          "n_graphs": 120,
          "stratified_bootstrap_ci95": [
            1.0,
            1.0
          ],
          "ci95": [
            0.9689808335812982,
            1
          ],
          "ci_method": "Wilson; graph unit, unconditional within the available denominator"
        },
        "parseable_pair_coverage_format_json": {
          "value": 0.35833333333333334,
          "n_graphs": 120,
          "stratified_bootstrap_ci95": [
            0.30833333333333335,
            0.4083333333333333
          ],
          "ci95": [
            0.27816079050320264,
            0.4472946399820963
          ],
          "ci_method": "Wilson; graph unit, unconditional within the available denominator"
        },
        "accuracy_order_shuffled": {
          "value": 0.3333333333333333,
          "n_graphs": 120,
          "stratified_bootstrap_ci95": [
            0.2833333333333333,
            0.38333333333333336
          ],
          "ci95": [
            0.2553174050125864,
            0.4216889837936476
          ],
          "ci_method": "Wilson; graph unit, unconditional within the available denominator"
        },
        "agreement_order_shuffled": {
          "value": 1.0,
          "n_graphs": 35,
          "stratified_bootstrap_ci95": [
            1.0,
            1.0
          ],
          "ci95": [
            0.9010990076756009,
            1
          ],
          "ci_method": "Wilson; graph unit, unconditional within the available denominator"
        },
        "survival_order_shuffled": {
          "value": 0.7446808510638298,
          "n_graphs": 47,
          "stratified_bootstrap_ci95": [
            0.6382495164410058,
            0.85
          ],
          "ci95": [
            0.6049182446958296,
            0.8474684596494507
          ],
          "ci_method": "Wilson; graph unit, unconditional within the available denominator"
        },
        "returned_pair_coverage_order_shuffled": {
          "value": 1.0,
          "n_graphs": 120,
          "stratified_bootstrap_ci95": [
            1.0,
            1.0
          ],
          "ci95": [
            0.9689808335812982,
            1
          ],
          "ci_method": "Wilson; graph unit, unconditional within the available denominator"
        },
        "parseable_pair_coverage_order_shuffled": {
          "value": 0.2916666666666667,
          "n_graphs": 120,
          "stratified_bootstrap_ci95": [
            0.24166666666666667,
            0.3416666666666667
          ],
          "ci95": [
            0.21781563621038724,
            0.3784423497974053
          ],
          "ci_method": "Wilson; graph unit, unconditional within the available denominator"
        },
        "accuracy_distractor": {
          "value": 0.4166666666666667,
          "n_graphs": 120,
          "stratified_bootstrap_ci95": [
            0.35833333333333334,
            0.48333333333333334
          ],
          "ci95": [
            0.3323835214047618,
            0.5061196729983553
          ],
          "ci_method": "Wilson; graph unit, unconditional within the available denominator"
        },
        "agreement_distractor": {
          "value": 1.0,
          "n_graphs": 40,
          "stratified_bootstrap_ci95": [
            1.0,
            1.0
          ],
          "ci95": [
            0.912378398802718,
            1
          ],
          "ci_method": "Wilson; graph unit, unconditional within the available denominator"
        },
        "survival_distractor": {
          "value": 0.851063829787234,
          "n_graphs": 47,
          "stratified_bootstrap_ci95": [
            0.75,
            0.94
          ],
          "ci95": [
            0.7231437757640686,
            0.9259328000356815
          ],
          "ci_method": "Wilson; graph unit, unconditional within the available denominator"
        },
        "returned_pair_coverage_distractor": {
          "value": 1.0,
          "n_graphs": 120,
          "stratified_bootstrap_ci95": [
            1.0,
            1.0
          ],
          "ci95": [
            0.9689808335812982,
            1
          ],
          "ci_method": "Wilson; graph unit, unconditional within the available denominator"
        },
        "parseable_pair_coverage_distractor": {
          "value": 0.3333333333333333,
          "n_graphs": 120,
          "stratified_bootstrap_ci95": [
            0.275,
            0.39166666666666666
          ],
          "ci95": [
            0.2553174050125864,
            0.4216889837936476
          ],
          "ci_method": "Wilson; graph unit, unconditional within the available denominator"
        },
        "matched_semantic_gap": {
          "value": 0.0,
          "n_graphs": 35,
          "stratified_bootstrap_ci95": [
            0.0,
            0.0
          ],
          "ci95": [
            0,
            0.09890099232439913
          ],
          "ci_method": "Wilson; graph unit, unconditional within the available denominator"
        }
      },
      "by_task": {
        "weighted_shortest_path": {
          "correct": 117,
          "collected": 160,
          "planned": 160
        },
        "shortest_path_count": {
          "correct": 19,
          "collected": 160,
          "planned": 160
        },
        "exact_walk_count": {
          "correct": 0,
          "collected": 160,
          "planned": 160
        },
        "triangle_count": {
          "correct": 8,
          "collected": 160,
          "planned": 160
        },
        "minimum_cut": {
          "correct": 78,
          "collected": 160,
          "planned": 160
        },
        "constrained_reachability": {
          "correct": 134,
          "collected": 160,
          "planned": 160
        }
      }
    },
    "ibm-granite/granite-4.2-8b": {
      "endpoint": "coreweave/bf16",
      "quantization": "bf16",
      "collected_slots": 960,
      "planned_slots": 960,
      "correct": 98,
      "completed_numeric_wrong": 0,
      "invalid_completed": 0,
      "cutoffs": 862,
      "provider_or_protocol_errors": 0,
      "cost_usd": 2.39171193,
      "metrics": {
        "accuracy_original": {
          "value": 0.125,
          "n_graphs": 120,
          "stratified_bootstrap_ci95": [
            0.08333333333333333,
            0.175
          ],
          "ci95": [
            0.07723497273395474,
            0.19602940208007164
          ],
          "ci_method": "Wilson; graph unit, unconditional within the available denominator"
        },
        "accuracy_repeat": {
          "value": 0.14166666666666666,
          "n_graphs": 120,
          "stratified_bootstrap_ci95": [
            0.09166666666666666,
            0.19166666666666668
          ],
          "ci95": [
            0.09036879103483349,
            0.21519494489856955
          ],
          "ci_method": "Wilson; graph unit, unconditional within the available denominator"
        },
        "agreement_repeat": {
          "value": 1.0,
          "n_graphs": 12,
          "stratified_bootstrap_ci95": [
            1.0,
            1.0
          ],
          "ci95": [
            0.7575059933447693,
            1
          ],
          "ci_method": "Wilson; graph unit, unconditional within the available denominator"
        },
        "survival_repeat": {
          "value": 0.8,
          "n_graphs": 15,
          "stratified_bootstrap_ci95": [
            0.5714285714285714,
            1.0
          ],
          "ci95": [
            0.5481455128483137,
            0.9295245065301825
          ],
          "ci_method": "Wilson; graph unit, unconditional within the available denominator"
        },
        "returned_pair_coverage_repeat": {
          "value": 1.0,
          "n_graphs": 120,
          "stratified_bootstrap_ci95": [
            1.0,
            1.0
          ],
          "ci95": [
            0.9689808335812982,
            1
          ],
          "ci_method": "Wilson; graph unit, unconditional within the available denominator"
        },
        "parseable_pair_coverage_repeat": {
          "value": 0.1,
          "n_graphs": 120,
          "stratified_bootstrap_ci95": [
            0.058333333333333334,
            0.14166666666666666
          ],
          "ci95": [
            0.05813359202601165,
            0.16668174110894984
          ],
          "ci_method": "Wilson; graph unit, unconditional within the available denominator"
        },
        "accuracy_neutral": {
          "value": 0.11666666666666667,
          "n_graphs": 120,
          "stratified_bootstrap_ci95": [
            0.075,
            0.15833333333333333
          ],
          "ci95": [
            0.07078117807830148,
            0.1863335161760366
          ],
          "ci_method": "Wilson; graph unit, unconditional within the available denominator"
        },
        "agreement_neutral": {
          "value": 1.0,
          "n_graphs": 9,
          "stratified_bootstrap_ci95": [
            1.0,
            1.0
          ],
          "ci95": [
            0.7008549515804674,
            1
          ],
          "ci_method": "Wilson; graph unit, unconditional within the available denominator"
        },
        "survival_neutral": {
          "value": 0.6,
          "n_graphs": 15,
          "stratified_bootstrap_ci95": [
            0.35294117647058826,
            0.857380952380951
          ],
          "ci95": [
            0.35746830120625167,
            0.8017550385865801
          ],
          "ci_method": "Wilson; graph unit, unconditional within the available denominator"
        },
        "returned_pair_coverage_neutral": {
          "value": 1.0,
          "n_graphs": 120,
          "stratified_bootstrap_ci95": [
            1.0,
            1.0
          ],
          "ci95": [
            0.9689808335812982,
            1
          ],
          "ci_method": "Wilson; graph unit, unconditional within the available denominator"
        },
        "parseable_pair_coverage_neutral": {
          "value": 0.075,
          "n_graphs": 120,
          "stratified_bootstrap_ci95": [
            0.03333333333333333,
            0.11666666666666667
          ],
          "ci95": [
            0.03995711539596095,
            0.1364091760599356
          ],
          "ci_method": "Wilson; graph unit, unconditional within the available denominator"
        },
        "accuracy_degree_aligned": {
          "value": 0.058333333333333334,
          "n_graphs": 120,
          "stratified_bootstrap_ci95": [
            0.025,
            0.1
          ],
          "ci95": [
            0.028541010000220822,
            0.1155259203362991
          ],
          "ci_method": "Wilson; graph unit, unconditional within the available denominator"
        },
        "agreement_degree_aligned": {
          "value": 1.0,
          "n_graphs": 4,
          "stratified_bootstrap_ci95": [
            1.0,
            1.0
          ],
          "ci95": [
            0.5101091635454165,
            1
          ],
          "ci_method": "Wilson; graph unit, unconditional within the available denominator"
        },
        "survival_degree_aligned": {
          "value": 0.26666666666666666,
          "n_graphs": 15,
          "stratified_bootstrap_ci95": [
            0.0625,
            0.5
          ],
          "ci95": [
            0.10897453325692635,
            0.5195043405597988
          ],
          "ci_method": "Wilson; graph unit, unconditional within the available denominator"
        },
        "returned_pair_coverage_degree_aligned": {
          "value": 1.0,
          "n_graphs": 120,
          "stratified_bootstrap_ci95": [
            1.0,
            1.0
          ],
          "ci95": [
            0.9689808335812982,
            1
          ],
          "ci_method": "Wilson; graph unit, unconditional within the available denominator"
        },
        "parseable_pair_coverage_degree_aligned": {
          "value": 0.03333333333333333,
          "n_graphs": 120,
          "stratified_bootstrap_ci95": [
            0.008333333333333333,
            0.06666666666666667
          ],
          "ci95": [
            0.013037551098350905,
            0.08258033755910414
          ],
          "ci_method": "Wilson; graph unit, unconditional within the available denominator"
        },
        "accuracy_degree_reversed": {
          "value": 0.075,
          "n_graphs": 120,
          "stratified_bootstrap_ci95": [
            0.041666666666666664,
            0.10833333333333334
          ],
          "ci95": [
            0.03995711539596095,
            0.1364091760599356
          ],
          "ci_method": "Wilson; graph unit, unconditional within the available denominator"
        },
        "agreement_degree_reversed": {
          "value": 1.0,
          "n_graphs": 8,
          "stratified_bootstrap_ci95": [
            1.0,
            1.0
          ],
          "ci95": [
            0.6755924351161318,
            1
          ],
          "ci_method": "Wilson; graph unit, unconditional within the available denominator"
        },
        "survival_degree_reversed": {
          "value": 0.5333333333333333,
          "n_graphs": 15,
          "stratified_bootstrap_ci95": [
            0.29411764705882354,
            0.7857142857142857
          ],
          "ci95": [
            0.3011698002549891,
            0.7519046463426214
          ],
          "ci_method": "Wilson; graph unit, unconditional within the available denominator"
        },
        "returned_pair_coverage_degree_reversed": {
          "value": 1.0,
          "n_graphs": 120,
          "stratified_bootstrap_ci95": [
            1.0,
            1.0
          ],
          "ci95": [
            0.9689808335812982,
            1
          ],
          "ci_method": "Wilson; graph unit, unconditional within the available denominator"
        },
        "parseable_pair_coverage_degree_reversed": {
          "value": 0.06666666666666667,
          "n_graphs": 120,
          "stratified_bootstrap_ci95": [
            0.03333333333333333,
            0.10833333333333334
          ],
          "ci95": [
            0.03416530686090694,
            0.12605130403530132
          ],
          "ci_method": "Wilson; graph unit, unconditional within the available denominator"
        },
        "accuracy_format_json": {
          "value": 0.15,
          "n_graphs": 120,
          "stratified_bootstrap_ci95": [
            0.10833333333333334,
            0.19166666666666668
          ],
          "ci95": [
            0.09703812292584856,
            0.22467529356724275
          ],
          "ci_method": "Wilson; graph unit, unconditional within the available denominator"
        },
        "agreement_format_json": {
          "value": 1.0,
          "n_graphs": 13,
          "stratified_bootstrap_ci95": [
            1.0,
            1.0
          ],
          "ci95": [
            0.7719046276458112,
            1
          ],
          "ci_method": "Wilson; graph unit, unconditional within the available denominator"
        },
        "survival_format_json": {
          "value": 0.8666666666666667,
          "n_graphs": 15,
          "stratified_bootstrap_ci95": [
            0.6875,
            1.0
          ],
          "ci95": [
            0.6211801716767842,
            0.9626387408969335
          ],
          "ci_method": "Wilson; graph unit, unconditional within the available denominator"
        },
        "returned_pair_coverage_format_json": {
          "value": 1.0,
          "n_graphs": 120,
          "stratified_bootstrap_ci95": [
            1.0,
            1.0
          ],
          "ci95": [
            0.9689808335812982,
            1
          ],
          "ci_method": "Wilson; graph unit, unconditional within the available denominator"
        },
        "parseable_pair_coverage_format_json": {
          "value": 0.10833333333333334,
          "n_graphs": 120,
          "stratified_bootstrap_ci95": [
            0.06666666666666667,
            0.15
          ],
          "ci95": [
            0.06441141154173535,
            0.17655360215291446
          ],
          "ci_method": "Wilson; graph unit, unconditional within the available denominator"
        },
        "accuracy_order_shuffled": {
          "value": 0.05,
          "n_graphs": 120,
          "stratified_bootstrap_ci95": [
            0.016666666666666666,
            0.08333333333333333
          ],
          "ci95": [
            0.02311436441446625,
            0.10480288536236539
          ],
          "ci_method": "Wilson; graph unit, unconditional within the available denominator"
        },
        "agreement_order_shuffled": {
          "value": 1.0,
          "n_graphs": 5,
          "stratified_bootstrap_ci95": [
            1.0,
            1.0
          ],
          "ci95": [
            0.5655175352168389,
            1
          ],
          "ci_method": "Wilson; graph unit, unconditional within the available denominator"
        },
        "survival_order_shuffled": {
          "value": 0.3333333333333333,
          "n_graphs": 15,
          "stratified_bootstrap_ci95": [
            0.1111111111111111,
            0.5833333333333334
          ],
          "ci95": [
            0.15176324439715058,
            0.5828645226147958
          ],
          "ci_method": "Wilson; graph unit, unconditional within the available denominator"
        },
        "returned_pair_coverage_order_shuffled": {
          "value": 1.0,
          "n_graphs": 120,
          "stratified_bootstrap_ci95": [
            1.0,
            1.0
          ],
          "ci95": [
            0.9689808335812982,
            1
          ],
          "ci_method": "Wilson; graph unit, unconditional within the available denominator"
        },
        "parseable_pair_coverage_order_shuffled": {
          "value": 0.041666666666666664,
          "n_graphs": 120,
          "stratified_bootstrap_ci95": [
            0.008333333333333333,
            0.075
          ],
          "ci95": [
            0.017926716880109897,
            0.09384085233703345
          ],
          "ci_method": "Wilson; graph unit, unconditional within the available denominator"
        },
        "accuracy_distractor": {
          "value": 0.1,
          "n_graphs": 120,
          "stratified_bootstrap_ci95": [
            0.058333333333333334,
            0.14166666666666666
          ],
          "ci95": [
            0.05813359202601165,
            0.16668174110894984
          ],
          "ci_method": "Wilson; graph unit, unconditional within the available denominator"
        },
        "agreement_distractor": {
          "value": 1.0,
          "n_graphs": 8,
          "stratified_bootstrap_ci95": [
            1.0,
            1.0
          ],
          "ci95": [
            0.6755924351161318,
            1
          ],
          "ci_method": "Wilson; graph unit, unconditional within the available denominator"
        },
        "survival_distractor": {
          "value": 0.5333333333333333,
          "n_graphs": 15,
          "stratified_bootstrap_ci95": [
            0.318122009569378,
            0.7692307692307693
          ],
          "ci95": [
            0.3011698002549891,
            0.7519046463426214
          ],
          "ci_method": "Wilson; graph unit, unconditional within the available denominator"
        },
        "returned_pair_coverage_distractor": {
          "value": 1.0,
          "n_graphs": 120,
          "stratified_bootstrap_ci95": [
            1.0,
            1.0
          ],
          "ci95": [
            0.9689808335812982,
            1
          ],
          "ci_method": "Wilson; graph unit, unconditional within the available denominator"
        },
        "parseable_pair_coverage_distractor": {
          "value": 0.06666666666666667,
          "n_graphs": 120,
          "stratified_bootstrap_ci95": [
            0.03333333333333333,
            0.1
          ],
          "ci95": [
            0.03416530686090694,
            0.12605130403530132
          ],
          "ci_method": "Wilson; graph unit, unconditional within the available denominator"
        },
        "matched_semantic_gap": {
          "value": 0.0,
          "n_graphs": 7,
          "stratified_bootstrap_ci95": [
            0.0,
            0.0
          ],
          "ci95": [
            0,
            0.35433043506667483
          ],
          "ci_method": "Wilson; graph unit, unconditional within the available denominator"
        }
      },
      "by_task": {
        "weighted_shortest_path": {
          "correct": 24,
          "collected": 160,
          "planned": 160
        },
        "shortest_path_count": {
          "correct": 0,
          "collected": 160,
          "planned": 160
        },
        "exact_walk_count": {
          "correct": 0,
          "collected": 160,
          "planned": 160
        },
        "triangle_count": {
          "correct": 0,
          "collected": 160,
          "planned": 160
        },
        "minimum_cut": {
          "correct": 4,
          "collected": 160,
          "planned": 160
        },
        "constrained_reachability": {
          "correct": 70,
          "collected": 160,
          "planned": 160
        }
      }
    }
  }
}
