{
  "model": [
    "jev-1.13.0"
  ],
  "traces_assessed": 4,
  "traces_selected": 24,
  "complete_selected_sample": false,
  "overall": {
    "step": {
      "n": 16,
      "correct": 11,
      "accuracy": 0.6875,
      "majority_baseline": 0.5625,
      "confusion_gold_rows_prediction_columns": {
        "-1": {
          "-1": 5,
          "0": 0,
          "1": 1
        },
        "0": {
          "-1": 0,
          "0": 0,
          "1": 1
        },
        "1": {
          "-1": 0,
          "0": 3,
          "1": 6
        }
      },
      "negative_precision": 1.0,
      "negative_recall": 0.8333333333333334
    },
    "outcome": {
      "n": 4,
      "correct": 3,
      "accuracy": 0.75,
      "majority_baseline": 1.0,
      "confusion_gold_rows_prediction_columns": {
        "-1": {
          "-1": 3,
          "0": 0,
          "1": 1
        },
        "0": {
          "-1": 0,
          "0": 0,
          "1": 0
        },
        "1": {
          "-1": 0,
          "0": 0,
          "1": 0
        }
      },
      "negative_precision": 1.0,
      "negative_recall": 0.75
    }
  },
  "by_domain": {
    "bfcl": {
      "step": {
        "n": 5,
        "correct": 5,
        "accuracy": 1.0,
        "majority_baseline": 0.6,
        "confusion_gold_rows_prediction_columns": {
          "-1": {
            "-1": 2,
            "0": 0,
            "1": 0
          },
          "0": {
            "-1": 0,
            "0": 0,
            "1": 0
          },
          "1": {
            "-1": 0,
            "0": 0,
            "1": 3
          }
        },
        "negative_precision": 1.0,
        "negative_recall": 1.0
      },
      "outcome": {
        "n": 1,
        "correct": 1,
        "accuracy": 1.0,
        "majority_baseline": 1.0,
        "confusion_gold_rows_prediction_columns": {
          "-1": {
            "-1": 1,
            "0": 0,
            "1": 0
          },
          "0": {
            "-1": 0,
            "0": 0,
            "1": 0
          },
          "1": {
            "-1": 0,
            "0": 0,
            "1": 0
          }
        },
        "negative_precision": 1.0,
        "negative_recall": 1.0
      }
    },
    "gaia_dev": {
      "step": {
        "n": 6,
        "correct": 3,
        "accuracy": 0.5,
        "majority_baseline": 0.6666666666666666,
        "confusion_gold_rows_prediction_columns": {
          "-1": {
            "-1": 1,
            "0": 0,
            "1": 0
          },
          "0": {
            "-1": 0,
            "0": 0,
            "1": 1
          },
          "1": {
            "-1": 0,
            "0": 2,
            "1": 2
          }
        },
        "negative_precision": 1.0,
        "negative_recall": 1.0
      },
      "outcome": {
        "n": 1,
        "correct": 1,
        "accuracy": 1.0,
        "majority_baseline": 1.0,
        "confusion_gold_rows_prediction_columns": {
          "-1": {
            "-1": 1,
            "0": 0,
            "1": 0
          },
          "0": {
            "-1": 0,
            "0": 0,
            "1": 0
          },
          "1": {
            "-1": 0,
            "0": 0,
            "1": 0
          }
        },
        "negative_precision": 1.0,
        "negative_recall": 1.0
      }
    },
    "hotpotqa": {
      "step": {
        "n": 1,
        "correct": 0,
        "accuracy": 0.0,
        "majority_baseline": 1.0,
        "confusion_gold_rows_prediction_columns": {
          "-1": {
            "-1": 0,
            "0": 0,
            "1": 1
          },
          "0": {
            "-1": 0,
            "0": 0,
            "1": 0
          },
          "1": {
            "-1": 0,
            "0": 0,
            "1": 0
          }
        },
        "negative_precision": null,
        "negative_recall": 0.0
      },
      "outcome": {
        "n": 1,
        "correct": 0,
        "accuracy": 0.0,
        "majority_baseline": 1.0,
        "confusion_gold_rows_prediction_columns": {
          "-1": {
            "-1": 0,
            "0": 0,
            "1": 1
          },
          "0": {
            "-1": 0,
            "0": 0,
            "1": 0
          },
          "1": {
            "-1": 0,
            "0": 0,
            "1": 0
          }
        },
        "negative_precision": null,
        "negative_recall": 0.0
      }
    },
    "tau2": {
      "step": {
        "n": 4,
        "correct": 3,
        "accuracy": 0.75,
        "majority_baseline": 0.5,
        "confusion_gold_rows_prediction_columns": {
          "-1": {
            "-1": 2,
            "0": 0,
            "1": 0
          },
          "0": {
            "-1": 0,
            "0": 0,
            "1": 0
          },
          "1": {
            "-1": 0,
            "0": 1,
            "1": 1
          }
        },
        "negative_precision": 1.0,
        "negative_recall": 1.0
      },
      "outcome": {
        "n": 1,
        "correct": 1,
        "accuracy": 1.0,
        "majority_baseline": 1.0,
        "confusion_gold_rows_prediction_columns": {
          "-1": {
            "-1": 1,
            "0": 0,
            "1": 0
          },
          "0": {
            "-1": 0,
            "0": 0,
            "1": 0
          },
          "1": {
            "-1": 0,
            "0": 0,
            "1": 0
          }
        },
        "negative_precision": 1.0,
        "negative_recall": 1.0
      }
    }
  },
  "usage": [
    {
      "input_tokens": 1446,
      "output_tokens": 93
    },
    {
      "input_tokens": 7413,
      "output_tokens": 318
    },
    {
      "input_tokens": 13315,
      "output_tokens": 273
    },
    {
      "input_tokens": 8603,
      "output_tokens": 228
    }
  ]
}