Reliability diagram — verifier calibration with ECE and MCE

{
  "$schema": "https://vega.github.io/schema/vega-lite/v5.json",
  "title": {
    "text": "Reliability diagram — RAG answer verifier, out-of-time slice",
    "subtitle": ["Predicted P(answer unsupported) against the observed unsupported rate, 10 equal-width bins.", "n = 38 412 graded answers · base rate 17.1% · ECE 0.0142 · MCE 0.110 in the 0.9-1.0 bin.", "Above the diagonal the verifier is under-confident; below it, over-confident."],
    "anchor": "start",
    "fontSize": 15,
    "subtitleFontSize": 11
  },
  "data": {
    "values": [
      {"bin":"0.0-0.1","mean_pred":0.031,"observed":0.038,"n":21540,"lo":0.036,"hi":0.041},
      {"bin":"0.1-0.2","mean_pred":0.146,"observed":0.161,"n":6120,"lo":0.152,"hi":0.170},
      {"bin":"0.2-0.3","mean_pred":0.246,"observed":0.268,"n":3480,"lo":0.253,"hi":0.283},
      {"bin":"0.3-0.4","mean_pred":0.347,"observed":0.372,"n":2210,"lo":0.352,"hi":0.392},
      {"bin":"0.4-0.5","mean_pred":0.446,"observed":0.463,"n":1580,"lo":0.438,"hi":0.488},
      {"bin":"0.5-0.6","mean_pred":0.547,"observed":0.541,"n":1140,"lo":0.512,"hi":0.570},
      {"bin":"0.6-0.7","mean_pred":0.646,"observed":0.611,"n":852,"lo":0.578,"hi":0.644},
      {"bin":"0.7-0.8","mean_pred":0.748,"observed":0.702,"n":624,"lo":0.666,"hi":0.738},
      {"bin":"0.8-0.9","mean_pred":0.845,"observed":0.771,"n":489,"lo":0.734,"hi":0.808},
      {"bin":"0.9-1.0","mean_pred":0.938,"observed":0.828,"n":377,"lo":0.790,"hi":0.866}
    ]
  },
  "vconcat": [
    {
      "width": 400,
      "height": 300,
      "layer": [
        {
          "data": {"values": [{"d":0},{"d":1}]},
          "mark": {"type": "line", "strokeDash": [5, 4], "color": "#9aa0a6"},
          "encoding": {
            "x": {"field": "d", "type": "quantitative"},
            "y": {"field": "d", "type": "quantitative"}
          }
        },
        {
          "mark": {"type": "rule", "color": "#5b6470"},
          "encoding": {
            "x": {"field": "mean_pred", "type": "quantitative"},
            "y": {"field": "lo", "type": "quantitative"},
            "y2": {"field": "hi"}
          }
        },
        {
          "mark": {"type": "line", "color": "#1f47f5", "point": false},
          "encoding": {
            "x": {"field": "mean_pred", "type": "quantitative", "scale": {"domain": [0, 1]}, "title": "Mean predicted probability in bin"},
            "y": {"field": "observed", "type": "quantitative", "scale": {"domain": [0, 1]}, "title": "Observed unsupported rate"}
          }
        },
        {
          "mark": {"type": "point", "filled": true, "size": 90, "color": "#1f47f5"},
          "encoding": {
            "x": {"field": "mean_pred", "type": "quantitative"},
            "y": {"field": "observed", "type": "quantitative"},
            "tooltip": [
              {"field": "bin", "type": "nominal", "title": "Score bin"},
              {"field": "n", "type": "quantitative", "title": "Answers in bin"},
              {"field": "mean_pred", "type": "quantitative", "title": "Mean predicted", "format": ".3f"},
              {"field": "observed", "type": "quantitative", "title": "Observed", "format": ".3f"}
            ]
          }
        }
      ]
    },
    {
      "width": 400,
      "height": 90,
      "mark": {"type": "bar", "color": "#8fa4c8"},
      "encoding": {
        "x": {"field": "bin", "type": "ordinal", "title": "Predicted probability bin", "axis": {"labelAngle": 0, "labelFontSize": 9}},
        "y": {"field": "n", "type": "quantitative", "title": "Answers in bin"},
        "tooltip": [{"field": "bin", "type": "nominal"}, {"field": "n", "type": "quantitative"}]
      }
    }
  ],
  "config": {"view": {"stroke": "#d7dae0"}, "axis": {"labelFontSize": 11, "titleFontSize": 12}}
}