Confusion matrix — intent classifier held-out set
Row-normalised 6x6 confusion matrix with raw counts for a customer-support intent classifier on a 4,000-utterance held-out set, the figure an ML engineer files with every classifier release to show per-class recall and the dominant confusions.
Rendering…
Make it your own.
{
"$schema": "https://vega.github.io/schema/vega-lite/v5.json",
"title": {
"text": "Intent classifier v4.2 — confusion matrix, held-out set",
"subtitle": ["n = 4 000 utterances · overall accuracy 87.0% (3 480/4 000) · macro-recall 0.843", "Rows are the adjudicated true intent; columns the argmax prediction. Colour is row-normalised recall."],
"anchor": "start",
"fontSize": 15,
"subtitleFontSize": 11
},
"data": {
"values": [
{"actual":"card_lost","predicted":"card_lost","n":471,"rate":0.906},
{"actual":"card_lost","predicted":"dispute_txn","n":18,"rate":0.035},
{"actual":"card_lost","predicted":"balance_enquiry","n":6,"rate":0.012},
{"actual":"card_lost","predicted":"payment_failed","n":9,"rate":0.017},
{"actual":"card_lost","predicted":"close_account","n":3,"rate":0.006},
{"actual":"card_lost","predicted":"other","n":13,"rate":0.025},
{"actual":"dispute_txn","predicted":"card_lost","n":22,"rate":0.051},
{"actual":"dispute_txn","predicted":"dispute_txn","n":341,"rate":0.793},
{"actual":"dispute_txn","predicted":"balance_enquiry","n":4,"rate":0.009},
{"actual":"dispute_txn","predicted":"payment_failed","n":31,"rate":0.072},
{"actual":"dispute_txn","predicted":"close_account","n":2,"rate":0.005},
{"actual":"dispute_txn","predicted":"other","n":30,"rate":0.070},
{"actual":"balance_enquiry","predicted":"card_lost","n":5,"rate":0.005},
{"actual":"balance_enquiry","predicted":"dispute_txn","n":7,"rate":0.007},
{"actual":"balance_enquiry","predicted":"balance_enquiry","n":918,"rate":0.937},
{"actual":"balance_enquiry","predicted":"payment_failed","n":24,"rate":0.024},
{"actual":"balance_enquiry","predicted":"close_account","n":1,"rate":0.001},
{"actual":"balance_enquiry","predicted":"other","n":25,"rate":0.026},
{"actual":"payment_failed","predicted":"card_lost","n":8,"rate":0.013},
{"actual":"payment_failed","predicted":"dispute_txn","n":27,"rate":0.044},
{"actual":"payment_failed","predicted":"balance_enquiry","n":19,"rate":0.031},
{"actual":"payment_failed","predicted":"payment_failed","n":512,"rate":0.839},
{"actual":"payment_failed","predicted":"close_account","n":2,"rate":0.003},
{"actual":"payment_failed","predicted":"other","n":42,"rate":0.069},
{"actual":"close_account","predicted":"card_lost","n":4,"rate":0.022},
{"actual":"close_account","predicted":"dispute_txn","n":3,"rate":0.017},
{"actual":"close_account","predicted":"balance_enquiry","n":2,"rate":0.011},
{"actual":"close_account","predicted":"payment_failed","n":1,"rate":0.006},
{"actual":"close_account","predicted":"close_account","n":129,"rate":0.717},
{"actual":"close_account","predicted":"other","n":41,"rate":0.228},
{"actual":"other","predicted":"card_lost","n":16,"rate":0.013},
{"actual":"other","predicted":"dispute_txn","n":34,"rate":0.027},
{"actual":"other","predicted":"balance_enquiry","n":52,"rate":0.041},
{"actual":"other","predicted":"payment_failed","n":47,"rate":0.037},
{"actual":"other","predicted":"close_account","n":22,"rate":0.017},
{"actual":"other","predicted":"other","n":1109,"rate":0.866}
]
},
"width": 430,
"height": 330,
"encoding": {
"x": {
"field": "predicted", "type": "nominal", "title": "Predicted intent",
"sort": ["card_lost","dispute_txn","balance_enquiry","payment_failed","close_account","other"],
"axis": {"orient": "top", "labelAngle": -35}
},
"y": {
"field": "actual", "type": "nominal", "title": "True intent (adjudicated)",
"sort": ["card_lost","dispute_txn","balance_enquiry","payment_failed","close_account","other"]
}
},
"layer": [
{
"mark": {"type": "rect", "stroke": "#ffffff", "strokeWidth": 1},
"encoding": {
"color": {
"field": "rate", "type": "quantitative",
"title": "Row-normalised rate",
"scale": {"scheme": "blues", "domain": [0, 1]},
"legend": {"format": ".0%"}
},
"tooltip": [
{"field": "actual", "type": "nominal", "title": "True"},
{"field": "predicted", "type": "nominal", "title": "Predicted"},
{"field": "n", "type": "quantitative", "title": "Count"},
{"field": "rate", "type": "quantitative", "title": "Rate", "format": ".1%"}
]
}
},
{
"mark": {"type": "text", "fontSize": 11},
"encoding": {
"text": {"field": "n", "type": "quantitative"},
"color": {
"condition": {"test": "datum.rate > 0.5", "value": "#ffffff"},
"value": "#333333"
}
}
}
],
"config": {"view": {"stroke": null}, "axis": {"labelFontSize": 11, "titleFontSize": 12}}
}