Retrieval pipeline ablation

// Compare removals from an evidence retrieval pipeline, from the permission prefilter to the citation verifier, against its fixed baseline; illustrative deltas are not measured performance or causal evidence.
// Synthetic worked example: all entities, schedules and measurements are illustrative.
// Every string below is EXAMPLE text from a fictional model: replace it.
// Replace assumptions and inputs before applying this model to actual work.

model baseline_model {
  title: "Illustrative Evidence retrieval"
  params: 12000000
  family: "Research prototype"
}

dataset evaluation_set {
  title: "Synthetic Held-out question set"
  n: 500
  split: "held-out"
}

eval baseline_eval {
  model: baseline_model
  dataset: evaluation_set
  metric: "Illustrative bounded quality score; higher is better"
  score: 0.82
}

ablation variant_0 {
  base: baseline_eval
  delta: -0.08
  change: "Remove permission prefilter"
}

ablation variant_1 {
  base: baseline_eval
  delta: -0.05
  change: "Remove reranker"
}

ablation variant_2 {
  base: baseline_eval
  delta: -0.035
  change: "Use fixed chunk boundaries"
}

ablation variant_3 {
  base: baseline_eval
  delta: -0.02
  change: "Remove query expansion"
}

ablation variant_4 {
  base: baseline_eval
  delta: -0.07
  change: "Remove citation verifier"
}

view variants: ablation_table(baseline_eval)