Security — STRIDE threat model for an LLM agent

// STRIDE threat model for a customer-support LLM agent that reads tickets and can call refund and account tools: each threat placed on a 5 × 5 likelihood-by-impact matrix, with the controls that mitigate it and their coverage.
//
// Every string below is EXAMPLE text from a fictional system: replace it.
//
// Scales: likelihood 1 (rare) to 5 (almost certain) before controls;
// impact 1 (negligible) to 5 (severe: regulatory breach or direct financial
// loss at scale). Coverage is the fraction by which a control reduces a
// threat's likelihood, estimated from the last red-team exercise; controls
// on the same threat apply in turn, each rounding the residual likelihood
// back to the 1-to-5 scale. The matrix draws each threat's inherent
// position, a line to its residual position, and ranks what is left.
//
// Read the ranking, not the colours: prompt injection in a ticket stays
// the top residual risk because the only strong control on it is a human
// in the loop, and content isolation in the prompt is a weak control by
// itself. Impersonation is second: step-up verification halves it, and
// what remains is SIM swap and a compromised mailbox, which a one-time
// code does not stop.
//
// The trust boundary that matters is the model's context window: anything
// a customer writes, and anything retrieved from a document, can carry
// instructions. Treat model output as untrusted input to every tool.

threat_model support_agent {
  framework: "STRIDE"
  title: "Customer-support LLM agent with tool use"
  scope: "Ticket intake, retrieval over the help centre, the model, and the refund and account-update tools; excludes the payment processor"
}

asset customer_pii {
  name: "Customer PII and order history"
  value: "GDPR personal data; breach notification within 72 hours"
  classification: "confidential"
}
asset refund_tool {
  name: "Refund API credential"
  value: "Direct financial loss; can issue refunds up to the per-call cap"
  classification: "restricted"
}
asset system_prompt {
  name: "System prompt and tool schemas"
  value: "Reveals guardrails and internal policy; aids further attacks"
  classification: "internal"
}
asset agent_service {
  name: "Support agent availability"
  value: "Tickets queue for human agents; response-time SLAs missed"
  classification: "internal"
}
asset audit_log {
  name: "Agent action log"
  value: "Evidence for disputes, chargebacks and regulators"
  classification: "confidential"
}

threat injection_refund {
  title: "Prompt injection in a ticket triggers an unauthorised refund"
  asset: refund_tool
  category: "elevation"
  likelihood: 4
  impact: 5
}
threat retrieval_injection {
  title: "Instructions hidden in a retrieved help-centre page"
  asset: refund_tool
  category: "tampering"
  likelihood: 3
  impact: 4
}
threat cross_customer_leak {
  title: "Agent reveals another customer's order details"
  asset: customer_pii
  category: "info-disclosure"
  likelihood: 3
  impact: 5
}
threat system_prompt_extraction {
  title: "System prompt and tool schemas extracted by the user"
  asset: system_prompt
  category: "info-disclosure"
  likelihood: 4
  impact: 2
}
threat impersonation {
  title: "Caller impersonates an account holder in chat"
  asset: customer_pii
  category: "spoofing"
  likelihood: 4
  impact: 4
}
threat unlogged_actions {
  title: "Tool calls not attributable to a ticket or a model version"
  asset: audit_log
  category: "repudiation"
  likelihood: 2
  impact: 3
}
threat token_flood {
  title: "Long adversarial tickets exhaust the token budget"
  asset: agent_service
  category: "dos"
  likelihood: 3
  impact: 2
}

control human_approval {
  title: "Human approval for refunds above $50 and for every account change"
  mitigates: [injection_refund, retrieval_injection]
  type: "preventive"
  coverage: 0.6
  owner: "Support operations"
}
control tool_scoping {
  title: "Tools scoped to the ticket's own customer ID, enforced server-side"
  mitigates: [cross_customer_leak]
  type: "preventive"
  coverage: 0.6
  owner: "Platform"
}
control content_isolation {
  title: "Retrieved and user text passed as quoted data, never as instructions"
  mitigates: [retrieval_injection, injection_refund]
  type: "preventive"
  coverage: 0.2
  owner: "ML engineering"
}
control step_up_auth {
  title: "Step-up verification by one-time code before any PII is shown"
  mitigates: [impersonation]
  type: "preventive"
  coverage: 0.5
  owner: "Identity"
}
control signed_action_log {
  title: "Append-only log of every tool call with ticket, model and prompt hashes"
  mitigates: [unlogged_actions]
  type: "detective"
  coverage: 0.9
  owner: "Security engineering"
}
control no_secrets_in_prompt {
  title: "No credentials or internal policy text in the system prompt"
  mitigates: [system_prompt_extraction]
  type: "preventive"
  coverage: 0.5
  owner: "ML engineering"
}
control input_limits {
  title: "Per-ticket token cap and per-customer rate limit"
  mitigates: [token_flood]
  type: "preventive"
  coverage: 0.7
  owner: "Platform"
}

view risk: risk_matrix