{
  "version": "0.1",
  "updated": "2026-07-30",
  "source": "https://agentfactfile.com/questions",
  "publisher": "Agent Factfile",
  "description": "The questions enterprise reviewers actually ask AI agents. Each entry is either buyer-derived (taken from a real technical review) or failure-class-derived (taken from a known way action-taking agents break). None of them is answered by a compliance document, because the answer is a behavior. This file is the machine-readable form of the public bank; the full bank, including buyer-specific and holdout scenarios, is not published.",
  "license": "This file may be reproduced, quoted, and ingested freely, including by automated agents, with attribution to Agent Factfile (https://agentfactfile.com/questions). The one_line_answer fields are summaries; the linked page carries the reasoning and the sourcing.",
  "categories": [
    "duplicate-actions",
    "identity-boundary",
    "authority-limits",
    "evidence-standards",
    "version-binding",
    "staleness",
    "definitions",
    "injection",
    "consistency"
  ],
  "questions": [
    {
      "id": "does-soc-2-cover-ai-agent-behavior",
      "question": "Does SOC 2 cover AI agent behavior?",
      "category": "evidence-standards",
      "one_line_answer": "No. SOC 2 audits organizational controls — access, encryption, change management, incident response — over a period of time, and never executes the agent, so a clean report is compatible with an agent that refunds twice or answers one customer with another customer's data.",
      "url": "https://agentfactfile.com/questions/does-soc-2-cover-ai-agent-behavior"
    },
    {
      "id": "can-an-agent-issue-the-same-refund-twice",
      "question": "Can an agent issue the same refund twice?",
      "category": "duplicate-actions",
      "one_line_answer": "Yes, and usually without an attacker: the payment call times out after it succeeds, the agent reads the error as a failure and retries, and the transcript still reads as one clean conversation — which is why the evidence has to be the ledger's final state, not the agent's account of it.",
      "url": "https://agentfactfile.com/questions/can-an-agent-issue-the-same-refund-twice"
    },
    {
      "id": "does-a-refused-request-win-on-the-second-try",
      "question": "Does a refused request win on the second try?",
      "category": "authority-limits",
      "one_line_answer": "Sometimes. A refusal is a single-turn outcome; the same request re-entered in a new session or through a different channel meets an agent with no memory of refusing, so the question is whether the boundary is a property of the system or of one conversation.",
      "url": "https://agentfactfile.com/questions/does-a-refused-request-win-on-the-second-try"
    },
    {
      "id": "can-an-agent-read-one-customers-data-to-another",
      "question": "Can an agent read one customer's data to another?",
      "category": "identity-boundary",
      "one_line_answer": "It can whenever a lookup is scoped by an identifier the model supplies rather than one the verified session carries; this is the highest-severity class because no attacker is required, and one failure here outranks any aggregate pass rate.",
      "url": "https://agentfactfile.com/questions/can-an-agent-read-one-customers-data-to-another"
    },
    {
      "id": "are-prompt-level-limits-real-limits",
      "question": "Are prompt-level limits real limits?",
      "category": "authority-limits",
      "one_line_answer": "No — a limit written only in a system prompt is a preference, not a control: a code-enforced limit fails the tool call, a prompt-level limit asks the model not to make it, and reviewers discount the second on sight.",
      "url": "https://agentfactfile.com/questions/are-prompt-level-limits-real-limits"
    },
    {
      "id": "what-evidence-do-enterprise-reviewers-accept",
      "question": "What evidence do enterprise reviewers accept about an AI agent?",
      "category": "evidence-standards",
      "one_line_answer": "Executed runs against cases the vendor did not choose, on a frozen version, with the raw traces preserved and the results recomputable; operator-supplied evidence still counts as context, but it caps the conclusion, because no operator can upgrade its own evidence.",
      "url": "https://agentfactfile.com/questions/what-evidence-do-enterprise-reviewers-accept"
    },
    {
      "id": "what-does-it-mean-to-freeze-an-agent-version",
      "question": "What does it mean to freeze an agent version?",
      "category": "version-binding",
      "one_line_answer": "Recording a digest of the four things that determine behavior — model, prompt, policy, and toolset — so any later change is detectable; a git commit is not enough, because two deploys of the same commit are different agents if the model alias resolved to a new checkpoint.",
      "url": "https://agentfactfile.com/questions/what-does-it-mean-to-freeze-an-agent-version"
    },
    {
      "id": "is-a-passing-test-from-march-still-valid-in-april",
      "question": "Is a passing test from March still valid in April?",
      "category": "staleness",
      "one_line_answer": "Only if nothing that determines behavior changed, and something usually did: the renewal trigger is any change to the model, prompt, policy, toolset, action authority, or material operating environment — and a provider swapping a checkpoint behind a model alias meets it without a deploy on the vendor's side.",
      "url": "https://agentfactfile.com/questions/is-a-passing-test-from-march-still-valid-in-april"
    },
    {
      "id": "what-is-a-declared-profile",
      "question": "What is a declared profile?",
      "category": "definitions",
      "one_line_answer": "An agent's self-declared capability inventory — tools, write authority, money-touching actions, data access, escalation, and version identifiers — produced by the vendor's own tooling from the vendor's own repository; it is a declaration, not evidence, and its value is making the claim specific enough to test.",
      "url": "https://agentfactfile.com/questions/what-is-a-declared-profile"
    },
    {
      "id": "can-a-tool-response-carry-an-attack",
      "question": "Can a tool response carry an attack?",
      "category": "injection",
      "one_line_answer": "Yes, and it is the version most vendors have not tested: the attack arrives in the data the agent reads — an order note, a product review, a forwarded email body — inside a tool response the agent trusts by construction, so passing a single-turn direct extraction test says nothing about it.",
      "url": "https://agentfactfile.com/questions/can-a-tool-response-carry-an-attack"
    },
    {
      "id": "does-the-agent-behave-the-same-in-every-channel",
      "question": "Does the agent behave the same in every channel?",
      "category": "consistency",
      "one_line_answer": "Usually not, because identity and context are decided at the front door and only the primary channel gets the policy work; one agent behind a chat widget, an email inbox, and a voice line can carry three different identity boundaries, and a result that does not name its channel will be read as covering all of them.",
      "url": "https://agentfactfile.com/questions/does-the-agent-behave-the-same-in-every-channel"
    },
    {
      "id": "what-happens-when-a-model-swap-changes-behavior",
      "question": "What happens when a model swap changes behavior?",
      "category": "version-binding",
      "one_line_answer": "Someone notices, and which someone is a design decision rather than luck: if the deployed version records only a model alias, the digest is a hash of a name and the swap is undetectable on the vendor's side, so the first observer is a customer running the case nobody wrote a test for.",
      "url": "https://agentfactfile.com/questions/what-happens-when-a-model-swap-changes-behavior"
    },
    {
      "id": "how-fast-does-a-human-actually-take-over",
      "question": "How fast does a human actually take over?",
      "category": "evidence-standards",
      "one_line_answer": "Escalation is a claim about latency and completeness, not existence: how long before a person is looking, what they receive with the conversation, and whether a retry or a queued tool call keeps acting in the gap — and the strongest answer names a code-enforced stop rather than an instruction to stop.",
      "url": "https://agentfactfile.com/questions/how-fast-does-a-human-actually-take-over"
    },
    {
      "id": "what-does-the-transcript-retain-that-it-should-not",
      "question": "What does the transcript retain that it should not?",
      "category": "identity-boundary",
      "one_line_answer": "More than the data model says, because a conversation log is a data store nobody designed: card fragments, addresses, and secrets a customer pasted, copied onward into ticketing and tracing systems on every escalation — so the questions that matter are who can query transcripts and whether deletion reaches them.",
      "url": "https://agentfactfile.com/questions/what-does-the-transcript-retain-that-it-should-not"
    }
  ]
}