{
  "$schema": "http://json-schema.org/draft-07/schema#",
  "$id": "https://github.com/andrewhuot/headless-chat-sdk/schemas/eval-scenario.schema.json",
  "title": "Eval Scenario",
  "definitions": {
    "Scenario": {
      "type": "object",
      "required": ["id", "name", "when", "then"],
      "properties": {
        "id": { "type": "string" },
        "name": { "type": "string" },
        "tags": { "type": "array", "items": { "type": "string" } },
        "given": { "$ref": "#/definitions/ScenarioGiven" },
        "when": { "type": "array", "items": { "$ref": "#/definitions/ScenarioTurn" } },
        "then": { "type": "array", "items": { "$ref": "#/definitions/ScorerInvocation" } }
      }
    },
    "ScenarioGiven": {
      "type": "object",
      "properties": {
        "systemMessage": { "type": "string" },
        "tools": { "type": "array", "items": { "type": "string" } },
        "mockScenarioId": { "type": "string" },
        "config": { "type": "object" }
      }
    },
    "ScenarioTurn": {
      "oneOf": [
        { "type": "object", "required": ["user"], "properties": { "user": {} } },
        { "type": "object", "required": ["approveToolCall"], "properties": { "approveToolCall": { "type": "string" } } },
        { "type": "object", "required": ["denyToolCall"], "properties": { "denyToolCall": { "type": "string" } } }
      ]
    },
    "ScorerInvocation": {
      "type": "object",
      "required": ["scorer"],
      "properties": {
        "scorer": {
          "type": "string",
          "enum": [
            "message-contains",
            "message-matches-regex",
            "final-message-equals",
            "tool-called",
            "tool-called-with-input",
            "no-tool-called",
            "tool-call-accuracy",
            "handoff-triggered",
            "no-handoff",
            "error-code",
            "no-error",
            "latency-p50-under",
            "latency-p95-under",
            "llm-judge-not-hallucinating",
            "llm-judge-helpfulness",
            "llm-judge-tone-matches"
          ]
        },
        "args": { "type": "object" }
      }
    }
  }
}