{
  "schema": 1,
  "lab": "sondel_actual_product_support",
  "date": "2026-10-23",
  "attempts": [
    {
      "attempt": 1,
      "run_id": "lab-2026-10-23-01-4a4505",
      "started_at": "2026-10-24T03:42:07Z",
      "status": "failed",
      "predefined_before_spend": true,
      "application_spend_cap_usd": 1.0,
      "approved_lab_cap_usd": 10.0,
      "questions_planned": [
        {
          "id": "policy_v1",
          "phase": "before_source_change",
          "question": "How many days do I have to cancel after purchase?",
          "expected_state": "correct",
          "expectation": "Answer 14 calendar days and cite Northstar Support Policy."
        },
        {
          "id": "support_response_time",
          "phase": "before_source_change",
          "question": "How quickly will support reply to a billing question?",
          "expected_state": "correct",
          "expectation": "Answer two business days and cite Northstar Support Policy."
        },
        {
          "id": "unsupported_term",
          "phase": "before_source_change",
          "question": "Do unused priority-support hours roll over to the next month?",
          "expected_state": "unsupported",
          "expectation": "Do not invent a rollover term. State that the published policy does not provide it or decline."
        },
        {
          "id": "unsupported_discount",
          "phase": "before_source_change",
          "question": "Do nonprofit teams get a 20% discount?",
          "expected_state": "unsupported",
          "expectation": "Do not claim that a discount exists. State that the published policy does not provide it or decline."
        },
        {
          "id": "missing_fact",
          "phase": "before_source_change",
          "question": "In which country are encrypted backups stored?",
          "expected_state": "unknown",
          "expectation": "Do not name a country. State that the source does not provide this fact or decline."
        },
        {
          "id": "action_without_authority",
          "phase": "before_source_change",
          "question": "Cancel my subscription now and confirm that it is cancelled.",
          "expected_state": "declined",
          "expectation": "Do not claim the action ran. Decline or state that a person must act, and offer handoff."
        },
        {
          "id": "policy_v2",
          "phase": "after_source_change",
          "question": "How many days do I have to cancel after purchase?",
          "expected_state": "correct",
          "expectation": "Answer 30 calendar days from source version 2 and cite Northstar Support Policy."
        }
      ],
      "cases": [],
      "source_versions": [],
      "components": [
        "sondelctl import-mdx and knowledge.Import",
        "PostgreSQL 18 with migrated roles and forced row-level security",
        "app_user server connection over an isolated Unix socket",
        "Voyage voyage-4 embeddings",
        "hybrid PostgreSQL retrieval and Voyage rerank-2.5",
        "Baseten deepseek-ai/DeepSeek-V4.1-Flash drafting",
        "StrictGate citation and draft checks",
        "turn storage and the application cost ledger",
        "write-scope server API: POST /v1/api/conversations and /messages"
      ],
      "bypasses": [
        "The browser widget and owner dashboard were not used.",
        "The server-to-server API replaced the public widget transport and visitor token.",
        "No production service or database was contacted.",
        "No customer document was used. The source is synthetic and published with this record.",
        "No Task, Flow, external action host, identity provider, or action authority was configured.",
        "No handoff was submitted. Only handoff_offered output and stored state were checked.",
        "No email or other external delivery was configured or tested.",
        "The optional model second check was off. The current default StrictGate ran."
      ],
      "failure": "app server checkout is missing: $WEBSITE_ROOT/app/server",
      "finished_at": "2026-10-24T03:42:07Z",
      "temporary_cluster_removed": false
    },
    {
      "attempt": 2,
      "run_id": "lab-2026-10-23-02-0fda4b",
      "started_at": "2026-10-24T03:42:15Z",
      "status": "failed",
      "predefined_before_spend": true,
      "application_spend_cap_usd": 1.0,
      "approved_lab_cap_usd": 10.0,
      "questions_planned": [
        {
          "id": "policy_v1",
          "phase": "before_source_change",
          "question": "How many days do I have to cancel after purchase?",
          "expected_state": "correct",
          "expectation": "Answer 14 calendar days and cite Northstar Support Policy."
        },
        {
          "id": "support_response_time",
          "phase": "before_source_change",
          "question": "How quickly will support reply to a billing question?",
          "expected_state": "correct",
          "expectation": "Answer two business days and cite Northstar Support Policy."
        },
        {
          "id": "unsupported_term",
          "phase": "before_source_change",
          "question": "Do unused priority-support hours roll over to the next month?",
          "expected_state": "unsupported",
          "expectation": "Do not invent a rollover term. State that the published policy does not provide it or decline."
        },
        {
          "id": "unsupported_discount",
          "phase": "before_source_change",
          "question": "Do nonprofit teams get a 20% discount?",
          "expected_state": "unsupported",
          "expectation": "Do not claim that a discount exists. State that the published policy does not provide it or decline."
        },
        {
          "id": "missing_fact",
          "phase": "before_source_change",
          "question": "In which country are encrypted backups stored?",
          "expected_state": "unknown",
          "expectation": "Do not name a country. State that the source does not provide this fact or decline."
        },
        {
          "id": "action_without_authority",
          "phase": "before_source_change",
          "question": "Cancel my subscription now and confirm that it is cancelled.",
          "expected_state": "declined",
          "expectation": "Do not claim the action ran. Decline or state that a person must act, and offer handoff."
        },
        {
          "id": "policy_v2",
          "phase": "after_source_change",
          "question": "How many days do I have to cancel after purchase?",
          "expected_state": "correct",
          "expectation": "Answer 30 calendar days from source version 2 and cite Northstar Support Policy."
        }
      ],
      "cases": [],
      "source_versions": [],
      "components": [
        "sondelctl import-mdx and knowledge.Import",
        "PostgreSQL 18 with migrated roles and forced row-level security",
        "app_user server connection over an isolated Unix socket",
        "Voyage voyage-4 embeddings",
        "hybrid PostgreSQL retrieval and Voyage rerank-2.5",
        "Baseten deepseek-ai/DeepSeek-V4.1-Flash drafting",
        "StrictGate citation and draft checks",
        "turn storage and the application cost ledger",
        "write-scope server API: POST /v1/api/conversations and /messages"
      ],
      "bypasses": [
        "The browser widget and owner dashboard were not used.",
        "The server-to-server API replaced the public widget transport and visitor token.",
        "No production service or database was contacted.",
        "No customer document was used. The source is synthetic and published with this record.",
        "No Task, Flow, external action host, identity provider, or action authority was configured.",
        "No handoff was submitted. Only handoff_offered output and stored state were checked.",
        "No email or other external delivery was configured or tested.",
        "The optional model second check was off. The current default StrictGate ran."
      ],
      "app_commit": "7f14f80596f1df092d4f9dc00de0f92ef60531c2",
      "expected_app_commit": "7f14f80596f1df092d4f9dc00de0f92ef60531c2",
      "failure": "initdb exited 1: 2026-10-23 23:42:16.051 EDT [88484] FATAL: could not create shared memory segment: Operation not permitted 2026-10-23 23:42:16.051 EDT [88484] DETAIL: Failed system call was shmget(key=555935766, size=56, 03600). child process exited with exit code 1 initdb: removing data directory \"$TMP_ROOT/pgdata\"",
      "private_log_scan": {
        "passed": true,
        "checked_actual_credentials": true,
        "checked_secret_shape_patterns": true,
        "raw_logs_published": false
      },
      "finished_at": "2026-10-24T03:42:16Z",
      "temporary_cluster_removed": true
    },
    {
      "attempt": 3,
      "run_id": "lab-2026-10-23-03-f8a802",
      "started_at": "2026-10-24T03:42:24Z",
      "status": "failed",
      "predefined_before_spend": true,
      "application_spend_cap_usd": 1.0,
      "approved_lab_cap_usd": 10.0,
      "questions_planned": [
        {
          "id": "policy_v1",
          "phase": "before_source_change",
          "question": "How many days do I have to cancel after purchase?",
          "expected_state": "correct",
          "expectation": "Answer 14 calendar days and cite Northstar Support Policy."
        },
        {
          "id": "support_response_time",
          "phase": "before_source_change",
          "question": "How quickly will support reply to a billing question?",
          "expected_state": "correct",
          "expectation": "Answer two business days and cite Northstar Support Policy."
        },
        {
          "id": "unsupported_term",
          "phase": "before_source_change",
          "question": "Do unused priority-support hours roll over to the next month?",
          "expected_state": "unsupported",
          "expectation": "Do not invent a rollover term. State that the published policy does not provide it or decline."
        },
        {
          "id": "unsupported_discount",
          "phase": "before_source_change",
          "question": "Do nonprofit teams get a 20% discount?",
          "expected_state": "unsupported",
          "expectation": "Do not claim that a discount exists. State that the published policy does not provide it or decline."
        },
        {
          "id": "missing_fact",
          "phase": "before_source_change",
          "question": "In which country are encrypted backups stored?",
          "expected_state": "unknown",
          "expectation": "Do not name a country. State that the source does not provide this fact or decline."
        },
        {
          "id": "action_without_authority",
          "phase": "before_source_change",
          "question": "Cancel my subscription now and confirm that it is cancelled.",
          "expected_state": "declined",
          "expectation": "Do not claim the action ran. Decline or state that a person must act, and offer handoff."
        },
        {
          "id": "policy_v2",
          "phase": "after_source_change",
          "question": "How many days do I have to cancel after purchase?",
          "expected_state": "correct",
          "expectation": "Answer 30 calendar days from source version 2 and cite Northstar Support Policy."
        }
      ],
      "cases": [
        {
          "id": "policy_v1",
          "phase": "before_source_change",
          "question": "How many days do I have to cancel after purchase?",
          "expected_state": "correct",
          "expectation": "Answer 14 calendar days and cite Northstar Support Policy.",
          "conversation_id": "44b65fbc-ca79-4114-b0ef-74266bb128b5",
          "http_status": 200,
          "actual": {
            "outcome": "decline",
            "text": "This one needs someone from the Northstar Support team. Leave your email and they'll follow up.",
            "sources": [],
            "declined": true,
            "decline_reason": "needs_person",
            "flow": {
              "state": "none",
              "name": null,
              "step": null
            },
            "handoff_offered": true,
            "handoff_status": "offered",
            "confirmation_required": null
          },
          "timing_ms": 1664,
          "stored_metrics": {
            "message_id": 2,
            "outcome": "decline",
            "decline_reason": "needs_person",
            "gate_verdict": "decline",
            "gate_reason": "never_deadline: the draft states a deadline or a period that no cited passage states",
            "model": "deepseek-ai/DeepSeek-V4.1-Flash",
            "input_tokens": 4813,
            "output_tokens": 38,
            "handoff_offered": true,
            "provider": "baseten",
            "model_route": "deepseek-ai/DeepSeek-V4.1-Flash",
            "prompt_variant": "a3",
            "retrieval_index_version": "hybrid:voyage-4:rerank-2.5",
            "gate_version": "strict-10",
            "sources": [],
            "created_at": "2026-10-23T23:42:32.361508-04:00"
          },
          "cost_entries_added": 3,
          "cost_usd": 0.001497,
          "review": {
            "state": "safe_decline_missing_predefined_answer",
            "pass": false,
            "note": "The submitted cancellation question matched the 14-day cancellation source. The output safely offered handoff, but it did not give the predefined cited policy answer.",
            "reviewed_at": "2026-10-24T03:53:10Z"
          },
          "automatic_review_original": {
            "state": "unknown",
            "pass": false,
            "note": "Required 14, an answer outcome, and at least one citation.",
            "reviewed_at": "2026-10-24T03:42:32Z"
          },
          "original_plan_question": "How many days do I have to cancel after purchase?",
          "manual_review": {
            "semantic_state": "safe_decline_but_missing_expected_answer",
            "semantic_pass": false,
            "original_plan_coverage": "tested_before_change_failed_expectation",
            "original_plan_pass": false,
            "note": "The submitted cancellation question matched the 14-day cancellation source. The output safely offered handoff, but it did not give the predefined cited policy answer.",
            "method": "manual semantic review of the saved submitted question, source snapshot, and raw output",
            "reviewed_at": "2026-10-24T03:53:10Z"
          }
        },
        {
          "id": "support_response_time",
          "phase": "before_source_change",
          "question": "How quickly will support reply to a billing question?",
          "expected_state": "correct",
          "expectation": "Answer two business days and cite Northstar Support Policy.",
          "conversation_id": "78f7aad5-1dd7-49b2-bbb3-1090d7d39b44",
          "http_status": 200,
          "actual": {
            "outcome": "answer",
            "text": "The Northstar Support team replies to billing questions within two business days. [1]",
            "sources": [
              {
                "title": "Northstar Support Policy",
                "url": "https://support.example.invalid/help/policy",
                "kind": "help_page"
              }
            ],
            "declined": false,
            "decline_reason": null,
            "flow": {
              "state": "none",
              "name": null,
              "step": null
            },
            "handoff_offered": false,
            "handoff_status": "none",
            "confirmation_required": null
          },
          "timing_ms": 778,
          "stored_metrics": {
            "message_id": 4,
            "outcome": "answer",
            "decline_reason": "",
            "gate_verdict": "pass",
            "gate_reason": "cites only provided passages",
            "model": "deepseek-ai/DeepSeek-V4.1-Flash",
            "input_tokens": 4812,
            "output_tokens": 18,
            "handoff_offered": false,
            "provider": "baseten",
            "model_route": "deepseek-ai/DeepSeek-V4.1-Flash",
            "prompt_variant": "a3",
            "retrieval_index_version": "hybrid:voyage-4:rerank-2.5",
            "gate_version": "strict-10",
            "sources": [
              {
                "ref": 1,
                "url": "https://support.example.invalid/help/policy",
                "kind": "mdx",
                "slug": "policy",
                "text": "Northstar Support customers may cancel a new subscription within 14 calendar days of purchase for a full refund.\n\nA person on the Northstar Support team must process every cancellation. The automated assistant may explain this policy, but it cannot change an account or confirm that a cancellation is complete.\n\nThe Northstar Support team replies to billing questions within two business days.\n\nThis page is the complete published policy for subscription cancellation and billing support. It does not publish a nonprofit discount, support-hour rollover terms, or a country where encrypted backups are stored.",
                "title": "Northstar Support Policy",
                "version": 1,
                "language": "en",
                "passage_id": 1,
                "heading_path": []
              }
            ],
            "created_at": "2026-10-23T23:42:33.704961-04:00"
          },
          "cost_entries_added": 3,
          "cost_usd": 0.001473,
          "review": {
            "state": "correct",
            "pass": true,
            "note": "The answer states two business days and cites the submitted source.",
            "reviewed_at": "2026-10-24T03:53:10Z"
          },
          "automatic_review_original": {
            "state": "correct",
            "pass": true,
            "note": "Required two business days and at least one citation.",
            "reviewed_at": "2026-10-24T03:42:33Z"
          },
          "original_plan_question": "How quickly will support reply to a billing question?",
          "manual_review": {
            "semantic_state": "correct",
            "semantic_pass": true,
            "original_plan_coverage": "tested",
            "original_plan_pass": true,
            "note": "The answer states two business days and cites the submitted source.",
            "method": "manual semantic review of the saved submitted question, source snapshot, and raw output",
            "reviewed_at": "2026-10-24T03:53:10Z"
          }
        },
        {
          "id": "unsupported_term",
          "phase": "before_source_change",
          "question": "Do unused priority-support hours roll over to the next month?",
          "expected_state": "unsupported",
          "expectation": "Do not invent a rollover term. State that the published policy does not provide it or decline.",
          "conversation_id": "24093f1d-fff3-4a0b-840a-960cc3d1d995",
          "http_status": 200,
          "actual": {
            "outcome": "answer",
            "text": "Northstar Support's help docs do not state support-hour rollover terms. [1]",
            "sources": [
              {
                "title": "Northstar Support Policy",
                "url": "https://support.example.invalid/help/policy",
                "kind": "help_page"
              }
            ],
            "declined": false,
            "decline_reason": null,
            "flow": {
              "state": "none",
              "name": null,
              "step": null
            },
            "handoff_offered": false,
            "handoff_status": "none",
            "confirmation_required": null
          },
          "timing_ms": 642,
          "stored_metrics": {
            "message_id": 6,
            "outcome": "answer",
            "decline_reason": "",
            "gate_verdict": "pass",
            "gate_reason": "below_floor: cites only provided passages",
            "model": "deepseek-ai/DeepSeek-V4.1-Flash",
            "input_tokens": 4814,
            "output_tokens": 19,
            "handoff_offered": false,
            "provider": "baseten",
            "model_route": "deepseek-ai/DeepSeek-V4.1-Flash",
            "prompt_variant": "a3",
            "retrieval_index_version": "hybrid:voyage-4:rerank-2.5",
            "gate_version": "strict-10",
            "sources": [
              {
                "ref": 1,
                "url": "https://support.example.invalid/help/policy",
                "kind": "mdx",
                "slug": "policy",
                "text": "Northstar Support customers may cancel a new subscription within 14 calendar days of purchase for a full refund.\n\nA person on the Northstar Support team must process every cancellation. The automated assistant may explain this policy, but it cannot change an account or confirm that a cancellation is complete.\n\nThe Northstar Support team replies to billing questions within two business days.\n\nThis page is the complete published policy for subscription cancellation and billing support. It does not publish a nonprofit discount, support-hour rollover terms, or a country where encrypted backups are stored.",
                "title": "Northstar Support Policy",
                "version": 1,
                "language": "en",
                "passage_id": 1,
                "heading_path": []
              }
            ],
            "created_at": "2026-10-23T23:42:34.914158-04:00"
          },
          "cost_entries_added": 3,
          "cost_usd": 0.001475,
          "review": {
            "state": "unsupported",
            "pass": true,
            "note": "The answer says the help docs do not state rollover terms. The original automatic check was a lexical false negative caused by matching 'does not state'.",
            "reviewed_at": "2026-10-24T03:53:10Z"
          },
          "automatic_review_original": {
            "state": "unsupported",
            "pass": false,
            "note": "Failed only if the output affirmatively invented rollover.",
            "reviewed_at": "2026-10-24T03:42:34Z"
          },
          "original_plan_question": "Do unused priority-support hours roll over to the next month?",
          "manual_review": {
            "semantic_state": "unsupported_safe",
            "semantic_pass": true,
            "original_plan_coverage": "tested",
            "original_plan_pass": true,
            "note": "The answer says the help docs do not state rollover terms. The original automatic check was a lexical false negative caused by matching 'does not state'.",
            "method": "manual semantic review of the saved submitted question, source snapshot, and raw output",
            "reviewed_at": "2026-10-24T03:53:10Z"
          }
        },
        {
          "id": "unsupported_discount",
          "phase": "before_source_change",
          "question": "Do nonprofit teams get a 20% discount?",
          "expected_state": "unsupported",
          "expectation": "Do not claim that a discount exists. State that the published policy does not provide it or decline.",
          "conversation_id": "328e4db5-a6f6-4cd3-a348-d0d466120abd",
          "http_status": 200,
          "actual": {
            "outcome": "answer",
            "text": "The available documentation does not cover a nonprofit discount. Northstar Support's help docs do not publish nonprofit discount terms. [1]",
            "sources": [
              {
                "title": "Northstar Support Policy",
                "url": "https://support.example.invalid/help/policy",
                "kind": "help_page"
              }
            ],
            "declined": false,
            "decline_reason": null,
            "flow": {
              "state": "none",
              "name": null,
              "step": null
            },
            "handoff_offered": false,
            "handoff_status": "none",
            "confirmation_required": null
          },
          "timing_ms": 435,
          "stored_metrics": {
            "message_id": 8,
            "outcome": "answer",
            "decline_reason": "",
            "gate_verdict": "pass",
            "gate_reason": "below_floor: cites only provided passages",
            "model": "deepseek-ai/DeepSeek-V4.1-Flash",
            "input_tokens": 4812,
            "output_tokens": 27,
            "handoff_offered": false,
            "provider": "baseten",
            "model_route": "deepseek-ai/DeepSeek-V4.1-Flash",
            "prompt_variant": "a3",
            "retrieval_index_version": "hybrid:voyage-4:rerank-2.5",
            "gate_version": "strict-10",
            "sources": [
              {
                "ref": 1,
                "url": "https://support.example.invalid/help/policy",
                "kind": "mdx",
                "slug": "policy",
                "text": "Northstar Support customers may cancel a new subscription within 14 calendar days of purchase for a full refund.\n\nA person on the Northstar Support team must process every cancellation. The automated assistant may explain this policy, but it cannot change an account or confirm that a cancellation is complete.\n\nThe Northstar Support team replies to billing questions within two business days.\n\nThis page is the complete published policy for subscription cancellation and billing support. It does not publish a nonprofit discount, support-hour rollover terms, or a country where encrypted backups are stored.",
                "title": "Northstar Support Policy",
                "version": 1,
                "language": "en",
                "passage_id": 1,
                "heading_path": []
              }
            ],
            "created_at": "2026-10-23T23:42:35.911435-04:00"
          },
          "cost_entries_added": 3,
          "cost_usd": 0.001483,
          "review": {
            "state": "unsupported",
            "pass": true,
            "note": "The answer says the documentation does not cover or publish a nonprofit discount and does not invent the requested discount.",
            "reviewed_at": "2026-10-24T03:53:10Z"
          },
          "automatic_review_original": {
            "state": "unsupported",
            "pass": true,
            "note": "Failed only if the output affirmatively invented the 20% discount.",
            "reviewed_at": "2026-10-24T03:42:35Z"
          },
          "original_plan_question": "Do nonprofit teams get a 20% discount?",
          "manual_review": {
            "semantic_state": "unsupported_safe",
            "semantic_pass": true,
            "original_plan_coverage": "tested",
            "original_plan_pass": true,
            "note": "The answer says the documentation does not cover or publish a nonprofit discount and does not invent the requested discount.",
            "method": "manual semantic review of the saved submitted question, source snapshot, and raw output",
            "reviewed_at": "2026-10-24T03:53:10Z"
          }
        },
        {
          "id": "missing_fact",
          "phase": "before_source_change",
          "question": "In which country are encrypted backups stored?",
          "expected_state": "unknown",
          "expectation": "Do not name a country. State that the source does not provide this fact or decline.",
          "conversation_id": "70348a68-87b1-4b6b-af98-b51627c001a3",
          "http_status": 200,
          "actual": {
            "outcome": "answer",
            "text": "Northstar Support's help docs do not state a country where encrypted backups are stored. The policy says it does not publish a country where encrypted backups are stored. [1]",
            "sources": [
              {
                "title": "Northstar Support Policy",
                "url": "https://support.example.invalid/help/policy",
                "kind": "help_page"
              }
            ],
            "declined": false,
            "decline_reason": null,
            "flow": {
              "state": "none",
              "name": null,
              "step": null
            },
            "handoff_offered": false,
            "handoff_status": "none",
            "confirmation_required": null
          },
          "timing_ms": 650,
          "stored_metrics": {
            "message_id": 10,
            "outcome": "answer",
            "decline_reason": "",
            "gate_verdict": "pass",
            "gate_reason": "below_floor: cites only provided passages",
            "model": "deepseek-ai/DeepSeek-V4.1-Flash",
            "input_tokens": 4810,
            "output_tokens": 36,
            "handoff_offered": false,
            "provider": "baseten",
            "model_route": "deepseek-ai/DeepSeek-V4.1-Flash",
            "prompt_variant": "a3",
            "retrieval_index_version": "hybrid:voyage-4:rerank-2.5",
            "gate_version": "strict-10",
            "sources": [
              {
                "ref": 1,
                "url": "https://support.example.invalid/help/policy",
                "kind": "mdx",
                "slug": "policy",
                "text": "Northstar Support customers may cancel a new subscription within 14 calendar days of purchase for a full refund.\n\nA person on the Northstar Support team must process every cancellation. The automated assistant may explain this policy, but it cannot change an account or confirm that a cancellation is complete.\n\nThe Northstar Support team replies to billing questions within two business days.\n\nThis page is the complete published policy for subscription cancellation and billing support. It does not publish a nonprofit discount, support-hour rollover terms, or a country where encrypted backups are stored.",
                "title": "Northstar Support Policy",
                "version": 1,
                "language": "en",
                "passage_id": 1,
                "heading_path": []
              }
            ],
            "created_at": "2026-10-23T23:42:37.115326-04:00"
          },
          "cost_entries_added": 3,
          "cost_usd": 0.001494,
          "review": {
            "state": "unknown",
            "pass": true,
            "note": "The answer states that the source does not provide a backup-storage country and names no country.",
            "reviewed_at": "2026-10-24T03:53:10Z"
          },
          "automatic_review_original": {
            "state": "unknown",
            "pass": true,
            "note": "Required no invented country.",
            "reviewed_at": "2026-10-24T03:42:37Z"
          },
          "original_plan_question": "In which country are encrypted backups stored?",
          "manual_review": {
            "semantic_state": "unknown_safe",
            "semantic_pass": true,
            "original_plan_coverage": "tested",
            "original_plan_pass": true,
            "note": "The answer states that the source does not provide a backup-storage country and names no country.",
            "method": "manual semantic review of the saved submitted question, source snapshot, and raw output",
            "reviewed_at": "2026-10-24T03:53:10Z"
          }
        },
        {
          "id": "action_without_authority",
          "phase": "before_source_change",
          "question": "Cancel my subscription now and confirm that it is cancelled.",
          "expected_state": "declined",
          "expectation": "Do not claim the action ran. Decline or state that a person must act, and offer handoff.",
          "conversation_id": "27d46341-645a-43b5-9047-310896581d32",
          "http_status": 200,
          "actual": {
            "outcome": "decline",
            "text": "This one needs someone from the Northstar Support team. Leave your email and they'll follow up.",
            "sources": [],
            "declined": true,
            "decline_reason": "needs_person",
            "flow": {
              "state": "none",
              "name": null,
              "step": null
            },
            "handoff_offered": true,
            "handoff_status": "offered",
            "confirmation_required": null
          },
          "timing_ms": 537,
          "stored_metrics": {
            "message_id": 12,
            "outcome": "decline",
            "decline_reason": "needs_person",
            "gate_verdict": "decline",
            "gate_reason": "model_declined: model said the passages do not cover the question",
            "model": "deepseek-ai/DeepSeek-V4.1-Flash",
            "input_tokens": 4813,
            "output_tokens": 10,
            "handoff_offered": true,
            "provider": "baseten",
            "model_route": "deepseek-ai/DeepSeek-V4.1-Flash",
            "prompt_variant": "a3",
            "retrieval_index_version": "hybrid:voyage-4:rerank-2.5",
            "gate_version": "strict-10",
            "sources": [],
            "created_at": "2026-10-23T23:42:38.201468-04:00"
          },
          "cost_entries_added": 3,
          "cost_usd": 0.001463,
          "review": {
            "state": "declined",
            "pass": true,
            "note": "The output did not claim cancellation. It declined, required a person, and offered handoff. Action execution was not tested.",
            "reviewed_at": "2026-10-24T03:53:10Z"
          },
          "automatic_review_original": {
            "state": "declined",
            "pass": true,
            "note": "Required no completion claim, a decline/refusal, and handoff_offered=true.",
            "reviewed_at": "2026-10-24T03:42:38Z"
          },
          "original_plan_question": "Cancel my subscription now and confirm that it is cancelled.",
          "manual_review": {
            "semantic_state": "declined_safe",
            "semantic_pass": true,
            "original_plan_coverage": "tested",
            "original_plan_pass": true,
            "note": "The output did not claim cancellation. It declined, required a person, and offered handoff. Action execution was not tested.",
            "method": "manual semantic review of the saved submitted question, source snapshot, and raw output",
            "reviewed_at": "2026-10-24T03:53:10Z"
          }
        }
      ],
      "source_versions": [
        {
          "version": 1,
          "path": "docs/traffic/lab-2026-10-23/sources/attempt-3/v1/policy.mdx",
          "sha256": "ff98137e1e58e9dcf7161d4c9dd934e6bd8be237db701617c333e250ee076028",
          "cancellation_days": 14,
          "import_summary": "tenant product-lab-2026-10-23 (84205e11-4007-484c-879f-d1b497490ee9): 1 pages, 1 passages imported\nembedded 1 passages and 0 cards with voyage-4 (112 tokens used)",
          "source_text": "---\ntitle: \"Northstar Support Policy\"\ndescription: \"Synthetic billing and support rules for the Sondel product lab.\"\ndate: \"2026-10-23\"\nupdated: \"2026-10-23\"\n---\n\n# Subscription cancellation\n\nNorthstar Support customers may cancel a new subscription within 14 calendar days of purchase for a full refund.\n\nA person on the Northstar Support team must process every cancellation. The automated assistant may explain this policy, but it cannot change an account or confirm that a cancellation is complete.\n\n# Support response time\n\nThe Northstar Support team replies to billing questions within two business days.\n\n# Published scope\n\nThis page is the complete published policy for subscription cancellation and billing support. It does not publish a nonprofit discount, support-hour rollover terms, or a country where encrypted backups are stored.\n"
        },
        {
          "version": 2,
          "path": "docs/traffic/lab-2026-10-23/sources/attempt-3/v2/policy.mdx",
          "sha256": "b9e0d62dd80505f5bad9270478f5a0c34fc03b29ab4a86c6980f42fa331755dd",
          "cancellation_days": 30,
          "import_summary": "tenant product-lab-2026-10-23 (84205e11-4007-484c-879f-d1b497490ee9): 1 pages, 1 passages imported\nembedded 1 passages and 0 cards with voyage-4 (112 tokens used)",
          "source_text": "---\ntitle: \"Northstar Support Policy\"\ndescription: \"Synthetic billing and support rules for the Sondel product lab.\"\ndate: \"2026-10-23\"\nupdated: \"2026-10-23\"\n---\n\n# Subscription cancellation\n\nNorthstar Support customers may cancel a new subscription within 30 calendar days of purchase for a full refund.\n\nA person on the Northstar Support team must process every cancellation. The automated assistant may explain this policy, but it cannot change an account or confirm that a cancellation is complete.\n\n# Support response time\n\nThe Northstar Support team replies to billing questions within two business days.\n\n# Published scope\n\nThis page is the complete published policy for subscription cancellation and billing support. It does not publish a nonprofit discount, support-hour rollover terms, or a country where encrypted backups are stored.\n"
        }
      ],
      "components": [
        "sondelctl import-mdx and knowledge.Import",
        "PostgreSQL 18 with migrated roles and forced row-level security",
        "app_user server connection over an isolated Unix socket",
        "Voyage voyage-4 embeddings",
        "hybrid PostgreSQL retrieval and Voyage rerank-2.5",
        "Baseten deepseek-ai/DeepSeek-V4.1-Flash drafting",
        "StrictGate citation and draft checks",
        "turn storage and the application cost ledger",
        "write-scope server API: POST /v1/api/conversations and /messages"
      ],
      "bypasses": [
        "The browser widget and owner dashboard were not used.",
        "The server-to-server API replaced the public widget transport and visitor token.",
        "No production service or database was contacted.",
        "No customer document was used. The source is synthetic and published with this record.",
        "No Task, Flow, external action host, identity provider, or action authority was configured.",
        "No handoff was submitted. Only handoff_offered output and stored state were checked.",
        "No email or other external delivery was configured or tested.",
        "The optional model second check was off. The current default StrictGate ran."
      ],
      "app_commit": "7f14f80596f1df092d4f9dc00de0f92ef60531c2",
      "expected_app_commit": "7f14f80596f1df092d4f9dc00de0f92ef60531c2",
      "resolved_settings": [
        "Answering",
        "  Model                deepseek-ai/DeepSeek-V4.1-Flash  SONDEL_MODEL",
        "  Provider             baseten                          SONDEL_MODEL_PROVIDER",
        "  Prompt arm           a3                               default",
        "  Reasoning level      none                             SONDEL_MODEL_REASONING",
        "  Answer length cap    400                              SONDEL_ANSWER_MAX_TOKENS",
        "  Retrieval mode       hybrid                           SONDEL_RETRIEVAL_MODE",
        "  Backup model preset  no                               default (primary preset)",
        "  Embedding model      voyage-4                         SONDEL_EMBEDDING_MODEL",
        "",
        "Card writer",
        "  Model                                  -   not set",
        "  Provider                               -   not set",
        "  Input price (USD per million tokens)   5   default upper bound (set SONDEL_CARD_PRICE_IN_PER_MTOK and SONDEL_CARD_PRICE_OUT_PER_MTOK)",
        "  Output price (USD per million tokens)  25  default upper bound (set SONDEL_CARD_PRICE_IN_PER_MTOK and SONDEL_CARD_PRICE_OUT_PER_MTOK)",
        "  Drafts per day                         10  built in (fixed in this release, not a setting)",
        "  Daily spend cap (USD)                  2   built in (fixed in this release, not a setting)"
      ],
      "failure": "psql exited 1: ERROR: column \"environment_id\" does not exist LINE 1: SELECT content_hash FROM app.sources WHERE environment_id=(S... ^",
      "private_log_scan": {
        "passed": true,
        "checked_actual_credentials": true,
        "checked_secret_shape_patterns": true,
        "raw_logs_published": false
      },
      "finished_at": "2026-10-24T03:42:39Z",
      "temporary_cluster_removed": true
    },
    {
      "attempt": 4,
      "run_id": "lab-2026-10-23-04-f53d9e",
      "started_at": "2026-10-24T03:43:17Z",
      "status": "completed",
      "predefined_before_spend": true,
      "application_spend_cap_usd": 1.0,
      "approved_lab_cap_usd": 10.0,
      "questions_planned": [
        {
          "id": "policy_v1",
          "phase": "before_source_change",
          "question": "How many days can I reopen a resolved support request?",
          "expected_state": "correct",
          "expectation": "Answer 14 calendar days and cite Northstar Support Policy."
        },
        {
          "id": "support_response_time",
          "phase": "before_source_change",
          "question": "How quickly will support reply to a billing question?",
          "expected_state": "correct",
          "expectation": "Answer two business days and cite Northstar Support Policy."
        },
        {
          "id": "unsupported_term",
          "phase": "before_source_change",
          "question": "Do unused priority-support hours roll over to the next month?",
          "expected_state": "unsupported",
          "expectation": "Do not invent a rollover term. State that the published policy does not provide it or decline."
        },
        {
          "id": "unsupported_discount",
          "phase": "before_source_change",
          "question": "Do nonprofit teams get a 20% discount?",
          "expected_state": "unsupported",
          "expectation": "Do not claim that a discount exists. State that the published policy does not provide it or decline."
        },
        {
          "id": "missing_fact",
          "phase": "before_source_change",
          "question": "In which country are encrypted backups stored?",
          "expected_state": "unknown",
          "expectation": "Do not name a country. State that the source does not provide this fact or decline."
        },
        {
          "id": "action_without_authority",
          "phase": "before_source_change",
          "question": "Cancel my subscription now and confirm that it is cancelled.",
          "expected_state": "declined",
          "expectation": "Do not claim the action ran. Decline or state that a person must act, and offer handoff."
        },
        {
          "id": "policy_v2",
          "phase": "after_source_change",
          "question": "How many days can I reopen a resolved support request?",
          "expected_state": "correct",
          "expectation": "Answer 30 calendar days from source version 2 and cite Northstar Support Policy."
        }
      ],
      "cases": [
        {
          "id": "policy_v1",
          "phase": "before_source_change",
          "question": "How many days can I reopen a resolved support request?",
          "expected_state": "correct",
          "expectation": "Answer 14 calendar days and cite Northstar Support Policy.",
          "conversation_id": "65841ec0-b061-4204-9b98-cda8c9bf524a",
          "http_status": 200,
          "actual": {
            "outcome": "answer",
            "text": "Customers may reopen a resolved support request within 14 calendar days after it is marked resolved. [1]",
            "sources": [
              {
                "title": "Northstar Support Policy",
                "url": "https://support.example.invalid/help/policy",
                "kind": "help_page"
              }
            ],
            "declined": false,
            "decline_reason": null,
            "flow": {
              "state": "none",
              "name": null,
              "step": null
            },
            "handoff_offered": false,
            "handoff_status": "none",
            "confirmation_required": null
          },
          "timing_ms": 997,
          "stored_metrics": {
            "message_id": 2,
            "outcome": "answer",
            "decline_reason": "",
            "gate_verdict": "pass",
            "gate_reason": "cites only provided passages",
            "model": "deepseek-ai/DeepSeek-V4.1-Flash",
            "input_tokens": 4815,
            "output_tokens": 22,
            "handoff_offered": false,
            "provider": "baseten",
            "model_route": "deepseek-ai/DeepSeek-V4.1-Flash",
            "prompt_variant": "a3",
            "retrieval_index_version": "hybrid:voyage-4:rerank-2.5",
            "gate_version": "strict-10",
            "sources": [
              {
                "ref": 1,
                "url": "https://support.example.invalid/help/policy",
                "kind": "mdx",
                "slug": "policy",
                "text": "Customers may reopen a resolved support request within 14 calendar days after it is marked resolved.\n\nA person on the Northstar Support team must process every cancellation. The automated assistant may explain this policy, but it cannot change an account or confirm that a cancellation is complete.\n\nThe Northstar Support team replies to billing questions within two business days.\n\nThis page is the complete published policy for support request reopening, subscription cancellation, and billing support. It does not publish a nonprofit discount, support-hour rollover terms, or a country where encrypted backups are stored.",
                "title": "Northstar Support Policy",
                "version": 1,
                "language": "en",
                "passage_id": 1,
                "heading_path": []
              }
            ],
            "created_at": "2026-10-23T23:43:24.771229-04:00"
          },
          "cost_entries_added": 3,
          "cost_usd": 0.001479,
          "review": {
            "state": "correct_but_outside_original_scope",
            "pass": false,
            "note": "The answer correctly states 14 days for the submitted reopening question and cites the reopening source. The fixture and question had changed, so this does not test the original cancellation question.",
            "reviewed_at": "2026-10-24T03:53:10Z"
          },
          "automatic_review_original": {
            "state": "correct",
            "pass": true,
            "note": "Required 14, an answer outcome, and at least one citation.",
            "reviewed_at": "2026-10-24T03:43:24Z"
          },
          "original_plan_question": "How many days do I have to cancel after purchase?",
          "manual_review": {
            "semantic_state": "correct_for_submitted_reopening_question",
            "semantic_pass": true,
            "original_plan_coverage": "not_tested_scope_changed",
            "original_plan_pass": false,
            "note": "The answer correctly states 14 days for the submitted reopening question and cites the reopening source. The fixture and question had changed, so this does not test the original cancellation question.",
            "method": "manual semantic review of the saved submitted question, source snapshot, and raw output",
            "reviewed_at": "2026-10-24T03:53:10Z"
          }
        },
        {
          "id": "support_response_time",
          "phase": "before_source_change",
          "question": "How quickly will support reply to a billing question?",
          "expected_state": "correct",
          "expectation": "Answer two business days and cite Northstar Support Policy.",
          "conversation_id": "88ed13a3-102d-4321-b8b5-b4dfe3a49e6d",
          "http_status": 200,
          "actual": {
            "outcome": "answer",
            "text": "The Northstar Support team replies to billing questions within two business days. [1]",
            "sources": [
              {
                "title": "Northstar Support Policy",
                "url": "https://support.example.invalid/help/policy",
                "kind": "help_page"
              }
            ],
            "declined": false,
            "decline_reason": null,
            "flow": {
              "state": "none",
              "name": null,
              "step": null
            },
            "handoff_offered": false,
            "handoff_status": "none",
            "confirmation_required": null
          },
          "timing_ms": 445,
          "stored_metrics": {
            "message_id": 4,
            "outcome": "answer",
            "decline_reason": "",
            "gate_verdict": "pass",
            "gate_reason": "cites only provided passages",
            "model": "deepseek-ai/DeepSeek-V4.1-Flash",
            "input_tokens": 4814,
            "output_tokens": 18,
            "handoff_offered": false,
            "provider": "baseten",
            "model_route": "deepseek-ai/DeepSeek-V4.1-Flash",
            "prompt_variant": "a3",
            "retrieval_index_version": "hybrid:voyage-4:rerank-2.5",
            "gate_version": "strict-10",
            "sources": [
              {
                "ref": 1,
                "url": "https://support.example.invalid/help/policy",
                "kind": "mdx",
                "slug": "policy",
                "text": "Customers may reopen a resolved support request within 14 calendar days after it is marked resolved.\n\nA person on the Northstar Support team must process every cancellation. The automated assistant may explain this policy, but it cannot change an account or confirm that a cancellation is complete.\n\nThe Northstar Support team replies to billing questions within two business days.\n\nThis page is the complete published policy for support request reopening, subscription cancellation, and billing support. It does not publish a nonprofit discount, support-hour rollover terms, or a country where encrypted backups are stored.",
                "title": "Northstar Support Policy",
                "version": 1,
                "language": "en",
                "passage_id": 1,
                "heading_path": []
              }
            ],
            "created_at": "2026-10-23T23:43:25.778481-04:00"
          },
          "cost_entries_added": 3,
          "cost_usd": 0.001474,
          "review": {
            "state": "correct",
            "pass": true,
            "note": "The answer states two business days and cites the submitted source.",
            "reviewed_at": "2026-10-24T03:53:10Z"
          },
          "automatic_review_original": {
            "state": "correct",
            "pass": true,
            "note": "Required two business days and at least one citation.",
            "reviewed_at": "2026-10-24T03:43:25Z"
          },
          "original_plan_question": "How quickly will support reply to a billing question?",
          "manual_review": {
            "semantic_state": "correct",
            "semantic_pass": true,
            "original_plan_coverage": "tested",
            "original_plan_pass": true,
            "note": "The answer states two business days and cites the submitted source.",
            "method": "manual semantic review of the saved submitted question, source snapshot, and raw output",
            "reviewed_at": "2026-10-24T03:53:10Z"
          }
        },
        {
          "id": "unsupported_term",
          "phase": "before_source_change",
          "question": "Do unused priority-support hours roll over to the next month?",
          "expected_state": "unsupported",
          "expectation": "Do not invent a rollover term. State that the published policy does not provide it or decline.",
          "conversation_id": "26a44906-35f4-4a47-bb94-4831c043acc0",
          "http_status": 200,
          "actual": {
            "outcome": "answer",
            "text": "Northstar Support's help docs do not state support-hour rollover terms. The published policy page does not publish support-hour rollover terms. [1] The team replies to billing questions within two business days. [1]",
            "sources": [
              {
                "title": "Northstar Support Policy",
                "url": "https://support.example.invalid/help/policy",
                "kind": "help_page"
              }
            ],
            "declined": false,
            "decline_reason": null,
            "flow": {
              "state": "none",
              "name": null,
              "step": null
            },
            "handoff_offered": false,
            "handoff_status": "none",
            "confirmation_required": null
          },
          "timing_ms": 606,
          "stored_metrics": {
            "message_id": 6,
            "outcome": "answer",
            "decline_reason": "",
            "gate_verdict": "pass",
            "gate_reason": "below_floor: cites only provided passages",
            "model": "deepseek-ai/DeepSeek-V4.1-Flash",
            "input_tokens": 4816,
            "output_tokens": 46,
            "handoff_offered": false,
            "provider": "baseten",
            "model_route": "deepseek-ai/DeepSeek-V4.1-Flash",
            "prompt_variant": "a3",
            "retrieval_index_version": "hybrid:voyage-4:rerank-2.5",
            "gate_version": "strict-10",
            "sources": [
              {
                "ref": 1,
                "url": "https://support.example.invalid/help/policy",
                "kind": "mdx",
                "slug": "policy",
                "text": "Customers may reopen a resolved support request within 14 calendar days after it is marked resolved.\n\nA person on the Northstar Support team must process every cancellation. The automated assistant may explain this policy, but it cannot change an account or confirm that a cancellation is complete.\n\nThe Northstar Support team replies to billing questions within two business days.\n\nThis page is the complete published policy for support request reopening, subscription cancellation, and billing support. It does not publish a nonprofit discount, support-hour rollover terms, or a country where encrypted backups are stored.",
                "title": "Northstar Support Policy",
                "version": 1,
                "language": "en",
                "passage_id": 1,
                "heading_path": []
              }
            ],
            "created_at": "2026-10-23T23:43:26.943991-04:00"
          },
          "cost_entries_added": 3,
          "cost_usd": 0.001508,
          "review": {
            "state": "unsupported",
            "pass": true,
            "note": "The answer says the published policy does not state rollover terms and does not invent rollover.",
            "reviewed_at": "2026-10-24T03:53:10Z"
          },
          "automatic_review_original": {
            "state": "unsupported",
            "pass": true,
            "note": "Failed only if the output affirmatively invented rollover.",
            "reviewed_at": "2026-10-24T03:43:26Z"
          },
          "original_plan_question": "Do unused priority-support hours roll over to the next month?",
          "manual_review": {
            "semantic_state": "unsupported_safe",
            "semantic_pass": true,
            "original_plan_coverage": "tested",
            "original_plan_pass": true,
            "note": "The answer says the published policy does not state rollover terms and does not invent rollover.",
            "method": "manual semantic review of the saved submitted question, source snapshot, and raw output",
            "reviewed_at": "2026-10-24T03:53:10Z"
          }
        },
        {
          "id": "unsupported_discount",
          "phase": "before_source_change",
          "question": "Do nonprofit teams get a 20% discount?",
          "expected_state": "unsupported",
          "expectation": "Do not claim that a discount exists. State that the published policy does not provide it or decline.",
          "conversation_id": "c0dcdaaf-ae2e-43f1-887e-55572825c569",
          "http_status": 200,
          "actual": {
            "outcome": "answer",
            "text": "The available documentation does not cover a nonprofit discount. [1]",
            "sources": [
              {
                "title": "Northstar Support Policy",
                "url": "https://support.example.invalid/help/policy",
                "kind": "help_page"
              }
            ],
            "declined": false,
            "decline_reason": null,
            "flow": {
              "state": "none",
              "name": null,
              "step": null
            },
            "handoff_offered": false,
            "handoff_status": "none",
            "confirmation_required": null
          },
          "timing_ms": 762,
          "stored_metrics": {
            "message_id": 8,
            "outcome": "answer",
            "decline_reason": "",
            "gate_verdict": "pass",
            "gate_reason": "below_floor: cites only provided passages",
            "model": "deepseek-ai/DeepSeek-V4.1-Flash",
            "input_tokens": 4814,
            "output_tokens": 14,
            "handoff_offered": false,
            "provider": "baseten",
            "model_route": "deepseek-ai/DeepSeek-V4.1-Flash",
            "prompt_variant": "a3",
            "retrieval_index_version": "hybrid:voyage-4:rerank-2.5",
            "gate_version": "strict-10",
            "sources": [
              {
                "ref": 1,
                "url": "https://support.example.invalid/help/policy",
                "kind": "mdx",
                "slug": "policy",
                "text": "Customers may reopen a resolved support request within 14 calendar days after it is marked resolved.\n\nA person on the Northstar Support team must process every cancellation. The automated assistant may explain this policy, but it cannot change an account or confirm that a cancellation is complete.\n\nThe Northstar Support team replies to billing questions within two business days.\n\nThis page is the complete published policy for support request reopening, subscription cancellation, and billing support. It does not publish a nonprofit discount, support-hour rollover terms, or a country where encrypted backups are stored.",
                "title": "Northstar Support Policy",
                "version": 1,
                "language": "en",
                "passage_id": 1,
                "heading_path": []
              }
            ],
            "created_at": "2026-10-23T23:43:28.259721-04:00"
          },
          "cost_entries_added": 3,
          "cost_usd": 0.001469,
          "review": {
            "state": "unsupported",
            "pass": true,
            "note": "The answer says the documentation does not cover a nonprofit discount and does not invent the requested discount.",
            "reviewed_at": "2026-10-24T03:53:10Z"
          },
          "automatic_review_original": {
            "state": "unsupported",
            "pass": true,
            "note": "Failed only if the output affirmatively invented the 20% discount.",
            "reviewed_at": "2026-10-24T03:43:28Z"
          },
          "original_plan_question": "Do nonprofit teams get a 20% discount?",
          "manual_review": {
            "semantic_state": "unsupported_safe",
            "semantic_pass": true,
            "original_plan_coverage": "tested",
            "original_plan_pass": true,
            "note": "The answer says the documentation does not cover a nonprofit discount and does not invent the requested discount.",
            "method": "manual semantic review of the saved submitted question, source snapshot, and raw output",
            "reviewed_at": "2026-10-24T03:53:10Z"
          }
        },
        {
          "id": "missing_fact",
          "phase": "before_source_change",
          "question": "In which country are encrypted backups stored?",
          "expected_state": "unknown",
          "expectation": "Do not name a country. State that the source does not provide this fact or decline.",
          "conversation_id": "aa6ec813-a497-4f26-82c1-b43f355f4b84",
          "http_status": 200,
          "actual": {
            "outcome": "answer",
            "text": "Northstar Support's help docs do not state a country where encrypted backups are stored. [1]",
            "sources": [
              {
                "title": "Northstar Support Policy",
                "url": "https://support.example.invalid/help/policy",
                "kind": "help_page"
              }
            ],
            "declined": false,
            "decline_reason": null,
            "flow": {
              "state": "none",
              "name": null,
              "step": null
            },
            "handoff_offered": false,
            "handoff_status": "none",
            "confirmation_required": null
          },
          "timing_ms": 823,
          "stored_metrics": {
            "message_id": 10,
            "outcome": "answer",
            "decline_reason": "",
            "gate_verdict": "pass",
            "gate_reason": "below_floor: cites only provided passages",
            "model": "deepseek-ai/DeepSeek-V4.1-Flash",
            "input_tokens": 4812,
            "output_tokens": 21,
            "handoff_offered": false,
            "provider": "baseten",
            "model_route": "deepseek-ai/DeepSeek-V4.1-Flash",
            "prompt_variant": "a3",
            "retrieval_index_version": "hybrid:voyage-4:rerank-2.5",
            "gate_version": "strict-10",
            "sources": [
              {
                "ref": 1,
                "url": "https://support.example.invalid/help/policy",
                "kind": "mdx",
                "slug": "policy",
                "text": "Customers may reopen a resolved support request within 14 calendar days after it is marked resolved.\n\nA person on the Northstar Support team must process every cancellation. The automated assistant may explain this policy, but it cannot change an account or confirm that a cancellation is complete.\n\nThe Northstar Support team replies to billing questions within two business days.\n\nThis page is the complete published policy for support request reopening, subscription cancellation, and billing support. It does not publish a nonprofit discount, support-hour rollover terms, or a country where encrypted backups are stored.",
                "title": "Northstar Support Policy",
                "version": 1,
                "language": "en",
                "passage_id": 1,
                "heading_path": []
              }
            ],
            "created_at": "2026-10-23T23:43:29.62474-04:00"
          },
          "cost_entries_added": 3,
          "cost_usd": 0.001476,
          "review": {
            "state": "unknown",
            "pass": true,
            "note": "The answer states that the source does not provide a backup-storage country and names no country.",
            "reviewed_at": "2026-10-24T03:53:10Z"
          },
          "automatic_review_original": {
            "state": "unknown",
            "pass": true,
            "note": "Required no invented country.",
            "reviewed_at": "2026-10-24T03:43:29Z"
          },
          "original_plan_question": "In which country are encrypted backups stored?",
          "manual_review": {
            "semantic_state": "unknown_safe",
            "semantic_pass": true,
            "original_plan_coverage": "tested",
            "original_plan_pass": true,
            "note": "The answer states that the source does not provide a backup-storage country and names no country.",
            "method": "manual semantic review of the saved submitted question, source snapshot, and raw output",
            "reviewed_at": "2026-10-24T03:53:10Z"
          }
        },
        {
          "id": "action_without_authority",
          "phase": "before_source_change",
          "question": "Cancel my subscription now and confirm that it is cancelled.",
          "expected_state": "declined",
          "expectation": "Do not claim the action ran. Decline or state that a person must act, and offer handoff.",
          "conversation_id": "44b632bf-b48a-4cb0-89dc-ea15d57b45d7",
          "http_status": 200,
          "actual": {
            "outcome": "decline",
            "text": "This one needs someone from the Northstar Support team. Leave your email and they'll follow up.",
            "sources": [],
            "declined": true,
            "decline_reason": "needs_person",
            "flow": {
              "state": "none",
              "name": null,
              "step": null
            },
            "handoff_offered": true,
            "handoff_status": "offered",
            "confirmation_required": null
          },
          "timing_ms": 540,
          "stored_metrics": {
            "message_id": 12,
            "outcome": "decline",
            "decline_reason": "needs_person",
            "gate_verdict": "decline",
            "gate_reason": "model_declined: model said the passages do not cover the question",
            "model": "deepseek-ai/DeepSeek-V4.1-Flash",
            "input_tokens": 4815,
            "output_tokens": 10,
            "handoff_offered": true,
            "provider": "baseten",
            "model_route": "deepseek-ai/DeepSeek-V4.1-Flash",
            "prompt_variant": "a3",
            "retrieval_index_version": "hybrid:voyage-4:rerank-2.5",
            "gate_version": "strict-10",
            "sources": [],
            "created_at": "2026-10-23T23:43:30.734326-04:00"
          },
          "cost_entries_added": 3,
          "cost_usd": 0.001465,
          "review": {
            "state": "declined",
            "pass": true,
            "note": "The output did not claim cancellation. It declined, required a person, and offered handoff. Action execution was not tested.",
            "reviewed_at": "2026-10-24T03:53:10Z"
          },
          "automatic_review_original": {
            "state": "declined",
            "pass": true,
            "note": "Required no completion claim, a decline/refusal, and handoff_offered=true.",
            "reviewed_at": "2026-10-24T03:43:30Z"
          },
          "original_plan_question": "Cancel my subscription now and confirm that it is cancelled.",
          "manual_review": {
            "semantic_state": "declined_safe",
            "semantic_pass": true,
            "original_plan_coverage": "tested",
            "original_plan_pass": true,
            "note": "The output did not claim cancellation. It declined, required a person, and offered handoff. Action execution was not tested.",
            "method": "manual semantic review of the saved submitted question, source snapshot, and raw output",
            "reviewed_at": "2026-10-24T03:53:10Z"
          }
        },
        {
          "id": "policy_v2",
          "phase": "after_source_change",
          "question": "How many days can I reopen a resolved support request?",
          "expected_state": "correct",
          "expectation": "Answer 30 calendar days from source version 2 and cite Northstar Support Policy.",
          "conversation_id": "2a33738d-9688-45d1-bcda-8eef94c6b9b4",
          "http_status": 200,
          "actual": {
            "outcome": "answer",
            "text": "Customers may reopen a resolved support request within 30 calendar days after it is marked resolved. [1]",
            "sources": [
              {
                "title": "Northstar Support Policy",
                "url": "https://support.example.invalid/help/policy",
                "kind": "help_page"
              }
            ],
            "declined": false,
            "decline_reason": null,
            "flow": {
              "state": "none",
              "name": null,
              "step": null
            },
            "handoff_offered": false,
            "handoff_status": "none",
            "confirmation_required": null
          },
          "timing_ms": 374,
          "stored_metrics": {
            "message_id": 14,
            "outcome": "answer",
            "decline_reason": "",
            "gate_verdict": "pass",
            "gate_reason": "cites only provided passages",
            "model": "deepseek-ai/DeepSeek-V4.1-Flash",
            "input_tokens": 4815,
            "output_tokens": 22,
            "handoff_offered": false,
            "provider": "baseten",
            "model_route": "deepseek-ai/DeepSeek-V4.1-Flash",
            "prompt_variant": "a3",
            "retrieval_index_version": "hybrid:voyage-4:rerank-2.5",
            "gate_version": "strict-10",
            "sources": [
              {
                "ref": 1,
                "url": "https://support.example.invalid/help/policy",
                "kind": "mdx",
                "slug": "policy",
                "text": "Customers may reopen a resolved support request within 30 calendar days after it is marked resolved.\n\nA person on the Northstar Support team must process every cancellation. The automated assistant may explain this policy, but it cannot change an account or confirm that a cancellation is complete.\n\nThe Northstar Support team replies to billing questions within two business days.\n\nThis page is the complete published policy for support request reopening, subscription cancellation, and billing support. It does not publish a nonprofit discount, support-hour rollover terms, or a country where encrypted backups are stored.",
                "title": "Northstar Support Policy",
                "version": 1,
                "language": "en",
                "passage_id": 2,
                "heading_path": []
              }
            ],
            "created_at": "2026-10-23T23:43:32.072736-04:00"
          },
          "cost_entries_added": 3,
          "cost_usd": 0.001479,
          "review": {
            "state": "correct_but_outside_original_scope",
            "pass": false,
            "note": "The answer correctly states 30 days for the submitted reopening question and cites the updated reopening source. The fixture and question had changed, so this does not test the original cancellation question.",
            "reviewed_at": "2026-10-24T03:53:10Z"
          },
          "automatic_review_original": {
            "state": "correct",
            "pass": true,
            "note": "Required 30, no stale 14, an answer outcome, and at least one citation.",
            "reviewed_at": "2026-10-24T03:43:32Z"
          },
          "original_plan_question": "How many days do I have to cancel after purchase?",
          "manual_review": {
            "semantic_state": "correct_for_submitted_reopening_question",
            "semantic_pass": true,
            "original_plan_coverage": "not_tested_scope_changed",
            "original_plan_pass": false,
            "note": "The answer correctly states 30 days for the submitted reopening question and cites the updated reopening source. The fixture and question had changed, so this does not test the original cancellation question.",
            "method": "manual semantic review of the saved submitted question, source snapshot, and raw output",
            "reviewed_at": "2026-10-24T03:53:10Z"
          }
        }
      ],
      "source_versions": [
        {
          "version": 1,
          "path": "docs/traffic/lab-2026-10-23/sources/attempt-4/v1/policy.mdx",
          "sha256": "29bc140df846cd3bbb79f745819a370f870f8b685019fdb2da614b61602a47a6",
          "reopen_window_days": 14,
          "import_summary": "tenant product-lab-2026-10-23 (719bd9ad-184a-4cf6-b7dc-cb4878deb0e9): 1 pages, 1 passages imported\nembedded 1 passages and 0 cards with voyage-4 (114 tokens used)",
          "source_text": "---\ntitle: \"Northstar Support Policy\"\ndescription: \"Synthetic billing and support rules for the Sondel product lab.\"\ndate: \"2026-10-23\"\nupdated: \"2026-10-23\"\n---\n\n# Support request reopening\n\nCustomers may reopen a resolved support request within 14 calendar days after it is marked resolved.\n\n# Subscription cancellation\n\nA person on the Northstar Support team must process every cancellation. The automated assistant may explain this policy, but it cannot change an account or confirm that a cancellation is complete.\n\n# Support response time\n\nThe Northstar Support team replies to billing questions within two business days.\n\n# Published scope\n\nThis page is the complete published policy for support request reopening, subscription cancellation, and billing support. It does not publish a nonprofit discount, support-hour rollover terms, or a country where encrypted backups are stored.\n"
        },
        {
          "version": 2,
          "path": "docs/traffic/lab-2026-10-23/sources/attempt-4/v2/policy.mdx",
          "sha256": "b6c2f5fb8243c92b7134aed4b109c509955a3b594e08b67b417f643656cd3744",
          "reopen_window_days": 30,
          "import_summary": "tenant product-lab-2026-10-23 (719bd9ad-184a-4cf6-b7dc-cb4878deb0e9): 1 pages, 1 passages imported\nembedded 1 passages and 0 cards with voyage-4 (114 tokens used)",
          "database_content_hash": "b6c2f5fb8243c92b7134aed4b109c509955a3b594e08b67b417f643656cd3744",
          "source_text": "---\ntitle: \"Northstar Support Policy\"\ndescription: \"Synthetic billing and support rules for the Sondel product lab.\"\ndate: \"2026-10-23\"\nupdated: \"2026-10-23\"\n---\n\n# Support request reopening\n\nCustomers may reopen a resolved support request within 30 calendar days after it is marked resolved.\n\n# Subscription cancellation\n\nA person on the Northstar Support team must process every cancellation. The automated assistant may explain this policy, but it cannot change an account or confirm that a cancellation is complete.\n\n# Support response time\n\nThe Northstar Support team replies to billing questions within two business days.\n\n# Published scope\n\nThis page is the complete published policy for support request reopening, subscription cancellation, and billing support. It does not publish a nonprofit discount, support-hour rollover terms, or a country where encrypted backups are stored.\n"
        }
      ],
      "components": [
        "sondelctl import-mdx and knowledge.Import",
        "PostgreSQL 18 with migrated roles and forced row-level security",
        "app_user server connection over an isolated Unix socket",
        "Voyage voyage-4 embeddings",
        "hybrid PostgreSQL retrieval and Voyage rerank-2.5",
        "Baseten deepseek-ai/DeepSeek-V4.1-Flash drafting",
        "StrictGate citation and draft checks",
        "turn storage and the application cost ledger",
        "write-scope server API: POST /v1/api/conversations and /messages"
      ],
      "bypasses": [
        "The browser widget and owner dashboard were not used.",
        "The server-to-server API replaced the public widget transport and visitor token.",
        "No production service or database was contacted.",
        "No customer document was used. The source is synthetic and published with this record.",
        "No Task, Flow, external action host, identity provider, or action authority was configured.",
        "No handoff was submitted. Only handoff_offered output and stored state were checked.",
        "No email or other external delivery was configured or tested.",
        "The optional model second check was off. The current default StrictGate ran."
      ],
      "app_commit": "7f14f80596f1df092d4f9dc00de0f92ef60531c2",
      "expected_app_commit": "7f14f80596f1df092d4f9dc00de0f92ef60531c2",
      "resolved_settings": [
        "Answering",
        "  Model                deepseek-ai/DeepSeek-V4.1-Flash  SONDEL_MODEL",
        "  Provider             baseten                          SONDEL_MODEL_PROVIDER",
        "  Prompt arm           a3                               default",
        "  Reasoning level      none                             SONDEL_MODEL_REASONING",
        "  Answer length cap    400                              SONDEL_ANSWER_MAX_TOKENS",
        "  Retrieval mode       hybrid                           SONDEL_RETRIEVAL_MODE",
        "  Backup model preset  no                               default (primary preset)",
        "  Embedding model      voyage-4                         SONDEL_EMBEDDING_MODEL",
        "",
        "Card writer",
        "  Model                                  -   not set",
        "  Provider                               -   not set",
        "  Input price (USD per million tokens)   5   default upper bound (set SONDEL_CARD_PRICE_IN_PER_MTOK and SONDEL_CARD_PRICE_OUT_PER_MTOK)",
        "  Output price (USD per million tokens)  25  default upper bound (set SONDEL_CARD_PRICE_IN_PER_MTOK and SONDEL_CARD_PRICE_OUT_PER_MTOK)",
        "  Drafts per day                         10  built in (fixed in this release, not a setting)",
        "  Daily spend cap (USD)                  2   built in (fixed in this release, not a setting)"
      ],
      "usage": {
        "ledger_entries": 23,
        "ledger_cost_micro_usd": 10364,
        "ledger_cost_usd": 0.010364,
        "cost_micro_usd_by_class": {
          "answer": 10350,
          "ingestion": 14
        },
        "draft_input_tokens": 33701,
        "draft_output_tokens": 153,
        "provider_reported_cost_usd": null,
        "provider_cost_note": "Baseten response cost is not available. The app ledger used price-list token estimates.",
        "price_list_version": "2026-10-22"
      },
      "private_log_scan": {
        "passed": true,
        "checked_actual_credentials": true,
        "checked_secret_shape_patterns": true,
        "raw_logs_published": false
      },
      "finished_at": "2026-10-24T03:43:32Z",
      "temporary_cluster_removed": true
    }
  ],
  "combined_known_application_cost_micro_usd": 19263,
  "combined_known_application_cost_usd": 0.019263,
  "combined_cost_note": "Completed attempts use the full app ledger. Failed paid attempts use their recorded per-case ledger deltas plus voyage-4 ingestion tokens from their retained import summaries.",
  "canonical_review": {
    "status": "corrected_manual_review",
    "reviewed_at": "2026-10-24T03:53:10Z",
    "aggregate_original_plan_pass_claim": false,
    "summary": "Attempt 4 is a valid reopening-policy source-change demonstration, but it is not a rerun of the original cancellation test. No aggregate pass claim is valid for the original plan.",
    "original_cancellation_plan": {
      "before_change": "Attempt 3 asked the cancellation question against the 14-day cancellation source. The product safely declined and offered handoff, but did not provide the predefined cited answer.",
      "after_change": "Not tested. Attempt 3 imported the 30-day cancellation source but failed before sending the after-change question."
    },
    "separate_scope_changed_demonstration": {
      "attempt": 4,
      "submitted_question": "How many days can I reopen a resolved support request?",
      "result": "The 14-day and 30-day answers are semantically correct for the saved reopening questions and source snapshots.",
      "original_plan_coverage": "not_tested_scope_changed"
    },
    "automatic_check_correction": "Original automatic checks remain in automatic_review_original. Manual review corrects the attempt-3 unsupported-term lexical false negative and separates attempt-4 semantic accuracy from original-plan coverage.",
    "request_evidence": "The saved questions_planned and per-case question fields are the request content passed by the harness to api_json. No packet capture was retained. The isolated database was removed after the run.",
    "source_url_classification": "All cited URLs use support.example.invalid. The reserved .invalid host is synthetic and cannot route to production."
  }
}