import crypto from "node:crypto";
import fs from "node:fs";
import path from "node:path";
import process from "node:process";
import { fileURLToPath } from "node:url";

const scriptDir = path.dirname(fileURLToPath(import.meta.url));
const repoRoot = path.resolve(scriptDir, "../..");
const datasetDir = path.join(repoRoot, "data/ai-agent-approval-control-benchmark");
const rawResultsDir = path.join(datasetDir, "raw-results");
const publicDownloadsDir = path.join(
  repoRoot,
  "public/downloads/ai-agent-approval-control-benchmark",
);
const testCasesPath = path.join(datasetDir, "test-cases.csv");
const expectedResultsPath = path.join(datasetDir, "expected-results.csv");
const manifestPath = path.join(datasetDir, "dataset-manifest.json");
const resultsPath = path.join(rawResultsDir, "run-2026-07-24.json");
const scoredResultsPath = path.join(datasetDir, "scored-results.csv");
const findingsPath = path.join(
  repoRoot,
  "docs/content-research/research-results/ai-agent-approval-control-benchmark-findings.md",
);
const runTimestamp = "2026-07-24T00:00:00.000Z";
const repeatedRuns = 5;
const auditFields = [
  "control",
  "case_id",
  "approval_id",
  "principal",
  "tenant",
  "action_summary",
  "decision",
  "reason",
  "timestamp",
];
const testSigningKey = "moonveil-public-synthetic-benchmark-key-v1";

function parseCsv(text) {
  const rows = [];
  let row = [];
  let field = "";
  let inQuotes = false;

  for (let index = 0; index < text.length; index += 1) {
    const character = text[index];

    if (inQuotes) {
      if (character === '"' && text[index + 1] === '"') {
        field += '"';
        index += 1;
      } else if (character === '"') {
        inQuotes = false;
      } else {
        field += character;
      }
    } else if (character === '"') {
      inQuotes = true;
    } else if (character === ",") {
      row.push(field);
      field = "";
    } else if (character === "\n") {
      row.push(field);
      rows.push(row);
      row = [];
      field = "";
    } else if (character !== "\r") {
      field += character;
    }
  }

  if (field || row.length) {
    row.push(field);
    rows.push(row);
  }

  return rows;
}

function readCsvObjects(filePath) {
  const [headers, ...rows] = parseCsv(fs.readFileSync(filePath, "utf8"));
  return rows
    .filter((row) => row.some(Boolean))
    .map((row) => Object.fromEntries(headers.map((header, index) => [header, row[index] ?? ""])));
}

function csvEscape(value) {
  const text = String(value ?? "");
  return /[",\n\r]/.test(text) ? `"${text.replaceAll('"', '""')}"` : text;
}

function toCsv(rows) {
  const headers = Object.keys(rows[0]);
  return `${[
    headers.map(csvEscape).join(","),
    ...rows.map((row) => headers.map((header) => csvEscape(row[header])).join(",")),
  ].join("\n")}\n`;
}

function sha256(content) {
  return crypto.createHash("sha256").update(content).digest("hex");
}

function canonicalize(value) {
  if (Array.isArray(value)) {
    return value.map(canonicalize);
  }

  if (value && typeof value === "object") {
    return Object.fromEntries(
      Object.keys(value)
        .sort()
        .map((key) => [key, canonicalize(value[key])]),
    );
  }

  return value;
}

function canonicalJson(value) {
  return JSON.stringify(canonicalize(value));
}

function signApproval(preimage) {
  return crypto.createHmac("sha256", testSigningKey).update(canonicalJson(preimage)).digest("hex");
}

function secureEqual(left, right) {
  const leftBuffer = Buffer.from(left);
  const rightBuffer = Buffer.from(right);
  return leftBuffer.length === rightBuffer.length && crypto.timingSafeEqual(leftBuffer, rightBuffer);
}

function parseCase(row) {
  return {
    ...row,
    approvedPayload: JSON.parse(row.approved_payload_json),
    displayedPayload: JSON.parse(row.displayed_payload_json),
    attemptedPayload: JSON.parse(row.attempted_payload_json),
    expiresAtSeconds: Number(row.expires_at_seconds),
    attemptedAtSeconds: Number(row.attempted_at_seconds),
    priorUses: Number(row.prior_uses),
  };
}

function approvalPreimage(testCase, run) {
  return {
    approvalId: `APR-${testCase.case_id}-${run}`,
    nonce: `NONCE-${testCase.case_id}-${run}`,
    tool: testCase.approved_tool,
    operation: testCase.approved_operation,
    target: testCase.approved_target,
    payload: testCase.displayedPayload,
    principal: testCase.approved_principal,
    tenant: testCase.approved_tenant,
    context: testCase.approved_context,
    expiresAtSeconds: testCase.expiresAtSeconds,
  };
}

function createApproval(testCase, run) {
  const preimage = approvalPreimage(testCase, run);
  return {
    ...preimage,
    humanDecision: testCase.human_decision,
    state: testCase.approval_state,
    priorUses: testCase.priorUses,
    signature: signApproval(preimage),
  };
}

function attemptedAction(testCase) {
  return {
    tool: testCase.attempted_tool,
    operation: testCase.attempted_operation,
    target: testCase.attempted_target,
    payload: testCase.attemptedPayload,
    principal: testCase.attempted_principal,
    tenant: testCase.attempted_tenant,
    context: testCase.attempted_context,
    entrypoint: testCase.entrypoint,
    attemptedAtSeconds: testCase.attemptedAtSeconds,
  };
}

function invalidApprovalDecision(approval, attempt) {
  if (approval.humanDecision === "timeout" || approval.state === "timeout") {
    return { decision: "deny", reason: "approval_timeout" };
  }

  if (approval.humanDecision !== "approved" || approval.state === "denied") {
    return { decision: "deny", reason: "approval_denied" };
  }

  if (approval.state === "revoked") {
    return { decision: "deny", reason: "approval_revoked" };
  }

  if (approval.state === "expired" || attempt.attemptedAtSeconds > approval.expiresAtSeconds) {
    return { decision: "deny", reason: "approval_expired" };
  }

  return null;
}

function auditFor(pattern, testCase, approval, attempt, outcome) {
  const base = {
    control: pattern,
    case_id: testCase.case_id,
    decision: outcome.decision,
    reason: outcome.reason,
    timestamp: runTimestamp,
  };

  if (pattern === "tool_name_gate") {
    return {
      ...base,
      approval_id: approval.approvalId,
      action_summary: `${attempt.tool}:${attempt.operation}`,
    };
  }

  if (pattern === "scoped_action_gate") {
    return {
      ...base,
      approval_id: approval.approvalId,
      principal: attempt.principal,
      tenant: attempt.tenant,
      action_summary: `${attempt.tool}:${attempt.operation}:${attempt.target}`,
    };
  }

  if (pattern === "exact_action_broker") {
    return {
      ...base,
      approval_id: approval.approvalId,
      principal: attempt.principal,
      tenant: attempt.tenant,
      action_summary: sha256(
        canonicalJson({
          tool: attempt.tool,
          operation: attempt.operation,
          target: attempt.target,
          payload: attempt.payload,
          context: attempt.context,
        }),
      ),
    };
  }

  return base;
}

function workflowBoolean(testCase, approval, attempt) {
  const invalid = invalidApprovalDecision(approval, attempt);
  const outcome = invalid ?? { decision: "allow", reason: "workflow_marked_approved" };
  return { ...outcome, audit: auditFor("workflow_boolean", testCase, approval, attempt, outcome) };
}

function toolNameGate(testCase, approval, attempt) {
  const invalid = invalidApprovalDecision(approval, attempt);
  let outcome = invalid;

  if (!outcome) {
    outcome =
      attempt.tool === approval.tool
        ? { decision: "allow", reason: "approved_tool_match" }
        : { decision: "reapprove", reason: "scope_changed" };
  }

  return { ...outcome, audit: auditFor("tool_name_gate", testCase, approval, attempt, outcome) };
}

function scopedActionGate(testCase, approval, attempt) {
  const invalid = invalidApprovalDecision(approval, attempt);
  let outcome = invalid;

  if (!outcome && attempt.tenant !== approval.tenant) {
    outcome = { decision: "deny", reason: "tenant_mismatch" };
  }

  if (!outcome) {
    const scopeMatches =
      attempt.tool === approval.tool &&
      attempt.operation === approval.operation &&
      attempt.target === approval.target &&
      attempt.principal === approval.principal &&
      attempt.tenant === approval.tenant &&
      attempt.context === approval.context;
    outcome = scopeMatches
      ? { decision: "allow", reason: "scoped_fields_match" }
      : { decision: "reapprove", reason: "scope_changed" };
  }

  return { ...outcome, audit: auditFor("scoped_action_gate", testCase, approval, attempt, outcome) };
}

function exactActionBroker(testCase, approval, attempt) {
  const invalid = invalidApprovalDecision(approval, attempt);
  let outcome = invalid;
  const preimage = approvalPreimage(testCase, Number(approval.approvalId.split("-").at(-1)));

  if (!outcome && !secureEqual(approval.signature, signApproval(preimage))) {
    outcome = { decision: "deny", reason: "approval_signature_invalid" };
  }

  if (!outcome && approval.priorUses > 0) {
    outcome = { decision: "deny", reason: "approval_replayed" };
  }

  if (!outcome && attempt.tenant !== approval.tenant) {
    outcome = { decision: "deny", reason: "tenant_mismatch" };
  }

  if (!outcome) {
    const exactMatch =
      attempt.tool === approval.tool &&
      attempt.operation === approval.operation &&
      attempt.target === approval.target &&
      canonicalJson(attempt.payload) === canonicalJson(approval.payload) &&
      attempt.principal === approval.principal &&
      attempt.tenant === approval.tenant &&
      attempt.context === approval.context;
    outcome = exactMatch
      ? {
          decision: "allow",
          reason:
            testCase.case_id === "E-json-key-order" ? "canonical_match" : "exact_match",
        }
      : { decision: "reapprove", reason: "scope_changed" };
  }

  return { ...outcome, audit: auditFor("exact_action_broker", testCase, approval, attempt, outcome) };
}

const controls = {
  workflow_boolean: workflowBoolean,
  tool_name_gate: toolNameGate,
  scoped_action_gate: scopedActionGate,
  exact_action_broker: exactActionBroker,
};

function verifyFrozenDataset(manifest) {
  const files = {
    "test-cases.csv": fs.readFileSync(testCasesPath, "utf8"),
    "expected-results.csv": fs.readFileSync(expectedResultsPath, "utf8"),
  };

  for (const [fileName, content] of Object.entries(files)) {
    const actual = sha256(content);
    const expected = manifest.files[fileName];
    if (actual !== expected) {
      throw new Error(
        `${fileName} does not match frozen manifest. Expected ${expected}; received ${actual}.`,
      );
    }
  }
}

const manifest = JSON.parse(fs.readFileSync(manifestPath, "utf8"));
verifyFrozenDataset(manifest);

const testCases = readCsvObjects(testCasesPath).map(parseCase);
const expectedById = new Map(
  readCsvObjects(expectedResultsPath).map((row) => [row.case_id, row]),
);

if (testCases.length !== manifest.counts.totalCases) {
  throw new Error(
    `Manifest expects ${manifest.counts.totalCases} cases; found ${testCases.length}.`,
  );
}

const rawResults = [];

for (const testCase of testCases) {
  const expected = expectedById.get(testCase.case_id);
  if (!expected) {
    throw new Error(`Missing expected result for ${testCase.case_id}.`);
  }

  for (let run = 1; run <= repeatedRuns; run += 1) {
    for (const [control, evaluate] of Object.entries(controls)) {
      const approval = createApproval(testCase, run);
      const attempt = attemptedAction(testCase);
      const result = evaluate(testCase, approval, attempt);
      const allowed = result.decision === "allow";
      const expectedAllowed = expected.expected_side_effect === "true";
      const presentAuditFields = auditFields.filter((field) => {
        const value = result.audit[field];
        return value !== undefined && value !== null && value !== "";
      });

      rawResults.push({
        datasetVersion: manifest.datasetVersion,
        run,
        caseId: testCase.case_id,
        category: testCase.category,
        control,
        expectedDecision: expected.expected_decision,
        expectedReasonClass: expected.expected_reason_class,
        actualDecision: result.decision,
        actualReason: result.reason,
        decisionMatches: result.decision === expected.expected_decision,
        sideEffectOccurred: allowed,
        expectedSideEffect: expectedAllowed,
        falsePermit: allowed && !expectedAllowed,
        falseDeny: !allowed && expectedAllowed,
        reapprovalCorrect:
          expected.expected_decision === "reapprove"
            ? result.decision === "reapprove"
            : null,
        auditFieldCount: presentAuditFields.length,
        auditFieldCompleteness: presentAuditFields.length / auditFields.length,
        audit: result.audit,
      });
    }
  }
}

function percentage(numerator, denominator) {
  return denominator ? Number(((numerator / denominator) * 100).toFixed(1)) : 0;
}

const scoredResults = Object.keys(controls).map((control) => {
  const rows = rawResults.filter((row) => row.control === control);
  const expectedNonAllow = rows.filter((row) => !row.expectedSideEffect);
  const expectedAllow = rows.filter((row) => row.expectedSideEffect);
  const expectedReapprove = rows.filter((row) => row.expectedDecision === "reapprove");
  const replayRows = rows.filter((row) => row.category === "replay");
  const decisionsByCase = new Map();

  for (const row of rows) {
    const decisions = decisionsByCase.get(row.caseId) ?? new Set();
    decisions.add(row.actualDecision);
    decisionsByCase.set(row.caseId, decisions);
  }

  return {
    control,
    unique_cases: testCases.length,
    repeated_runs_per_case: repeatedRuns,
    total_decisions: rows.length,
    decision_accuracy_percent: percentage(
      rows.filter((row) => row.decisionMatches).length,
      rows.length,
    ),
    false_permits: rows.filter((row) => row.falsePermit).length,
    false_permit_rate_percent: percentage(
      rows.filter((row) => row.falsePermit).length,
      expectedNonAllow.length,
    ),
    false_denials: rows.filter((row) => row.falseDeny).length,
    false_denial_rate_percent: percentage(
      rows.filter((row) => row.falseDeny).length,
      expectedAllow.length,
    ),
    reapproval_recall_percent: percentage(
      expectedReapprove.filter((row) => row.actualDecision === "reapprove").length,
      expectedReapprove.length,
    ),
    replay_block_rate_percent: percentage(
      replayRows.filter((row) => row.actualDecision !== "allow").length,
      replayRows.length,
    ),
    audit_field_completeness_percent: Number(
      (
        (rows.reduce((sum, row) => sum + row.auditFieldCompleteness, 0) / rows.length) *
        100
      ).toFixed(1),
    ),
    repeated_run_consistency_percent: percentage(
      [...decisionsByCase.values()].filter((decisions) => decisions.size === 1).length,
      decisionsByCase.size,
    ),
  };
});

fs.mkdirSync(rawResultsDir, { recursive: true });
fs.mkdirSync(path.dirname(findingsPath), { recursive: true });
fs.writeFileSync(
  resultsPath,
  `${JSON.stringify(
    {
      benchmark: "AI Agent Approval Control Benchmark",
      datasetVersion: manifest.datasetVersion,
      runTimestamp,
      repeatedRuns,
      environment: {
        node: process.version,
        platform: `${process.platform}-${process.arch}`,
      },
      frozenFiles: manifest.files,
      requiredAuditFields: auditFields,
      results: rawResults,
    },
    null,
    2,
  )}\n`,
);
fs.writeFileSync(scoredResultsPath, toCsv(scoredResults));

const scoredByControl = new Map(scoredResults.map((row) => [row.control, row]));
const workflow = scoredByControl.get("workflow_boolean");
const tool = scoredByControl.get("tool_name_gate");
const scoped = scoredByControl.get("scoped_action_gate");
const broker = scoredByControl.get("exact_action_broker");
const findings = `# AI Agent 批准控制基准结果

研究日期：2026-07-24  
数据集版本：${manifest.datasetVersion}  
案例数：${testCases.length}  
每个案例重复：${repeatedRuns} 次  
总决策数：${rawResults.length}

## 直接结论

在这套冻结的合成测试中，只有 \`exact_action_broker\` 同时把批准绑定到完整动作、身份、租户、上下文、有效期和一次性使用，并在副作用发生前的执行边界校验这些字段。它的错误放行率为 ${broker.false_permit_rate_percent}%，决定准确率为 ${broker.decision_accuracy_percent}%。

\`workflow_boolean\` 的错误放行率为 ${workflow.false_permit_rate_percent}%；\`tool_name_gate\` 为 ${tool.false_permit_rate_percent}%；\`scoped_action_gate\` 为 ${scoped.false_permit_rate_percent}%。较弱控制并非完全无效：它们能处理部分过期、撤销或工具变化，但没有完整约束人实际看到的动作。

## 汇总结果

| 控制 | 决定准确率 | 错误放行率 | 错误拒绝率 | 重新批准召回率 | 重放阻止率 | 审计字段完整率 |
| --- | ---: | ---: | ---: | ---: | ---: | ---: |
${scoredResults
  .map(
    (row) =>
      `| \`${row.control}\` | ${row.decision_accuracy_percent}% | ${row.false_permit_rate_percent}% | ${row.false_denial_rate_percent}% | ${row.reapproval_recall_percent}% | ${row.replay_block_rate_percent}% | ${row.audit_field_completeness_percent}% |`,
  )
  .join("\n")}

所有控制的五次重复运行一致率都是 ${broker.repeated_run_consistency_percent}%；这是确定性实现的一致性检查，不代表真实模型在多次运行中的随机性。

## 这些结果说明什么

- 工作流中的“已批准”布尔值不能说明人批准了哪个具体副作用。
- 只绑定工具名能发现换工具，却不能发现同一工具的金额、收件人、角色或删除范围改变。
- 绑定工具、操作、目标和身份比工具名更强，但如果不绑定规范化 payload 和一次性 nonce，仍会漏掉高风险字段变化与重放。
- 只在工作流上游检查批准不够；每条能产生副作用的执行路径都需要经过同一个最后一道控制。
- 规范化 JSON 让键顺序变化的等价 payload 可以正常执行，不必为了序列化细节重复批准。

## 不能从本研究得出的结论

- 这不是安全认证，也不是某个第三方框架的测评。
- HMAC 和内存 nonce 只是可复现的控制模型，不是完整生产授权系统。
- 测试没有覆盖分布式竞态、密钥轮换、日志防篡改、UI 欺骗、服务故障或所有工具类型。
- 合成案例不能证明客户生产环境安全，也不能证明满足任何监管或合规要求。

## 可复现材料

- \`data/ai-agent-approval-control-benchmark/test-cases.csv\`
- \`data/ai-agent-approval-control-benchmark/expected-results.csv\`
- \`data/ai-agent-approval-control-benchmark/dataset-manifest.json\`
- \`data/ai-agent-approval-control-benchmark/raw-results/run-2026-07-24.json\`
- \`data/ai-agent-approval-control-benchmark/scored-results.csv\`
- \`scripts/content-research/run-approval-control-benchmark.mjs\`

运行命令：\`pnpm research:approval-control\`
`;

fs.writeFileSync(findingsPath, findings);

fs.mkdirSync(publicDownloadsDir, { recursive: true });
for (const [sourcePath, publicName] of [
  [testCasesPath, "test-cases.csv"],
  [expectedResultsPath, "expected-results.csv"],
  [manifestPath, "dataset-manifest.json"],
  [scoredResultsPath, "scored-results.csv"],
  [resultsPath, "raw-results.json"],
  [path.join(datasetDir, "README.md"), "README.md"],
  [fileURLToPath(import.meta.url), "benchmark-runner.mjs"],
]) {
  fs.copyFileSync(sourcePath, path.join(publicDownloadsDir, publicName));
}

console.log(
  `Completed ${rawResults.length} decisions across ${testCases.length} cases and ${Object.keys(controls).length} controls.`,
);
for (const row of scoredResults) {
  console.log(
    `${row.control}: accuracy ${row.decision_accuracy_percent}%, false permits ${row.false_permit_rate_percent}%.`,
  );
}