Fix the edit model emitting a bad integration op + harden the edit eval

Two real bugs the edit eval surfaced (it was crashing before, so they were hidden):

1. add-integration: the prompt described the action but never named the field, so the
   model emitted `{op:"add-integration", kind:"slack"}` — the schema wants `integration`,
   so parseEditOp threw and killed the whole run (and would throw in the live app too).
   Name the field explicitly in the prompt ("integration", NOT "kind"). Verified: 6/6
   slack/jira replies now carry `integration`.

2. The eval aborted on the first un-parseable reply, hiding the other 9 cases. Catch per
   case: a bad op is a FAILURE for that case, not a crash.

Also: add the user's exact phrasing "Require CFO approval for all invoices over $50k" as a
case (must map to add-approval, a NEW gate, not set-threshold on the existing director),
and fix the remove-co-approver case, its fixture had no co-approver to remove (the model
rightly returned "none"), so the director gate now seeds Jordan Ellis and the add case
adds a different person. Live eval: 11/11.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>

Author: DylanMerigaud

eval/edit-cases.ts

@@ -31,6 +31,9 @@ export const EDIT_FIXTURE: ApprovalWorkflow = {
       when: { kind: "leaf", field: "amount", op: ">", value: 10000 },
       approverTitle: "Director",
       approverName: "Cameron Diaz",
+      // Jordan Ellis is a co-approver here so the remove-co-approver case has a real
+      // one to drop; the add-co-approver case adds a DIFFERENT person (Taylor Nguyen).
+      approvers: ["Jordan Ellis"],
       next: ["post-netsuite"],
     },
     {
@@ -74,6 +77,16 @@ export const EDIT_CASES: EditCase[] = [
       /cfo|chief financial/i.test(op.approverTitle),
     why: "a new gate above a NEW threshold (50k ≠ the existing 10k director)",
   },
+  {
+    id: "require-cfo-over",
+    instruction: "Require CFO approval for all invoices over $50k",
+    expectedOp: "add-approval",
+    check: (op) =>
+      op.op === "add-approval" &&
+      op.amountOver === 50000 &&
+      /cfo|chief financial/i.test(op.approverTitle),
+    why: "a NEW CFO gate, not raising the existing director's threshold (the phrasing without 'also' must still add, not mutate)",
+  },
   {
     id: "add-slack",
     instruction: "Send a Slack message whenever an invoice is posted",
@@ -108,13 +121,13 @@ export const EDIT_CASES: EditCase[] = [
   {
     id: "add-director-co-approver",
     instruction:
-      "The director review should also require Jordan Ellis to sign off",
+      "The director review should also require Taylor Nguyen to sign off",
     expectedOp: "add-approver",
     check: (op) =>
       op.op === "add-approver" &&
       op.stepId === "director-review" &&
-      /jordan ellis/i.test(op.approverName),
-    why: "ADD a co-approver to the director gate (keep Cameron Diaz), not replace the approver",
+      /taylor nguyen/i.test(op.approverName),
+    why: "ADD a co-approver to the director gate (keep Cameron Diaz + Jordan Ellis), not replace the approver",
   },
   {
     id: "remove-director-co-approver",

eval/edit-run.ts

@@ -62,11 +62,23 @@ const main = async (): Promise<void> => {
 
   const results: Result[] = [];
   for (const c of EDIT_CASES) {
-    const op =
-      dryRun || !planEdit
-        ? stubOp(c)
-        : await planEdit(EDIT_FIXTURE, c.instruction);
-    results.push(scoreOne(c, op));
+    // A model reply that fails to parse (bad JSON, or an op the schema rejects) is a
+    // FAILURE for that case, not a crash of the whole run, so one flaky reply doesn't
+    // hide the other nine cases' results.
+    try {
+      const op =
+        dryRun || !planEdit
+          ? stubOp(c)
+          : await planEdit(EDIT_FIXTURE, c.instruction);
+      results.push(scoreOne(c, op));
+    } catch (err) {
+      results.push({
+        id: c.id,
+        pass: false,
+        got: "error",
+        why: `${c.why}: ${err instanceof Error ? err.message : String(err)}`,
+      });
+    }
   }
 
   const idW = Math.max(8, ...results.map((r) => r.id.length));
@@ -127,7 +139,7 @@ const stubOp = (c: EditCase): WorkflowEditOp => {
       return {
         op: "add-approver",
         stepId: "director-review",
-        approverName: "Jordan Ellis",
+        approverName: "Taylor Nguyen",
       };
     case "remove-approver":
       return {

lib/workflow-edit.ts

@@ -651,7 +651,7 @@ export const proposeEdit = async (
 export const WORKFLOW_EDIT_SYSTEM_PROMPT = `You translate a plain-language instruction into ONE structured edit for a procure-to-pay approval workflow. You are given the current workflow's steps (id, label, kind, approver) and the instruction. Return a single edit op:
 
 - add-approval: a new human approval gate. Set "label" to a SHORT title only (e.g. "CFO review" or "VP sign-off"), do NOT put the threshold or department in the label, they're shown separately. Set "approverTitle" (the role, e.g. "CFO"), "amountOver" (the dollar threshold it applies above, or null for every invoice), and "department" (scope to one department like "Product", or null for any).
-- add-integration: a system action, "slack", "jira", or "netsuite", that runs after the bill posts. Set "label" to a short title (e.g. "Notify on Slack", "Open Jira ticket").
+- add-integration: a system action that runs after the bill posts. Set "integration" to exactly one of "slack" | "jira" | "netsuite" (the field is named "integration", NOT "kind"), and "label" to a short title (e.g. "Notify on Slack", "Open Jira ticket").
 - set-threshold: change an existing approval step's amount threshold. Use the step's "stepId" from the current workflow.
 - set-approver: set the PRIMARY person on an existing approval step (by "stepId"). Use for "make X the approver" / "assign X".
 - add-approver: add ANOTHER required approver to a gate that already has one (by "stepId" + "approverName"). Use for "also require X" / "add X as a co-approver" / "X should sign off too". The gate keeps its existing approver and also routes to this person.