diff --git a/README.md b/README.md index 5ec38c8..d370154 100644 --- a/README.md +++ b/README.md @@ -22,7 +22,11 @@ ships, on disk and runnable today: - a runnable example whose `false_completion: false` is backed by a committed, real gate verdict — `.loop/artifacts/holdout-verdict.json`, not a hand-set flag. - +![The inspector scores a self-asserted DIY loop 0/weak, then the gate-backed example 90/strong — both runs live](docs/demo.gif) + +Filmed on the real tools ([`docs/demo.cast`](docs/demo.cast)); reproduce both +verdicts yourself: `python3 -m loop inspect examples/naive-loop` then +`examples/coverage-repair`. ```bash # Score a loop's proof-of-done in 5 seconds — no install, no agent, no API key: diff --git a/docs/demo.cast b/docs/demo.cast new file mode 100644 index 0000000..5eafd4a --- /dev/null +++ b/docs/demo.cast @@ -0,0 +1,294 @@ +{"version": 2, "width": 100, "height": 28, "timestamp": 1783097456, "env": {"SHELL": "/bin/bash", "TERM": "xterm-256color"}} +[0.003751, "o", "\u001b[2m#"] +[0.016676, "o", " "] +[0.029435, "o", "Y"] +[0.042222, "o", "o"] +[0.055061, "o", "u"] +[0.06797, "o", "r"] +[0.080737, "o", " "] +[0.093553, "o", "A"] +[0.10639, "o", "I"] +[0.119219, "o", " "] +[0.132319, "o", "a"] +[0.145231, "o", "g"] +[0.15805, "o", "e"] +[0.170858, "o", "n"] +[0.183655, "o", "t"] +[0.196472, "o", " "] +[0.209241, "o", "s"] +[0.222021, "o", "a"] +[0.234775, "o", "y"] +[0.24754, "o", "s"] +[0.26031, "o", " "] +[0.273078, "o", "i"] +[0.285833, "o", "t"] +[0.298632, "o", "'"] +[0.311427, "o", "s"] +[0.324233, "o", " "] +[0.337056, "o", "d"] +[0.349879, "o", "o"] +[0.362953, "o", "n"] +[0.375677, "o", "e"] +[0.388442, "o", "."] +[0.401189, "o", "\u001b[0m\r\n"] +[1.001995, "o", "\u001b[1;32m$\u001b[0m c"] +[1.030849, "o", "a"] +[1.059643, "o", "t"] +[1.088478, "o", " "] +[1.117304, "o", "e"] +[1.146208, "o", "x"] +[1.175061, "o", "a"] +[1.203856, "o", "m"] +[1.232734, "o", "p"] +[1.261495, "o", "l"] +[1.290267, "o", "e"] +[1.319042, "o", "s"] +[1.347784, "o", "/"] +[1.376593, "o", "n"] +[1.405937, "o", "a"] +[1.43423, "o", "i"] +[1.462973, "o", "v"] +[1.492358, "o", "e"] +[1.520597, "o", "-"] +[1.549373, "o", "l"] +[1.578112, "o", "o"] +[1.606945, "o", "o"] +[1.635774, "o", "p"] +[1.66462, "o", "/"] +[1.693437, "o", "."] +[1.722353, "o", "l"] +[1.751124, "o", "o"] +[1.7799, "o", "o"] +[1.808847, "o", "p"] +[1.837619, "o", "/"] +[1.866412, "o", "s"] +[1.895189, "o", "t"] +[1.923942, "o", "a"] +[1.952691, "o", "t"] +[1.982368, "o", "e"] +[2.012155, "o", "."] +[2.040805, "o", "j"] +[2.069739, "o", "s"] +[2.098494, "o", "o"] +[2.127332, "o", "n"] +[2.506989, "o", "\r\n"] +[2.511613, "o", "{\r\n \"status\": \"completed\",\r\n \"done\": true,\r\n \"notes\": \"agent says all tasks finished\"\r\n}\r\n"] +[4.113441, "o", "\u001b[1;32m$\u001b[0m p"] +[4.14235, "o", "y"] +[4.171136, "o", "t"] +[4.199958, "o", "h"] +[4.228824, "o", "o"] +[4.257596, "o", "n"] +[4.286391, "o", "3"] +[4.315351, "o", " "] +[4.344191, "o", "-"] +[4.373071, "o", "m"] +[4.401863, "o", " "] +[4.430725, "o", "l"] +[4.45956, "o", "o"] +[4.488591, "o", "o"] +[4.517393, "o", "p"] +[4.546189, "o", " "] +[4.575006, "o", "i"] +[4.603793, "o", "n"] +[4.632665, "o", "s"] +[4.661456, "o", "p"] +[4.690361, "o", "e"] +[4.719184, "o", "c"] +[4.747997, "o", "t"] +[4.777803, "o", " "] +[4.806698, "o", "e"] +[4.835557, "o", "x"] +[4.864354, "o", "a"] +[4.893282, "o", "m"] +[4.92208, "o", "p"] +[4.950892, "o", "l"] +[4.97978, "o", "e"] +[5.008596, "o", "s"] +[5.037428, "o", "/"] +[5.066309, "o", "n"] +[5.095141, "o", "a"] +[5.12399, "o", "i"] +[5.152872, "o", "v"] +[5.181711, "o", "e"] +[5.210541, "o", "-"] +[5.239352, "o", "l"] +[5.268105, "o", "o"] +[5.297315, "o", "o"] +[5.325895, "o", "p"] +[5.705481, "o", "\r\n"] +[5.850314, "o", "{\r\n \"target\": \"examples/naive-loop\",\r\n \"score\": 0,\r\n \"terminal_states_covered\": 0,\r\n \"present\": [],\r\n \"gaps\": [\r\n \"no defined success criteria (SPEC.md ## Success Criteria) \\u2014 loop can only claim completion\",\r\n \"no independent verification (verify-* script / TASKS verify command) \\u2014 success is self-asserted\",\r\n \"no approval gates declared for side-effects (destructive / secret / production / money)\",\r\n \"no false-completion defense: no recorded holdout/anti-cheat invocation (a self-asserted false_completion flag or prose mention earns no credit)\",\r\n \"no plan-then-execute discipline for untrusted/web reads (prompt-injection surface)\",\r\n \"0/7 terminal states present \\u2014 missing Succeeded, FailedUnverifiable, FailedBlocked, FailedBudget, FailedSafety, FailedSpecGap, AbortedByHuman (loop can end in a silent 'completed')\"\r\n ],\r\n \"verdict\": \"weak\"\r\n}\r\n"] +[9.255784, "o", "\u001b[H\u001b[2J\u001b[3J"] +[9.25588, "o", "\u001b[2m#"] +[9.268602, "o", " "] +[9.281358, "o", "L"] +[9.29426, "o", "o"] +[9.30705, "o", "o"] +[9.319937, "o", "p"] +[9.332718, "o", " "] +[9.345452, "o", "E"] +[9.358195, "o", "n"] +[9.370952, "o", "g"] +[9.383759, "o", "i"] +[9.396464, "o", "n"] +[9.409163, "o", "e"] +[9.421807, "o", "e"] +[9.434483, "o", "r"] +[9.447207, "o", " "] +[9.459948, "o", "m"] +[9.472705, "o", "a"] +[9.485576, "o", "k"] +[9.498421, "o", "e"] +[9.511327, "o", "s"] +[9.524072, "o", " "] +[9.536828, "o", "i"] +[9.549566, "o", "t"] +[9.562332, "o", " "] +[9.575032, "o", "p"] +[9.587863, "o", "r"] +[9.600534, "o", "o"] +[9.613313, "o", "v"] +[9.625944, "o", "e"] +[9.63865, "o", " "] +[9.65131, "o", "i"] +[9.664392, "o", "t"] +[9.677131, "o", "."] +[9.689887, "o", "\u001b[0m\r\n"] +[10.290722, "o", "\u001b[1;32m$\u001b[0m p"] +[10.319571, "o", "y"] +[10.348421, "o", "t"] +[10.377218, "o", "h"] +[10.406007, "o", "o"] +[10.434802, "o", "n"] +[10.463636, "o", "3"] +[10.492541, "o", " "] +[10.521364, "o", "-"] +[10.550167, "o", "m"] +[10.578962, "o", " "] +[10.607764, "o", "l"] +[10.636578, "o", "o"] +[10.665368, "o", "o"] +[10.694157, "o", "p"] +[10.722998, "o", " "] +[10.751738, "o", "i"] +[10.780515, "o", "n"] +[10.809568, "o", "s"] +[10.838327, "o", "p"] +[10.867052, "o", "e"] +[10.895859, "o", "c"] +[10.924722, "o", "t"] +[10.953502, "o", " "] +[10.982316, "o", "e"] +[11.011142, "o", "x"] +[11.03992, "o", "a"] +[11.068709, "o", "m"] +[11.097483, "o", "p"] +[11.126272, "o", "l"] +[11.155055, "o", "e"] +[11.183836, "o", "s"] +[11.212676, "o", "/"] +[11.241588, "o", "c"] +[11.270561, "o", "o"] +[11.299327, "o", "v"] +[11.328182, "o", "e"] +[11.357096, "o", "r"] +[11.385915, "o", "a"] +[11.414698, "o", "g"] +[11.443447, "o", "e"] +[11.472268, "o", "-"] +[11.501038, "o", "r"] +[11.529851, "o", "e"] +[11.55883, "o", "p"] +[11.587552, "o", "a"] +[11.616362, "o", "i"] +[11.645211, "o", "r"] +[12.024817, "o", "\r\n"] +[12.118224, "o", "{\r\n \"target\": \"examples/coverage-repair\",\r\n \"score\": 90,\r\n \"terminal_states_covered\": 7,\r\n \"present\": [\r\n \"defines verifiable success criteria\",\r\n \"independent verification\",\r\n \"approval gates on side-effects\",\r\n \"false-completion defense (invoked)\",\r\n \"all 7 terminal states reachable\"\r\n ],\r\n \"gaps\": [\r\n \"no plan-then-execute discipline for untrusted/web reads (prompt-injection surface)\"\r\n ],\r\n \"verdict\": \"strong\"\r\n}\r\n"] +[15.722909, "o", "\u001b[2m#"] +[15.735923, "o", " "] +[15.748788, "o", "s"] +[15.761587, "o", "c"] +[15.774528, "o", "o"] +[15.787357, "o", "r"] +[15.800178, "o", "e"] +[15.812991, "o", " "] +[15.825809, "o", "9"] +[15.838603, "o", "0"] +[15.851411, "o", " "] +[15.864135, "o", "—"] +[15.87686, "o", " "] +[15.88957, "o", "f"] +[15.902248, "o", "a"] +[15.914984, "o", "l"] +[15.92775, "o", "s"] +[15.940575, "o", "e"] +[15.953392, "o", "_"] +[15.966352, "o", "c"] +[15.979139, "o", "o"] +[15.991949, "o", "m"] +[16.004824, "o", "p"] +[16.017624, "o", "l"] +[16.03035, "o", "e"] +[16.043084, "o", "t"] +[16.055833, "o", "i"] +[16.068678, "o", "o"] +[16.081457, "o", "n"] +[16.094194, "o", ":"] +[16.106978, "o", " "] +[16.119801, "o", "f"] +[16.132607, "o", "a"] +[16.145355, "o", "l"] +[16.158085, "o", "s"] +[16.170928, "o", "e"] +[16.183701, "o", " "] +[16.196437, "o", "i"] +[16.20918, "o", "s"] +[16.221979, "o", " "] +[16.234911, "o", "b"] +[16.24771, "o", "a"] +[16.260481, "o", "c"] +[16.273261, "o", "k"] +[16.286136, "o", "e"] +[16.298857, "o", "d"] +[16.311649, "o", " "] +[16.324488, "o", "b"] +[16.337334, "o", "y"] +[16.350149, "o", " "] +[16.36288, "o", "a"] +[16.375686, "o", " "] +[16.388435, "o", "c"] +[16.401181, "o", "o"] +[16.414035, "o", "m"] +[16.426839, "o", "m"] +[16.439588, "o", "i"] +[16.452318, "o", "t"] +[16.465088, "o", "t"] +[16.477849, "o", "e"] +[16.490606, "o", "d"] +[16.503385, "o", " "] +[16.516165, "o", "g"] +[16.528997, "o", "a"] +[16.541769, "o", "t"] +[16.554567, "o", "e"] +[16.567545, "o", " "] +[16.580391, "o", "v"] +[16.594163, "o", "e"] +[16.607823, "o", "r"] +[16.620733, "o", "d"] +[16.633533, "o", "i"] +[16.646314, "o", "c"] +[16.65907, "o", "t"] +[16.671844, "o", ","] +[16.684641, "o", " "] +[16.697391, "o", "n"] +[16.710168, "o", "o"] +[16.723132, "o", "t"] +[16.735938, "o", " "] +[16.74867, "o", "a"] +[16.761538, "o", " "] +[16.774569, "o", "f"] +[16.787414, "o", "l"] +[16.800221, "o", "a"] +[16.813, "o", "g"] +[16.825771, "o", "."] +[16.838538, "o", "\u001b[0m\r\n"] diff --git a/docs/demo.gif b/docs/demo.gif new file mode 100644 index 0000000..3f02236 Binary files /dev/null and b/docs/demo.gif differ diff --git a/docs/social-card.png b/docs/social-card.png new file mode 100644 index 0000000..9c0063f Binary files /dev/null and b/docs/social-card.png differ diff --git a/examples/naive-loop/.loop/state.json b/examples/naive-loop/.loop/state.json new file mode 100644 index 0000000..aac1f5e --- /dev/null +++ b/examples/naive-loop/.loop/state.json @@ -0,0 +1,5 @@ +{ + "status": "completed", + "done": true, + "notes": "agent says all tasks finished" +} diff --git a/examples/naive-loop/README.md b/examples/naive-loop/README.md new file mode 100644 index 0000000..502c821 --- /dev/null +++ b/examples/naive-loop/README.md @@ -0,0 +1,22 @@ +# naive-loop — the deliberately weak fixture + +This is the "before" subject of the demo GIF: a typical DIY agent loop whose +entire completion story is a self-asserted flag in `.loop/state.json`. No +success criteria, no independent verification, no approval gates, no typed +terminal states — just `"status": "completed"`. + +Score it yourself: + +```bash +python3 -m loop inspect examples/naive-loop +``` + +The inspector gives it **score 0, verdict `weak`** (exit code 1) and lists every +missing proof mechanism. Contrast with the gate-backed example: + +```bash +python3 -m loop inspect examples/coverage-repair # score 90, verdict strong +``` + +Nothing here is a strawman: this shape — a status field somewhere that the +agent itself sets — is how most ad-hoc agent loops record "done" today.