Skip to content

Commit eb45fb2

Browse files
feat(pr4): LLM roles + multi-round loop + history injection + cost guard
Four deliverables: - config: add llm/coding_agent/history sections; add max_total_usd and max_total_tokens to hard_stops for configurable cost guard - hard_stops: accumulate total_usd/total_tokens in state; precheck and record_outcome enforce cost limits alongside iteration limits - governor: inject history[] into planner_input each round (last N reflections from ledger); history_max_entries is configurable - cli: add --loop flag; _run_loop drives multi-round evolution until any hard stop triggers; _make_governor helper deduplicates setup - roles/planner.py: LLM planner (anthropic/openai, configurable via config.json in run dir) - roles/executor.sh: coding-agent wrapper (aider/claude-code, configurable) - roles/evaluator.py: LLM evaluator; reports cost_usd + tokens_used so kernel can enforce cost guard All choices (LLM provider/model/key, coding tool) are config-driven, nothing hardcoded. 39/39 tests pass. Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
1 parent 4658aa6 commit eb45fb2

9 files changed

Lines changed: 830 additions & 36 deletions

File tree

‎evolution_kernel/cli.py‎

Lines changed: 85 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -1,18 +1,15 @@
11
"""Evolution Kernel command-line entry point.
22
3-
The CLI shape mirrors the suggested form in the project's MVP brief:
3+
Usage:
44
5-
python -m evolution_kernel.cli \
6-
--config examples/evolution.yml \
7-
--repo /path/to/target-repo \
8-
--ledger /tmp/evolution-ledger
5+
# Run once:
6+
evolution-kernel --config examples/evolution.yml --repo /path/to/repo --ledger /tmp/ledger
97
10-
Two extra modes are supported alongside this primary form:
8+
# Run until hard stops trigger (multi-round loop):
9+
evolution-kernel --config examples/evolution.yml --repo /path/to/repo --ledger /tmp/ledger --loop
1110
12-
* ``--goal goal.json`` runs the legacy direct-flags loop (no observer / scope /
13-
hard-stops) so the original golden-case tests keep working unchanged.
14-
* ``--reset`` clears the persisted hard-stop state for the given ledger and
15-
exits — used to re-enable a halted loop after a human review.
11+
# Reset hard-stop state:
12+
evolution-kernel --ledger /tmp/ledger --reset
1613
"""
1714

1815
from __future__ import annotations
@@ -32,7 +29,7 @@
3229
def main(argv: Sequence[str] | None = None) -> int:
3330
parser = argparse.ArgumentParser(
3431
prog="evolution-kernel",
35-
description="Run one Evolution Kernel experiment under MVP constraints.",
32+
description="Run Evolution Kernel experiments.",
3633
)
3734
parser.add_argument("--repo", help="Target git repository (required unless --reset)")
3835
parser.add_argument("--ledger", required=True, help="Ledger directory")
@@ -43,6 +40,7 @@ def main(argv: Sequence[str] | None = None) -> int:
4340
parser.add_argument("--executor", nargs="+", help="Executor argv (overrides config.roles.executor)")
4441
parser.add_argument("--evaluator", nargs="+", help="Evaluator argv (overrides config.roles.evaluator)")
4542
parser.add_argument("--run-id", default=None)
43+
parser.add_argument("--loop", action="store_true", help="Run until hard stops trigger (multi-round).")
4644
parser.add_argument(
4745
"--reset",
4846
action="store_true",
@@ -75,7 +73,7 @@ def _cmd_reset(args: argparse.Namespace) -> int:
7573
return 0
7674

7775

78-
def _run_with_config(args: argparse.Namespace, cfg: EvolutionConfig) -> int:
76+
def _make_governor(args: argparse.Namespace, cfg: EvolutionConfig) -> Governor:
7977
planner = tuple(args.planner) if args.planner else cfg.roles.planner
8078
executor = tuple(args.executor) if args.executor else cfg.roles.executor
8179
evaluator = tuple(args.evaluator) if args.evaluator else cfg.roles.evaluator
@@ -84,44 +82,103 @@ def _run_with_config(args: argparse.Namespace, cfg: EvolutionConfig) -> int:
8482
"error: planner/executor/evaluator must be defined in config.roles or via flags",
8583
file=sys.stderr,
8684
)
87-
return 2
85+
raise SystemExit(2)
86+
return Governor(
87+
target_repo=args.repo,
88+
ledger_dir=args.ledger,
89+
planner=RoleCommand(list(planner)),
90+
executor=RoleCommand(list(executor)),
91+
evaluator=RoleCommand(list(evaluator)),
92+
evidence_sources=cfg.evidence_sources,
93+
allowed_paths=cfg.mutation_scope.allowed_paths,
94+
config_snapshot=cfg.raw,
95+
history_max_entries=cfg.history.max_entries,
96+
)
97+
98+
99+
def _run_with_config(args: argparse.Namespace, cfg: EvolutionConfig) -> int:
100+
try:
101+
governor = _make_governor(args, cfg)
102+
except SystemExit as e:
103+
return int(e.code)
104+
105+
goal = {"name": cfg.mission, "objective": cfg.mission}
88106

107+
if args.loop:
108+
return _run_loop(args, cfg, governor, goal)
109+
110+
# Single run
89111
state = hard_stops.load_state(args.ledger)
90112
allowed, why = hard_stops.precheck(
91113
state,
92114
cfg.hard_stops.max_iterations,
93115
cfg.hard_stops.max_consecutive_failures,
116+
max_total_usd=cfg.hard_stops.max_total_usd,
117+
max_total_tokens=cfg.hard_stops.max_total_tokens,
94118
)
95119
if not allowed:
96-
# Even when blocked, leave an audit record so the ledger covers every
97-
# invocation, not just the ones that actually ran the loop.
98120
_record_halted(args.ledger, state, why)
99121
print(json.dumps({"halted": True, "reason": why}, indent=2, sort_keys=True))
100122
return 3
101123

102-
goal = {"name": cfg.mission, "objective": cfg.mission}
103-
governor = Governor(
104-
target_repo=args.repo,
105-
ledger_dir=args.ledger,
106-
planner=RoleCommand(list(planner)),
107-
executor=RoleCommand(list(executor)),
108-
evaluator=RoleCommand(list(evaluator)),
109-
evidence_sources=cfg.evidence_sources,
110-
allowed_paths=cfg.mutation_scope.allowed_paths,
111-
config_snapshot=cfg.raw,
112-
)
113124
result = governor.run_once(goal, run_id=args.run_id)
125+
cost_usd = float(result.evaluation.get("cost_usd", 0.0))
126+
tokens_used = int(result.evaluation.get("tokens_used", 0))
114127
new_state = hard_stops.record_outcome(
115128
state,
116129
accepted=result.decision.accepted,
117130
max_iterations=cfg.hard_stops.max_iterations,
118131
max_consecutive_failures=cfg.hard_stops.max_consecutive_failures,
132+
cost_usd=cost_usd,
133+
tokens_used=tokens_used,
134+
max_total_usd=cfg.hard_stops.max_total_usd,
135+
max_total_tokens=cfg.hard_stops.max_total_tokens,
119136
)
120137
hard_stops.save_state(args.ledger, new_state)
121138
_print_result(result, halted=new_state.halted, halt_reason=new_state.halt_reason)
122139
return 0
123140

124141

142+
def _run_loop(
143+
args: argparse.Namespace,
144+
cfg: EvolutionConfig,
145+
governor: Governor,
146+
goal: dict,
147+
) -> int:
148+
"""Run until hard stops trigger. Each iteration saves state immediately."""
149+
while True:
150+
state = hard_stops.load_state(args.ledger)
151+
allowed, why = hard_stops.precheck(
152+
state,
153+
cfg.hard_stops.max_iterations,
154+
cfg.hard_stops.max_consecutive_failures,
155+
max_total_usd=cfg.hard_stops.max_total_usd,
156+
max_total_tokens=cfg.hard_stops.max_total_tokens,
157+
)
158+
if not allowed:
159+
_record_halted(args.ledger, state, why)
160+
print(json.dumps({"halted": True, "reason": why}, indent=2, sort_keys=True))
161+
return 0
162+
163+
result = governor.run_once(goal)
164+
cost_usd = float(result.evaluation.get("cost_usd", 0.0))
165+
tokens_used = int(result.evaluation.get("tokens_used", 0))
166+
new_state = hard_stops.record_outcome(
167+
state,
168+
accepted=result.decision.accepted,
169+
max_iterations=cfg.hard_stops.max_iterations,
170+
max_consecutive_failures=cfg.hard_stops.max_consecutive_failures,
171+
cost_usd=cost_usd,
172+
tokens_used=tokens_used,
173+
max_total_usd=cfg.hard_stops.max_total_usd,
174+
max_total_tokens=cfg.hard_stops.max_total_tokens,
175+
)
176+
hard_stops.save_state(args.ledger, new_state)
177+
_print_result(result, halted=new_state.halted, halt_reason=new_state.halt_reason)
178+
if new_state.halted:
179+
return 0
180+
181+
125182
def _run_legacy(args: argparse.Namespace) -> int:
126183
if not (args.planner and args.executor and args.evaluator):
127184
print(
@@ -155,8 +212,9 @@ def _record_halted(
155212
"reason": reason,
156213
"iterations": state.iterations,
157214
"consecutive_failures": state.consecutive_failures,
215+
"total_usd": state.total_usd,
216+
"total_tokens": state.total_tokens,
158217
}
159-
# Suffix with sequence number to avoid collisions within the same second.
160218
base = halted_dir / f"{ts}.json"
161219
target = base
162220
n = 1

‎evolution_kernel/config.py‎

Lines changed: 85 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,17 @@
44
55
mission: "free-text statement of intent"
66
7+
llm:
8+
provider: anthropic # anthropic | openai
9+
model: claude-sonnet-4-6
10+
api_key_env: ANTHROPIC_API_KEY
11+
12+
coding_agent:
13+
tool: aider # aider | claude-code
14+
15+
history:
16+
max_entries: 10
17+
718
evidence_sources:
819
- type: file
920
path: "./metrics.json"
@@ -16,8 +27,10 @@
1627
- "tests/"
1728
1829
hard_stops:
19-
max_iterations: 3
20-
max_consecutive_failures: 2
30+
max_iterations: 10
31+
max_consecutive_failures: 3
32+
max_total_usd: 1.00 # 0.0 = unlimited
33+
max_total_tokens: 500000 # 0 = unlimited
2134
2235
Validation prefers human-readable errors over raw tracebacks so that bad configs
2336
can be fixed without reading source.
@@ -52,6 +65,8 @@ class MutationScope:
5265
class HardStops:
5366
max_iterations: int = 1
5467
max_consecutive_failures: int = 1
68+
max_total_usd: float = 0.0 # 0.0 = unlimited
69+
max_total_tokens: int = 0 # 0 = unlimited
5570

5671

5772
@dataclass(frozen=True)
@@ -61,13 +76,33 @@ class Roles:
6176
evaluator: tuple[str, ...] = ()
6277

6378

79+
@dataclass(frozen=True)
80+
class LLMConfig:
81+
provider: str = "anthropic" # anthropic | openai
82+
model: str = "claude-sonnet-4-6"
83+
api_key_env: str = "ANTHROPIC_API_KEY"
84+
85+
86+
@dataclass(frozen=True)
87+
class CodingAgentConfig:
88+
tool: str = "aider" # aider | claude-code
89+
90+
91+
@dataclass(frozen=True)
92+
class HistoryConfig:
93+
max_entries: int = 10
94+
95+
6496
@dataclass(frozen=True)
6597
class EvolutionConfig:
6698
mission: str
6799
evidence_sources: tuple[EvidenceSource, ...] = ()
68100
mutation_scope: MutationScope = field(default_factory=MutationScope)
69101
hard_stops: HardStops = field(default_factory=HardStops)
70102
roles: Roles = field(default_factory=Roles)
103+
llm: LLMConfig = field(default_factory=LLMConfig)
104+
coding_agent: CodingAgentConfig = field(default_factory=CodingAgentConfig)
105+
history: HistoryConfig = field(default_factory=HistoryConfig)
71106
raw: Mapping[str, Any] = field(default_factory=dict)
72107

73108

@@ -97,13 +132,19 @@ def parse_config(raw: Mapping[str, Any]) -> EvolutionConfig:
97132
mutation_scope = _parse_mutation_scope(raw.get("mutation_scope", {}))
98133
hard_stops = _parse_hard_stops(raw.get("hard_stops", {}))
99134
roles = _parse_roles(raw.get("roles", {}))
135+
llm = _parse_llm(raw.get("llm", {}))
136+
coding_agent = _parse_coding_agent(raw.get("coding_agent", {}))
137+
history = _parse_history(raw.get("history", {}))
100138

101139
return EvolutionConfig(
102140
mission=mission.strip(),
103141
evidence_sources=evidence_sources,
104142
mutation_scope=mutation_scope,
105143
hard_stops=hard_stops,
106144
roles=roles,
145+
llm=llm,
146+
coding_agent=coding_agent,
147+
history=history,
107148
raw=dict(raw),
108149
)
109150

@@ -182,4 +223,45 @@ def _parse_hard_stops(value: Any) -> HardStops:
182223
for label, n in (("max_iterations", max_iterations), ("max_consecutive_failures", max_failures)):
183224
if not isinstance(n, int) or isinstance(n, bool) or n < 1:
184225
raise ConfigError(f"`hard_stops.{label}` must be a positive integer, got {n!r}")
185-
return HardStops(max_iterations=max_iterations, max_consecutive_failures=max_failures)
226+
max_total_usd = float(value.get("max_total_usd", 0.0))
227+
max_total_tokens = int(value.get("max_total_tokens", 0))
228+
if max_total_usd < 0:
229+
raise ConfigError("`hard_stops.max_total_usd` must be >= 0")
230+
if max_total_tokens < 0:
231+
raise ConfigError("`hard_stops.max_total_tokens` must be >= 0")
232+
return HardStops(
233+
max_iterations=max_iterations,
234+
max_consecutive_failures=max_failures,
235+
max_total_usd=max_total_usd,
236+
max_total_tokens=max_total_tokens,
237+
)
238+
239+
240+
def _parse_llm(value: Any) -> LLMConfig:
241+
if not isinstance(value, Mapping):
242+
raise ConfigError("`llm` must be a mapping")
243+
provider = value.get("provider", "anthropic")
244+
model = value.get("model", "claude-sonnet-4-6")
245+
api_key_env = value.get("api_key_env", "ANTHROPIC_API_KEY")
246+
for label, v in (("provider", provider), ("model", model), ("api_key_env", api_key_env)):
247+
if not isinstance(v, str) or not v.strip():
248+
raise ConfigError(f"`llm.{label}` must be a non-empty string")
249+
return LLMConfig(provider=provider.strip(), model=model.strip(), api_key_env=api_key_env.strip())
250+
251+
252+
def _parse_coding_agent(value: Any) -> CodingAgentConfig:
253+
if not isinstance(value, Mapping):
254+
raise ConfigError("`coding_agent` must be a mapping")
255+
tool = value.get("tool", "aider")
256+
if not isinstance(tool, str) or not tool.strip():
257+
raise ConfigError("`coding_agent.tool` must be a non-empty string")
258+
return CodingAgentConfig(tool=tool.strip())
259+
260+
261+
def _parse_history(value: Any) -> HistoryConfig:
262+
if not isinstance(value, Mapping):
263+
raise ConfigError("`history` must be a mapping")
264+
max_entries = value.get("max_entries", 10)
265+
if not isinstance(max_entries, int) or isinstance(max_entries, bool) or max_entries < 1:
266+
raise ConfigError("`history.max_entries` must be a positive integer")
267+
return HistoryConfig(max_entries=max_entries)

‎evolution_kernel/governor.py‎

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -52,6 +52,7 @@ def __init__(
5252
evidence_sources: Sequence[EvidenceSource] = (),
5353
allowed_paths: Sequence[str] = (),
5454
config_snapshot: Mapping[str, Any] | None = None,
55+
history_max_entries: int = 10,
5556
) -> None:
5657
self.target_repo = Path(target_repo).resolve()
5758
self.ledger_dir = Path(ledger_dir).resolve()
@@ -61,6 +62,7 @@ def __init__(
6162
self.evidence_sources = tuple(evidence_sources)
6263
self.allowed_paths = tuple(allowed_paths)
6364
self.config_snapshot = dict(config_snapshot) if config_snapshot else None
65+
self.history_max_entries = history_max_entries
6466

6567
def run_once(self, goal: Mapping[str, Any], run_id: str | None = None) -> RunResult:
6668
self._ensure_git_repo()
@@ -96,6 +98,7 @@ def run_once(self, goal: Mapping[str, Any], run_id: str | None = None) -> RunRes
9698
"ledger_dir": str(self.ledger_dir),
9799
"observation_path": str(observation_path),
98100
"allowed_paths": list(self.allowed_paths),
101+
"history": self._build_history(),
99102
},
100103
)
101104
self._run_role(self.planner, run_dir / "planner_input.json", run_dir / "plan.json", worktree)
@@ -203,6 +206,29 @@ def run_once(self, goal: Mapping[str, Any], run_id: str | None = None) -> RunRes
203206
if worktree.exists():
204207
self._git("worktree", "remove", "--force", str(worktree))
205208

209+
def _build_history(self) -> list[dict]:
210+
"""Scan ledger for past run reflections; return most recent N entries."""
211+
runs_dir = self.ledger_dir / "runs"
212+
if not runs_dir.exists():
213+
return []
214+
entries = []
215+
for run_dir in sorted(runs_dir.iterdir()):
216+
reflection = run_dir / "reflection.json"
217+
if not reflection.exists():
218+
continue
219+
try:
220+
data = self._read_json(reflection)
221+
entries.append({
222+
"run_id": data.get("run_id", run_dir.name),
223+
"accepted": data.get("accepted", False),
224+
"summary": data.get("reason", ""),
225+
"metrics": data.get("metrics", {}),
226+
})
227+
except Exception:
228+
pass
229+
n = self.history_max_entries
230+
return entries[-n:] if n > 0 else entries
231+
206232
def _decide(
207233
self,
208234
evaluation: Mapping[str, Any],

0 commit comments

Comments
 (0)